mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 00:38:14 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
import { createHash, timingSafeEqual } from 'crypto';
|
||||
import type {
|
||||
LegacyPanelAuthSnapshot,
|
||||
LegacyPanelAuthSnapshotReader,
|
||||
LegacyPanelPlatform,
|
||||
LegacyPanelSessionSource,
|
||||
} from '../../ports/legacyPanelSessionSource';
|
||||
|
||||
export const MAX_LEGACY_PANEL_TOKEN_LENGTH = 4096;
|
||||
export const MAX_LEGACY_PANEL_TOKENS_PER_PLATFORM = 64;
|
||||
|
||||
export class LegacyPanelSessionUnavailableError extends Error {
|
||||
readonly code = 'LEGACY_PANEL_SESSION_UNAVAILABLE';
|
||||
|
||||
constructor() {
|
||||
super('Legacy panel session state is unavailable');
|
||||
this.name = 'LegacyPanelSessionUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function assertToken(value: string): void {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > MAX_LEGACY_PANEL_TOKEN_LENGTH
|
||||
) {
|
||||
throw new LegacyPanelSessionUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function tokenMatches(left: string, right: string): boolean {
|
||||
assertToken(left);
|
||||
assertToken(right);
|
||||
return timingSafeEqual(
|
||||
createHash('sha256').update(left, 'utf8').digest(),
|
||||
createHash('sha256').update(right, 'utf8').digest(),
|
||||
);
|
||||
}
|
||||
|
||||
function candidates(
|
||||
snapshot: Readonly<LegacyPanelAuthSnapshot>,
|
||||
platform: LegacyPanelPlatform,
|
||||
): string[] {
|
||||
const result: string[] = [];
|
||||
if (snapshot.token !== undefined && snapshot.token !== '') {
|
||||
assertToken(snapshot.token);
|
||||
result.push(snapshot.token);
|
||||
}
|
||||
if (snapshot.tokens === undefined) return result;
|
||||
if (
|
||||
!snapshot.tokens ||
|
||||
typeof snapshot.tokens !== 'object' ||
|
||||
Array.isArray(snapshot.tokens)
|
||||
) {
|
||||
throw new LegacyPanelSessionUnavailableError();
|
||||
}
|
||||
const platformTokens = snapshot.tokens[platform];
|
||||
if (platformTokens === null || platformTokens === undefined) return result;
|
||||
if (typeof platformTokens === 'string') {
|
||||
if (platformTokens === '') return result;
|
||||
assertToken(platformTokens);
|
||||
result.push(platformTokens);
|
||||
return result;
|
||||
}
|
||||
if (
|
||||
!Array.isArray(platformTokens) ||
|
||||
platformTokens.length > MAX_LEGACY_PANEL_TOKENS_PER_PLATFORM
|
||||
) {
|
||||
throw new LegacyPanelSessionUnavailableError();
|
||||
}
|
||||
for (const item of platformTokens) {
|
||||
if (!item || typeof item !== 'object' || Array.isArray(item)) {
|
||||
throw new LegacyPanelSessionUnavailableError();
|
||||
}
|
||||
assertToken(item.value);
|
||||
result.push(item.value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export class LegacyAuthInfoSessionSource implements LegacyPanelSessionSource {
|
||||
constructor(private readonly read: LegacyPanelAuthSnapshotReader) {
|
||||
if (typeof read !== 'function') {
|
||||
throw new TypeError('Legacy panel auth snapshot reader is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async isActive(
|
||||
token: string,
|
||||
platform: LegacyPanelPlatform,
|
||||
): Promise<boolean> {
|
||||
assertToken(token);
|
||||
if (platform !== 'desktop' && platform !== 'mobile') {
|
||||
throw new TypeError('Legacy panel platform is invalid');
|
||||
}
|
||||
let snapshot: Readonly<LegacyPanelAuthSnapshot> | null;
|
||||
try {
|
||||
snapshot = await this.read();
|
||||
} catch {
|
||||
throw new LegacyPanelSessionUnavailableError();
|
||||
}
|
||||
if (!snapshot) return false;
|
||||
if (typeof snapshot !== 'object' || Array.isArray(snapshot)) {
|
||||
throw new LegacyPanelSessionUnavailableError();
|
||||
}
|
||||
return candidates(snapshot, platform).some((candidate) =>
|
||||
tokenMatches(candidate, token),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import { createCipheriv, createDecipheriv, randomBytes } from 'crypto';
|
||||
import { TextDecoder } from 'util';
|
||||
import {
|
||||
LOCAL_SECRET_ALGORITHM,
|
||||
LocalSecretUnavailableError,
|
||||
localSecretBinary,
|
||||
localSecretEnvelopeAad,
|
||||
normalizeLocalSecretEnvelope,
|
||||
type LocalSecretEnvelope,
|
||||
} from '../../domain/localSecret';
|
||||
|
||||
const SECRET_KEY_BYTES = 32;
|
||||
const SECRET_NONCE_BYTES = 12;
|
||||
const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
|
||||
|
||||
export type LocalSecretNonceFactory = () => Uint8Array;
|
||||
|
||||
function ownedSecretKey(key: Uint8Array): Buffer {
|
||||
if (!(key instanceof Uint8Array) || key.byteLength !== SECRET_KEY_BYTES) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
return Buffer.from(key);
|
||||
}
|
||||
|
||||
export function encryptLocalSecretEnvelope(
|
||||
metadata: Omit<LocalSecretEnvelope, 'nonce' | 'ciphertext' | 'authTag'>,
|
||||
plaintext: string,
|
||||
key: Uint8Array,
|
||||
nonceFactory: LocalSecretNonceFactory = () => randomBytes(SECRET_NONCE_BYTES),
|
||||
): LocalSecretEnvelope {
|
||||
const ownedKey = ownedSecretKey(key);
|
||||
const plaintextBuffer = Buffer.from(plaintext, 'utf8');
|
||||
let nonce: Buffer | undefined;
|
||||
try {
|
||||
nonce = Buffer.from(nonceFactory());
|
||||
if (nonce.length !== SECRET_NONCE_BYTES) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
const cipher = createCipheriv(LOCAL_SECRET_ALGORITHM, ownedKey, nonce, {
|
||||
authTagLength: 16,
|
||||
});
|
||||
cipher.setAAD(localSecretEnvelopeAad(metadata));
|
||||
const ciphertext = Buffer.concat([
|
||||
cipher.update(plaintextBuffer),
|
||||
cipher.final(),
|
||||
]);
|
||||
return normalizeLocalSecretEnvelope({
|
||||
...metadata,
|
||||
nonce: nonce.toString('base64url'),
|
||||
ciphertext: ciphertext.toString('base64url'),
|
||||
authTag: cipher.getAuthTag().toString('base64url'),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof LocalSecretUnavailableError) throw error;
|
||||
throw new LocalSecretUnavailableError();
|
||||
} finally {
|
||||
ownedKey.fill(0);
|
||||
plaintextBuffer.fill(0);
|
||||
nonce?.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
export function decryptLocalSecretEnvelopeToBuffer(
|
||||
envelope: LocalSecretEnvelope,
|
||||
key: Uint8Array,
|
||||
): Buffer {
|
||||
const normalized = normalizeLocalSecretEnvelope(envelope);
|
||||
const ownedKey = ownedSecretKey(key);
|
||||
const nonce = localSecretBinary('nonce', normalized.nonce);
|
||||
const ciphertext = localSecretBinary('ciphertext', normalized.ciphertext);
|
||||
const authTag = localSecretBinary('authTag', normalized.authTag);
|
||||
try {
|
||||
const decipher = createDecipheriv(LOCAL_SECRET_ALGORITHM, ownedKey, nonce, {
|
||||
authTagLength: 16,
|
||||
});
|
||||
decipher.setAAD(localSecretEnvelopeAad(normalized));
|
||||
decipher.setAuthTag(authTag);
|
||||
return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
|
||||
} catch {
|
||||
throw new LocalSecretUnavailableError();
|
||||
} finally {
|
||||
ownedKey.fill(0);
|
||||
nonce.fill(0);
|
||||
authTag.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
export function decodeLocalSecretPlaintext(plaintext: Buffer): string {
|
||||
try {
|
||||
return UTF8_DECODER.decode(plaintext);
|
||||
} catch {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { CompletionReceipt } from '../../domain/completionReceipt';
|
||||
import { matchesWorkerExecutionCompletionReceiptAuthentication } from '../../domain/workerExecutionCompletionReceiptAuthentication';
|
||||
import type { WorkerExecutionOfferJournalRecord } from '../../domain/workerExecutionOffer';
|
||||
import type { WorkerExecutionCompletionReceiptAuthenticator } from '../../ports/workerExecutionCompletionReceiptAuthenticator';
|
||||
|
||||
export class Sha256WorkerExecutionCompletionReceiptAuthenticator
|
||||
implements WorkerExecutionCompletionReceiptAuthenticator
|
||||
{
|
||||
authenticate(
|
||||
receipt: CompletionReceipt,
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
): boolean {
|
||||
if (
|
||||
record.completionReceiptCallbackSequence === undefined ||
|
||||
record.completionReceiptTokenDigest === undefined
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return matchesWorkerExecutionCompletionReceiptAuthentication(receipt, {
|
||||
callbackSequence: record.completionReceiptCallbackSequence,
|
||||
tokenDigest: record.completionReceiptTokenDigest,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { randomBytes } from 'crypto';
|
||||
import {
|
||||
assertCompletionReceiptId,
|
||||
InvalidCompletionReceiptError,
|
||||
MAX_COMPLETION_RECEIPT_BYTES,
|
||||
parseCompletionReceipt,
|
||||
serializeCompletionReceipt,
|
||||
type CompletionReceipt,
|
||||
} from '../../domain/completionReceipt';
|
||||
import type { CompletionReceiptStore } from '../../ports/completionReceiptStore';
|
||||
|
||||
export class CompletionReceiptAlreadyExistsError extends Error {
|
||||
constructor(readonly attemptId: string) {
|
||||
super(`Completion receipt already exists for Attempt ${attemptId}`);
|
||||
this.name = 'CompletionReceiptAlreadyExistsError';
|
||||
}
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
(error as { code?: string }).code === code
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A bounded local journal. It does not scan directories or own a lifecycle;
|
||||
* callers must discover active Attempts from the database.
|
||||
*/
|
||||
export class CompletionReceiptFileStore implements CompletionReceiptStore {
|
||||
constructor(private readonly root: string) {
|
||||
if (!path.isAbsolute(root)) {
|
||||
throw new RangeError('Completion receipt root must be absolute');
|
||||
}
|
||||
}
|
||||
|
||||
async publish(receipt: CompletionReceipt): Promise<void> {
|
||||
const serialized = serializeCompletionReceipt(receipt);
|
||||
const directory = this.directory(receipt.attemptId);
|
||||
const target = this.target(receipt.attemptId);
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const temporary = path.join(
|
||||
directory,
|
||||
`.${receipt.attemptId}.${randomBytes(16).toString('hex')}.tmp`,
|
||||
);
|
||||
let handle: fs.FileHandle | undefined;
|
||||
try {
|
||||
handle = await fs.open(temporary, 'wx', 0o600);
|
||||
await handle.writeFile(serialized, 'utf8');
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = undefined;
|
||||
|
||||
// A hard link publishes the fully written inode atomically and, unlike
|
||||
// plain rename(), never replaces an existing completion fact.
|
||||
try {
|
||||
await fs.link(temporary, target);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'EEXIST')) {
|
||||
throw new CompletionReceiptAlreadyExistsError(receipt.attemptId);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
await this.bestEffortUnlink(temporary);
|
||||
await this.bestEffortSyncDirectory(directory);
|
||||
} catch (error) {
|
||||
await handle?.close().catch(() => undefined);
|
||||
await this.bestEffortUnlink(temporary);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async read(attemptId: string): Promise<CompletionReceipt | undefined> {
|
||||
const target = this.target(attemptId);
|
||||
let handle: fs.FileHandle;
|
||||
try {
|
||||
handle = await fs.open(
|
||||
target,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return undefined;
|
||||
if (isCode(error, 'ELOOP')) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt must not be a symbolic link',
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (!stat.isFile()) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt must be a regular file',
|
||||
);
|
||||
}
|
||||
const bytes = Buffer.allocUnsafe(MAX_COMPLETION_RECEIPT_BYTES + 1);
|
||||
const { bytesRead } = await handle.read(bytes, 0, bytes.length, 0);
|
||||
if (bytesRead > MAX_COMPLETION_RECEIPT_BYTES) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt exceeds the byte limit',
|
||||
);
|
||||
}
|
||||
const receipt = parseCompletionReceipt(bytes.subarray(0, bytesRead));
|
||||
if (receipt.attemptId !== attemptId) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt path and Attempt do not match',
|
||||
);
|
||||
}
|
||||
return receipt;
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
async remove(attemptId: string): Promise<boolean> {
|
||||
const target = this.target(attemptId);
|
||||
try {
|
||||
await fs.unlink(target);
|
||||
await this.bestEffortSyncDirectory(path.dirname(target));
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async quarantine(attemptId: string): Promise<string | undefined> {
|
||||
const target = this.target(attemptId);
|
||||
const reference = this.quarantineReference(attemptId);
|
||||
const relativeDirectory = path.dirname(reference);
|
||||
const directory = path.join(this.root, relativeDirectory);
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
await fs.chmod(directory, 0o700);
|
||||
const quarantined = path.join(this.root, reference);
|
||||
try {
|
||||
await fs.link(target, quarantined);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) {
|
||||
return (await this.pathExists(quarantined)) ? reference : undefined;
|
||||
}
|
||||
if (!isCode(error, 'EEXIST')) throw error;
|
||||
}
|
||||
await this.unlinkIfPresent(target);
|
||||
await this.bestEffortSyncDirectory(path.dirname(target));
|
||||
await this.bestEffortSyncDirectory(directory);
|
||||
return reference;
|
||||
}
|
||||
|
||||
quarantineReference(attemptId: string): string {
|
||||
assertCompletionReceiptId(attemptId, 'attemptId');
|
||||
return path.posix.join(
|
||||
'.quarantine',
|
||||
attemptId.slice(0, 2),
|
||||
`${attemptId}.json`,
|
||||
);
|
||||
}
|
||||
|
||||
async purgeQuarantine(attemptId: string): Promise<boolean> {
|
||||
const target = path.join(this.root, this.quarantineReference(attemptId));
|
||||
try {
|
||||
await fs.unlink(target);
|
||||
await this.bestEffortSyncDirectory(path.dirname(target));
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private directory(attemptId: string): string {
|
||||
assertCompletionReceiptId(attemptId, 'attemptId');
|
||||
return path.join(this.root, attemptId.slice(0, 2));
|
||||
}
|
||||
|
||||
private target(attemptId: string): string {
|
||||
return path.join(this.directory(attemptId), `${attemptId}.json`);
|
||||
}
|
||||
|
||||
private async bestEffortUnlink(value: string): Promise<void> {
|
||||
try {
|
||||
await fs.unlink(value);
|
||||
} catch (error) {
|
||||
if (!isCode(error, 'ENOENT')) {
|
||||
// Temp cleanup is diagnostic-only; the immutable final fact wins.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async unlinkIfPresent(value: string): Promise<void> {
|
||||
try {
|
||||
await fs.unlink(value);
|
||||
} catch (error) {
|
||||
if (!isCode(error, 'ENOENT')) throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async pathExists(value: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.lstat(value);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async bestEffortSyncDirectory(directory: string): Promise<void> {
|
||||
try {
|
||||
const handle = await fs.open(directory, constants.O_RDONLY);
|
||||
try {
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
} catch {
|
||||
// Some supported filesystems cannot fsync directories. The receipt
|
||||
// remains restart-safe, while power-loss durability is best effort.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
import { createHash } from 'crypto';
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { assertCompletionReceiptId } from '../../domain/completionReceipt';
|
||||
import type {
|
||||
CompletionReceiptDirectoryEntry,
|
||||
CompletionReceiptDirectoryEntryKind,
|
||||
CompletionReceiptOrphanDirectory,
|
||||
CompletionReceiptOrphanQuarantineResult,
|
||||
CompletionReceiptShardSnapshot,
|
||||
} from '../../ports/completionReceiptOrphanMaintenance';
|
||||
|
||||
const SHARD_PATTERN = /^[0-9a-f]{2}$/;
|
||||
const TEMPORARY_PATTERN = /^\.([0-9a-f-]{36})\.[0-9a-f]{32}\.tmp$/;
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
(error as { code?: string }).code === code
|
||||
);
|
||||
}
|
||||
|
||||
function receiptAttemptId(name: string): string | undefined {
|
||||
if (!name.endsWith('.json')) return undefined;
|
||||
const attemptId = name.slice(0, -'.json'.length);
|
||||
try {
|
||||
assertCompletionReceiptId(attemptId, 'attemptId');
|
||||
return attemptId;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function temporaryAttemptId(name: string): string | undefined {
|
||||
const match = TEMPORARY_PATTERN.exec(name);
|
||||
if (!match) return undefined;
|
||||
try {
|
||||
assertCompletionReceiptId(match[1], 'attemptId');
|
||||
return match[1];
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function entryKind(
|
||||
shard: string,
|
||||
name: string,
|
||||
regularFile: boolean,
|
||||
): { kind: CompletionReceiptDirectoryEntryKind; attemptId?: string } {
|
||||
if (!regularFile) return { kind: 'unsafe' };
|
||||
const finalAttemptId = receiptAttemptId(name);
|
||||
if (finalAttemptId && finalAttemptId.startsWith(shard)) {
|
||||
return { kind: 'receipt', attemptId: finalAttemptId };
|
||||
}
|
||||
const tempAttemptId = temporaryAttemptId(name);
|
||||
if (tempAttemptId && tempAttemptId.startsWith(shard)) {
|
||||
return { kind: 'temporary', attemptId: tempAttemptId };
|
||||
}
|
||||
return { kind: 'unknown' };
|
||||
}
|
||||
|
||||
function filesystemIdentity(
|
||||
stat: Awaited<ReturnType<typeof fs.lstat>>,
|
||||
): string {
|
||||
return [stat.dev, stat.ino, stat.size, stat.mtimeMs].join(':');
|
||||
}
|
||||
|
||||
export class CompletionReceiptOrphanFileDirectory
|
||||
implements CompletionReceiptOrphanDirectory
|
||||
{
|
||||
constructor(private readonly root: string) {
|
||||
if (!path.isAbsolute(root) || root.includes('\0')) {
|
||||
throw new RangeError(
|
||||
'Completion receipt orphan root must be an absolute path containing no NUL',
|
||||
);
|
||||
}
|
||||
if (path.resolve(root) === path.parse(path.resolve(root)).root) {
|
||||
throw new RangeError(
|
||||
'Completion receipt orphan root must not be a filesystem root',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async inspectShard(
|
||||
shard: string,
|
||||
maxEntries: number,
|
||||
): Promise<CompletionReceiptShardSnapshot> {
|
||||
if (!SHARD_PATTERN.test(shard)) {
|
||||
throw new RangeError(
|
||||
'Completion receipt shard must be two lowercase hex digits',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxEntries) ||
|
||||
maxEntries < 1 ||
|
||||
maxEntries > 64
|
||||
) {
|
||||
throw new RangeError('maxEntries must be between 1 and 64');
|
||||
}
|
||||
const directoryPath = await this.resolveShardDirectory(shard);
|
||||
if (!directoryPath) return { shard, entries: [], overflow: false };
|
||||
const directory = await fs.opendir(directoryPath, { bufferSize: 1 });
|
||||
|
||||
const entries: CompletionReceiptDirectoryEntry[] = [];
|
||||
let overflow = false;
|
||||
try {
|
||||
for await (const dirent of directory) {
|
||||
if (entries.length === maxEntries) {
|
||||
overflow = true;
|
||||
break;
|
||||
}
|
||||
const entryPath = path.join(directoryPath, dirent.name);
|
||||
let stat: Awaited<ReturnType<typeof fs.lstat>>;
|
||||
try {
|
||||
stat = await fs.lstat(entryPath);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) continue;
|
||||
throw error;
|
||||
}
|
||||
const classified = entryKind(shard, dirent.name, stat.isFile());
|
||||
entries.push({
|
||||
shard,
|
||||
name: dirent.name,
|
||||
kind: classified.kind,
|
||||
modifiedAtMs: Math.max(0, Math.trunc(stat.mtimeMs)),
|
||||
sizeBytes: stat.size,
|
||||
filesystemIdentity: filesystemIdentity(stat),
|
||||
...(classified.attemptId ? { attemptId: classified.attemptId } : {}),
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await directory.close().catch((error) => {
|
||||
if (!isCode(error, 'ERR_DIR_CLOSED')) throw error;
|
||||
});
|
||||
}
|
||||
return { shard, entries, overflow };
|
||||
}
|
||||
|
||||
async quarantine(
|
||||
entry: CompletionReceiptDirectoryEntry,
|
||||
): Promise<CompletionReceiptOrphanQuarantineResult> {
|
||||
if (
|
||||
!SHARD_PATTERN.test(entry.shard) ||
|
||||
path.basename(entry.name) !== entry.name
|
||||
) {
|
||||
throw new RangeError('Completion receipt orphan entry path is invalid');
|
||||
}
|
||||
if (entry.kind === 'unsafe') return { status: 'changed' };
|
||||
const shardDirectory = await this.resolveShardDirectory(entry.shard);
|
||||
if (!shardDirectory) return { status: 'changed' };
|
||||
const source = path.join(shardDirectory, entry.name);
|
||||
let current: Awaited<ReturnType<typeof fs.lstat>>;
|
||||
try {
|
||||
current = await fs.lstat(source);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return { status: 'changed' };
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
!current.isFile() ||
|
||||
filesystemIdentity(current) !== entry.filesystemIdentity
|
||||
) {
|
||||
return { status: 'changed' };
|
||||
}
|
||||
|
||||
const digest = createHash('sha256')
|
||||
.update(`${entry.shard}/${entry.name}\0${entry.filesystemIdentity}`)
|
||||
.digest('hex');
|
||||
const reference = path.posix.join(
|
||||
'.orphan-quarantine',
|
||||
entry.shard,
|
||||
`${digest}.entry`,
|
||||
);
|
||||
const canonicalRoot = path.dirname(shardDirectory);
|
||||
const directory = await this.ensureQuarantineDirectory(
|
||||
canonicalRoot,
|
||||
entry.shard,
|
||||
);
|
||||
const target = path.join(directory, `${digest}.entry`);
|
||||
let linkedByThisCall = false;
|
||||
try {
|
||||
await fs.link(source, target);
|
||||
linkedByThisCall = true;
|
||||
} catch (error) {
|
||||
if (!isCode(error, 'EEXIST')) {
|
||||
if (isCode(error, 'ENOENT')) return { status: 'changed' };
|
||||
throw error;
|
||||
}
|
||||
const targetStat = await fs.lstat(target);
|
||||
const sourceStat = await fs.lstat(source).catch(() => undefined);
|
||||
if (
|
||||
!sourceStat ||
|
||||
targetStat.dev !== sourceStat.dev ||
|
||||
targetStat.ino !== sourceStat.ino
|
||||
) {
|
||||
return { status: 'changed' };
|
||||
}
|
||||
}
|
||||
let verifiedShardDirectory: string | undefined;
|
||||
try {
|
||||
verifiedShardDirectory = await this.resolveShardDirectory(entry.shard);
|
||||
} catch (error) {
|
||||
if (linkedByThisCall) await fs.unlink(target).catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
const verifiedSource = await fs.lstat(source).catch(() => undefined);
|
||||
if (
|
||||
verifiedShardDirectory !== shardDirectory ||
|
||||
!verifiedSource ||
|
||||
filesystemIdentity(verifiedSource) !== entry.filesystemIdentity
|
||||
) {
|
||||
if (linkedByThisCall) await fs.unlink(target).catch(() => undefined);
|
||||
return { status: 'changed' };
|
||||
}
|
||||
await fs.unlink(source);
|
||||
await this.bestEffortSync(path.dirname(source));
|
||||
await this.bestEffortSync(directory);
|
||||
return { status: 'quarantined', reference };
|
||||
}
|
||||
|
||||
private async resolveShardDirectory(
|
||||
shard: string,
|
||||
): Promise<string | undefined> {
|
||||
let canonicalRoot: string;
|
||||
try {
|
||||
canonicalRoot = await fs.realpath(this.root);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return undefined;
|
||||
throw error;
|
||||
}
|
||||
const candidate = path.join(canonicalRoot, shard);
|
||||
let stat: Awaited<ReturnType<typeof fs.lstat>>;
|
||||
try {
|
||||
stat = await fs.lstat(candidate);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return undefined;
|
||||
throw error;
|
||||
}
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
`Completion receipt shard ${shard} is not a safe directory`,
|
||||
);
|
||||
}
|
||||
const canonicalDirectory = await fs.realpath(candidate);
|
||||
if (
|
||||
path.dirname(canonicalDirectory) !== canonicalRoot ||
|
||||
path.basename(canonicalDirectory) !== shard
|
||||
) {
|
||||
throw new Error(`Completion receipt shard ${shard} escapes its root`);
|
||||
}
|
||||
return canonicalDirectory;
|
||||
}
|
||||
|
||||
private async ensureQuarantineDirectory(
|
||||
canonicalRoot: string,
|
||||
shard: string,
|
||||
): Promise<string> {
|
||||
const quarantineRoot = path.join(canonicalRoot, '.orphan-quarantine');
|
||||
await this.ensurePrivateDirectory(quarantineRoot, canonicalRoot);
|
||||
const directory = path.join(quarantineRoot, shard);
|
||||
await this.ensurePrivateDirectory(directory, quarantineRoot);
|
||||
return directory;
|
||||
}
|
||||
|
||||
private async ensurePrivateDirectory(
|
||||
directory: string,
|
||||
expectedParent: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await fs.mkdir(directory, { mode: 0o700 });
|
||||
} catch (error) {
|
||||
if (!isCode(error, 'EEXIST')) throw error;
|
||||
}
|
||||
const stat = await fs.lstat(directory);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new Error(
|
||||
'Completion receipt quarantine path is not a safe directory',
|
||||
);
|
||||
}
|
||||
const canonicalDirectory = await fs.realpath(directory);
|
||||
if (path.dirname(canonicalDirectory) !== expectedParent) {
|
||||
throw new Error('Completion receipt quarantine path escapes its root');
|
||||
}
|
||||
await fs.chmod(directory, 0o700);
|
||||
}
|
||||
|
||||
private async bestEffortSync(directory: string): Promise<void> {
|
||||
try {
|
||||
const handle = await fs.open(directory, constants.O_RDONLY);
|
||||
try {
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
} catch {
|
||||
// Directory fsync is unavailable on some supported filesystems.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { normalizeLocalArtifactReadRange } from '../../domain/artifactRead';
|
||||
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
|
||||
import type {
|
||||
AvailableLocalArtifactByteRange,
|
||||
LocalArtifactByteRangeReadResult,
|
||||
LocalArtifactByteRangeReader as LocalArtifactByteRangeReaderPort,
|
||||
} from '../../ports/localArtifactByteRangeReader';
|
||||
|
||||
export class UnsafeLocalArtifactReadTargetError extends Error {
|
||||
constructor() {
|
||||
super('Local Artifact read target is unsafe');
|
||||
this.name = 'UnsafeLocalArtifactReadTargetError';
|
||||
}
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
(error as { code?: string }).code === code
|
||||
);
|
||||
}
|
||||
|
||||
export class LocalArtifactByteRangeReader
|
||||
implements LocalArtifactByteRangeReaderPort
|
||||
{
|
||||
private readonly root: string;
|
||||
|
||||
constructor(root: string) {
|
||||
if (!path.isAbsolute(root) || root.includes('\0')) {
|
||||
throw new TypeError('Local Artifact read root must be absolute');
|
||||
}
|
||||
this.root = path.resolve(root);
|
||||
}
|
||||
|
||||
async read(
|
||||
logArtifactId: string,
|
||||
requestedRange: Parameters<LocalArtifactByteRangeReaderPort['read']>[1],
|
||||
): Promise<LocalArtifactByteRangeReadResult> {
|
||||
assertLocalExecutionArtifactId(logArtifactId);
|
||||
const range = normalizeLocalArtifactReadRange(requestedRange);
|
||||
const directory = path.join(this.root, logArtifactId.slice(6, 8));
|
||||
await this.assertDirectory(this.root);
|
||||
if (!(await this.optionalDirectory(directory)))
|
||||
return { status: 'missing' };
|
||||
const target = path.join(directory, `${logArtifactId}.log`);
|
||||
let handle;
|
||||
try {
|
||||
handle = await fs.open(
|
||||
target,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return { status: 'missing' };
|
||||
throw new UnsafeLocalArtifactReadTargetError();
|
||||
}
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (!stat.isFile() || !Number.isSafeInteger(stat.size) || stat.size < 0) {
|
||||
throw new UnsafeLocalArtifactReadTargetError();
|
||||
}
|
||||
const start = Math.min(range.offset, stat.size);
|
||||
const expected = Math.min(range.length, stat.size - start);
|
||||
const content = Buffer.allocUnsafe(expected);
|
||||
let read = 0;
|
||||
while (read < expected) {
|
||||
const result = await handle.read(
|
||||
content,
|
||||
read,
|
||||
expected - read,
|
||||
start + read,
|
||||
);
|
||||
if (result.bytesRead < 1) {
|
||||
throw new UnsafeLocalArtifactReadTargetError();
|
||||
}
|
||||
read += result.bytesRead;
|
||||
}
|
||||
const endExclusive = start + expected;
|
||||
const result: AvailableLocalArtifactByteRange = {
|
||||
status: 'available',
|
||||
content,
|
||||
start,
|
||||
endExclusive,
|
||||
totalBytes: stat.size,
|
||||
...(endExclusive < stat.size ? { nextOffset: endExclusive } : {}),
|
||||
};
|
||||
return Object.freeze(result);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
private async assertDirectory(value: string): Promise<void> {
|
||||
try {
|
||||
const stat = await fs.lstat(value);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new UnsafeLocalArtifactReadTargetError();
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof UnsafeLocalArtifactReadTargetError) throw error;
|
||||
throw new UnsafeLocalArtifactReadTargetError();
|
||||
}
|
||||
}
|
||||
|
||||
private async optionalDirectory(value: string): Promise<boolean> {
|
||||
try {
|
||||
const stat = await fs.lstat(value);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new UnsafeLocalArtifactReadTargetError();
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
if (error instanceof UnsafeLocalArtifactReadTargetError) throw error;
|
||||
throw new UnsafeLocalArtifactReadTargetError();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
import { constants, type Stats } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES } from '../../domain/localArtifactTruncation';
|
||||
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
|
||||
import type {
|
||||
LocalArtifactFileRetirementResult,
|
||||
LocalArtifactFileRetirementStore as LocalArtifactFileRetirementStorePort,
|
||||
} from '../../ports/localArtifactFileRetirementStore';
|
||||
import { localArtifactTruncationFactFileName } from './localArtifactTruncationFactStore';
|
||||
|
||||
export class UnsafeLocalArtifactRetirementError extends Error {
|
||||
constructor() {
|
||||
super('Local Artifact retirement target is unsafe');
|
||||
this.name = 'UnsafeLocalArtifactRetirementError';
|
||||
}
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
(error as { code?: string }).code === code
|
||||
);
|
||||
}
|
||||
|
||||
export class LocalArtifactFileRetirementStore
|
||||
implements LocalArtifactFileRetirementStorePort
|
||||
{
|
||||
private readonly root: string;
|
||||
|
||||
constructor(root: string) {
|
||||
if (!path.isAbsolute(root) || root.includes('\0')) {
|
||||
throw new TypeError('Local Artifact retirement root must be absolute');
|
||||
}
|
||||
this.root = path.resolve(root);
|
||||
}
|
||||
|
||||
async retire(
|
||||
logArtifactId: string,
|
||||
): Promise<LocalArtifactFileRetirementResult> {
|
||||
assertLocalExecutionArtifactId(logArtifactId);
|
||||
const directory = path.join(this.root, logArtifactId.slice(6, 8));
|
||||
await this.assertDirectory(this.root);
|
||||
if (!(await this.optionalDirectory(directory))) {
|
||||
return Object.freeze({
|
||||
disposition: 'already_absent',
|
||||
bytesReclaimed: 0,
|
||||
});
|
||||
}
|
||||
const target = path.join(directory, `${logArtifactId}.log`);
|
||||
const fifo = path.join(directory, `.${logArtifactId}.log.fifo`);
|
||||
const truncation = path.join(
|
||||
directory,
|
||||
localArtifactTruncationFactFileName(logArtifactId),
|
||||
);
|
||||
const truncationTemporary = path.join(
|
||||
directory,
|
||||
`.${logArtifactId}.log.truncated.tmp`,
|
||||
);
|
||||
const [targetStat, fifoStat, truncationStat, truncationTemporaryStat] =
|
||||
await Promise.all([
|
||||
this.lstat(target),
|
||||
this.lstat(fifo),
|
||||
this.lstat(truncation),
|
||||
this.lstat(truncationTemporary),
|
||||
]);
|
||||
if (targetStat && (!targetStat.isFile() || targetStat.isSymbolicLink())) {
|
||||
throw new UnsafeLocalArtifactRetirementError();
|
||||
}
|
||||
if (
|
||||
targetStat &&
|
||||
(!Number.isSafeInteger(targetStat.size) || targetStat.size < 0)
|
||||
) {
|
||||
throw new UnsafeLocalArtifactRetirementError();
|
||||
}
|
||||
if (fifoStat && (!fifoStat.isFIFO() || fifoStat.isSymbolicLink())) {
|
||||
throw new UnsafeLocalArtifactRetirementError();
|
||||
}
|
||||
for (const stat of [truncationStat, truncationTemporaryStat]) {
|
||||
if (
|
||||
stat &&
|
||||
(!stat.isFile() ||
|
||||
stat.isSymbolicLink() ||
|
||||
!Number.isSafeInteger(stat.size) ||
|
||||
stat.size < 0 ||
|
||||
stat.size > MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES)
|
||||
) {
|
||||
throw new UnsafeLocalArtifactRetirementError();
|
||||
}
|
||||
}
|
||||
let targetDeleted = false;
|
||||
let bytesReclaimed = 0;
|
||||
if (targetStat) {
|
||||
try {
|
||||
await fs.unlink(target);
|
||||
targetDeleted = true;
|
||||
bytesReclaimed = targetStat.size;
|
||||
} catch (error) {
|
||||
if (!isCode(error, 'ENOENT')) throw error;
|
||||
}
|
||||
}
|
||||
let auxiliaryRemoved = false;
|
||||
for (const [auxiliary, stat] of [
|
||||
[fifo, fifoStat],
|
||||
[truncation, truncationStat],
|
||||
[truncationTemporary, truncationTemporaryStat],
|
||||
] as const) {
|
||||
if (!stat) continue;
|
||||
try {
|
||||
await fs.unlink(auxiliary);
|
||||
auxiliaryRemoved = true;
|
||||
} catch (error) {
|
||||
if (!isCode(error, 'ENOENT')) throw error;
|
||||
}
|
||||
}
|
||||
if (targetDeleted || auxiliaryRemoved) {
|
||||
await this.syncDirectory(directory);
|
||||
}
|
||||
if (!targetDeleted) {
|
||||
return Object.freeze({
|
||||
disposition: 'already_absent',
|
||||
bytesReclaimed: 0,
|
||||
});
|
||||
}
|
||||
return Object.freeze({
|
||||
disposition: 'deleted',
|
||||
bytesReclaimed,
|
||||
});
|
||||
}
|
||||
|
||||
private async assertDirectory(value: string): Promise<void> {
|
||||
try {
|
||||
const stat = await fs.lstat(value);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new UnsafeLocalArtifactRetirementError();
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof UnsafeLocalArtifactRetirementError) throw error;
|
||||
throw new UnsafeLocalArtifactRetirementError();
|
||||
}
|
||||
}
|
||||
|
||||
private async optionalDirectory(value: string): Promise<boolean> {
|
||||
try {
|
||||
const stat = await fs.lstat(value);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new UnsafeLocalArtifactRetirementError();
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
if (error instanceof UnsafeLocalArtifactRetirementError) throw error;
|
||||
throw new UnsafeLocalArtifactRetirementError();
|
||||
}
|
||||
}
|
||||
|
||||
private async lstat(value: string): Promise<Stats | null> {
|
||||
try {
|
||||
return await fs.lstat(value);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async syncDirectory(directory: string): Promise<void> {
|
||||
const handle = await fs.open(
|
||||
directory,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
try {
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import {
|
||||
decodeLocalArtifactTruncationFact,
|
||||
MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES,
|
||||
type LocalArtifactTruncationFact,
|
||||
} from '../../domain/localArtifactTruncation';
|
||||
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
|
||||
import type { LocalArtifactTruncationFactStore as LocalArtifactTruncationFactStorePort } from '../../ports/localArtifactTruncationFactStore';
|
||||
|
||||
export function localArtifactTruncationFactFileName(
|
||||
logArtifactId: string,
|
||||
): string {
|
||||
assertLocalExecutionArtifactId(logArtifactId);
|
||||
return `.${logArtifactId}.log.truncated.json`;
|
||||
}
|
||||
|
||||
export class UnsafeLocalArtifactTruncationFactError extends Error {
|
||||
constructor() {
|
||||
super('Local Artifact truncation fact target is unsafe');
|
||||
this.name = 'UnsafeLocalArtifactTruncationFactError';
|
||||
}
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
(error as { code?: string }).code === code
|
||||
);
|
||||
}
|
||||
|
||||
export class LocalArtifactTruncationFactStore
|
||||
implements LocalArtifactTruncationFactStorePort
|
||||
{
|
||||
private readonly root: string;
|
||||
|
||||
constructor(root: string) {
|
||||
if (!path.isAbsolute(root) || root.includes('\0')) {
|
||||
throw new TypeError('Local Artifact truncation root must be absolute');
|
||||
}
|
||||
this.root = path.resolve(root);
|
||||
}
|
||||
|
||||
async read(
|
||||
logArtifactId: string,
|
||||
): Promise<Readonly<LocalArtifactTruncationFact> | null> {
|
||||
assertLocalExecutionArtifactId(logArtifactId);
|
||||
const directory = path.join(this.root, logArtifactId.slice(6, 8));
|
||||
await this.assertDirectory(this.root);
|
||||
if (!(await this.optionalDirectory(directory))) return null;
|
||||
const target = path.join(
|
||||
directory,
|
||||
localArtifactTruncationFactFileName(logArtifactId),
|
||||
);
|
||||
let handle;
|
||||
try {
|
||||
handle = await fs.open(
|
||||
target,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return null;
|
||||
throw new UnsafeLocalArtifactTruncationFactError();
|
||||
}
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
!Number.isSafeInteger(stat.size) ||
|
||||
stat.size < 1 ||
|
||||
stat.size > MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES
|
||||
) {
|
||||
throw new UnsafeLocalArtifactTruncationFactError();
|
||||
}
|
||||
const fact = decodeLocalArtifactTruncationFact(await handle.readFile());
|
||||
if (fact.logArtifactId !== logArtifactId) {
|
||||
throw new UnsafeLocalArtifactTruncationFactError();
|
||||
}
|
||||
return fact;
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
private async assertDirectory(value: string): Promise<void> {
|
||||
try {
|
||||
const stat = await fs.lstat(value);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new UnsafeLocalArtifactTruncationFactError();
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof UnsafeLocalArtifactTruncationFactError) throw error;
|
||||
throw new UnsafeLocalArtifactTruncationFactError();
|
||||
}
|
||||
}
|
||||
|
||||
private async optionalDirectory(value: string): Promise<boolean> {
|
||||
try {
|
||||
const stat = await fs.lstat(value);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new UnsafeLocalArtifactTruncationFactError();
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
if (error instanceof UnsafeLocalArtifactTruncationFactError) throw error;
|
||||
throw new UnsafeLocalArtifactTruncationFactError();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import type {
|
||||
ExecutionOutputChunk,
|
||||
ExecutionOutputSink,
|
||||
} from '../../domain/execution';
|
||||
import {
|
||||
LocalArtifactCapacityUnavailableError,
|
||||
LocalArtifactQuotaExceededError,
|
||||
normalizeLocalArtifactCapacityPolicy,
|
||||
type LocalArtifactCapacityPolicy,
|
||||
} from '../../domain/localArtifactCapacity';
|
||||
import {
|
||||
localExecutionArtifactId,
|
||||
assertLocalExecutionArtifactId,
|
||||
} from '../../domain/localExecutionArtifact';
|
||||
import type { RunDispatchCandidate } from '../../domain/runDispatchCandidate';
|
||||
import type {
|
||||
LocalExecutionArtifactAllocator,
|
||||
PreparedLocalExecutionArtifact,
|
||||
} from '../../ports/localExecutionArtifactAllocator';
|
||||
import type { LocalArtifactCapacityProbe } from '../../ports/localArtifactCapacityProbe';
|
||||
import { LocalFileSystemCapacityProbe } from './localFileSystemCapacityProbe';
|
||||
import { enableDurableLocalProcessOutput } from '../local-process/durableLocalProcessOutput';
|
||||
|
||||
async function privateDirectory(directory: string): Promise<void> {
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const stat = await fs.lstat(directory);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new TypeError('Local execution artifact directory is unsafe');
|
||||
}
|
||||
await fs.chmod(directory, 0o700);
|
||||
}
|
||||
|
||||
class LocalFileExecutionOutput implements ExecutionOutputSink {
|
||||
private pending: Promise<unknown> = Promise.resolve();
|
||||
private closed = false;
|
||||
|
||||
private remainingBytes: number;
|
||||
|
||||
constructor(
|
||||
private readonly file: fs.FileHandle,
|
||||
maximumBytes: number,
|
||||
existingBytes: number,
|
||||
) {
|
||||
this.remainingBytes = maximumBytes - existingBytes;
|
||||
}
|
||||
|
||||
write(output: ExecutionOutputChunk): Promise<void> {
|
||||
if (this.closed) {
|
||||
return Promise.reject(new Error('Local execution artifact is closed'));
|
||||
}
|
||||
const chunk = Buffer.from(output.chunk);
|
||||
const operation = this.pending.then(async () => {
|
||||
if (this.remainingBytes <= 0) {
|
||||
throw new LocalArtifactQuotaExceededError();
|
||||
}
|
||||
const accepted = chunk.subarray(
|
||||
0,
|
||||
Math.min(chunk.length, this.remainingBytes),
|
||||
);
|
||||
let written = 0;
|
||||
while (written < accepted.length) {
|
||||
const result = await this.file.write(accepted.subarray(written));
|
||||
if (result.bytesWritten < 1) {
|
||||
throw new Error('Local execution artifact write made no progress');
|
||||
}
|
||||
written += result.bytesWritten;
|
||||
this.remainingBytes -= result.bytesWritten;
|
||||
}
|
||||
if (accepted.length !== chunk.length) {
|
||||
throw new LocalArtifactQuotaExceededError();
|
||||
}
|
||||
});
|
||||
this.pending = operation.catch(() => undefined);
|
||||
return operation;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
await this.pending.catch(() => undefined);
|
||||
await this.file.close();
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalFileExecutionArtifactAllocator
|
||||
implements LocalExecutionArtifactAllocator
|
||||
{
|
||||
private readonly artifactRoot: string;
|
||||
private readonly completionReceiptRoot: string;
|
||||
private readonly policy: Readonly<LocalArtifactCapacityPolicy>;
|
||||
private readonly capacity: LocalArtifactCapacityProbe;
|
||||
|
||||
constructor(
|
||||
artifactRoot: string,
|
||||
completionReceiptRoot: string,
|
||||
policy: LocalArtifactCapacityPolicy,
|
||||
capacity: LocalArtifactCapacityProbe = new LocalFileSystemCapacityProbe(),
|
||||
) {
|
||||
if (
|
||||
!path.isAbsolute(artifactRoot) ||
|
||||
artifactRoot.includes('\0') ||
|
||||
!path.isAbsolute(completionReceiptRoot) ||
|
||||
completionReceiptRoot.includes('\0')
|
||||
) {
|
||||
throw new TypeError('Local execution artifact roots must be absolute');
|
||||
}
|
||||
this.artifactRoot = path.resolve(artifactRoot);
|
||||
this.completionReceiptRoot = path.resolve(completionReceiptRoot);
|
||||
this.policy = normalizeLocalArtifactCapacityPolicy(policy);
|
||||
this.capacity = capacity;
|
||||
}
|
||||
|
||||
async prepare(
|
||||
candidate: Readonly<RunDispatchCandidate>,
|
||||
): Promise<PreparedLocalExecutionArtifact> {
|
||||
const logArtifactId = localExecutionArtifactId(candidate);
|
||||
assertLocalExecutionArtifactId(logArtifactId);
|
||||
const shard = logArtifactId.slice('local-'.length, 'local-'.length + 2);
|
||||
const directory = path.join(this.artifactRoot, shard);
|
||||
await privateDirectory(this.artifactRoot);
|
||||
const capacity = await this.capacity.inspect(this.artifactRoot);
|
||||
const requiredBytes =
|
||||
BigInt(this.policy.minimumFreeBytes) +
|
||||
BigInt(this.policy.maximumAttemptBytes);
|
||||
if (capacity.availableBytes < requiredBytes) {
|
||||
throw new LocalArtifactCapacityUnavailableError();
|
||||
}
|
||||
await privateDirectory(directory);
|
||||
await privateDirectory(this.completionReceiptRoot);
|
||||
const outputFilePath = path.join(directory, `${logArtifactId}.log`);
|
||||
const file = await fs.open(
|
||||
outputFilePath,
|
||||
constants.O_WRONLY |
|
||||
constants.O_CREAT |
|
||||
constants.O_APPEND |
|
||||
(constants.O_NOFOLLOW ?? 0),
|
||||
0o600,
|
||||
);
|
||||
try {
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile()) {
|
||||
throw new TypeError('Local execution artifact target is unsafe');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(stat.size) ||
|
||||
stat.size < 0 ||
|
||||
stat.size > this.policy.maximumAttemptBytes
|
||||
) {
|
||||
throw new LocalArtifactQuotaExceededError();
|
||||
}
|
||||
await file.chmod(0o600);
|
||||
const output = new LocalFileExecutionOutput(
|
||||
file,
|
||||
this.policy.maximumAttemptBytes,
|
||||
stat.size,
|
||||
);
|
||||
return {
|
||||
logArtifactId,
|
||||
output: enableDurableLocalProcessOutput(output, {
|
||||
outputFilePath,
|
||||
completionReceiptRoot: this.completionReceiptRoot,
|
||||
maximumBytes: this.policy.maximumAttemptBytes,
|
||||
logArtifactId,
|
||||
}),
|
||||
dispose: () => output.close(),
|
||||
};
|
||||
} catch (error) {
|
||||
await file.close().catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { LocalArtifactCapacityUnavailableError } from '../../domain/localArtifactCapacity';
|
||||
import type {
|
||||
LocalArtifactCapacityProbe,
|
||||
LocalArtifactCapacitySnapshot,
|
||||
LocalArtifactCapacitySource,
|
||||
} from '../../ports/localArtifactCapacityProbe';
|
||||
|
||||
interface BigIntStatFs {
|
||||
bavail: bigint;
|
||||
blocks: bigint;
|
||||
bsize: bigint;
|
||||
}
|
||||
|
||||
export class RootedLocalFileSystemCapacitySource
|
||||
implements LocalArtifactCapacitySource
|
||||
{
|
||||
private readonly root: string;
|
||||
|
||||
constructor(
|
||||
root: string,
|
||||
private readonly probe: LocalArtifactCapacityProbe = new LocalFileSystemCapacityProbe(),
|
||||
) {
|
||||
if (!path.isAbsolute(root) || root.includes('\0')) {
|
||||
throw new TypeError('Local Artifact capacity root must be absolute');
|
||||
}
|
||||
this.root = path.resolve(root);
|
||||
}
|
||||
|
||||
inspect(): Promise<LocalArtifactCapacitySnapshot> {
|
||||
return this.probe.inspect(this.root);
|
||||
}
|
||||
}
|
||||
|
||||
interface StatFsPromises {
|
||||
statfs(value: string, options: { bigint: true }): Promise<BigIntStatFs>;
|
||||
}
|
||||
|
||||
export class LocalFileSystemCapacityProbe
|
||||
implements LocalArtifactCapacityProbe
|
||||
{
|
||||
async inspect(root: string): Promise<LocalArtifactCapacitySnapshot> {
|
||||
if (!path.isAbsolute(root) || root.includes('\0')) {
|
||||
throw new LocalArtifactCapacityUnavailableError();
|
||||
}
|
||||
try {
|
||||
const statfs = (fs as unknown as StatFsPromises).statfs;
|
||||
if (typeof statfs !== 'function') {
|
||||
throw new LocalArtifactCapacityUnavailableError();
|
||||
}
|
||||
const stat = await statfs.call(fs, root, { bigint: true });
|
||||
const availableBytes = stat.bavail * stat.bsize;
|
||||
const totalBytes = stat.blocks * stat.bsize;
|
||||
if (
|
||||
availableBytes < BigInt(0) ||
|
||||
totalBytes < BigInt(1) ||
|
||||
availableBytes > totalBytes
|
||||
) {
|
||||
throw new LocalArtifactCapacityUnavailableError();
|
||||
}
|
||||
return Object.freeze({ availableBytes, totalBytes });
|
||||
} catch (error) {
|
||||
if (error instanceof LocalArtifactCapacityUnavailableError) throw error;
|
||||
throw new LocalArtifactCapacityUnavailableError();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import {
|
||||
LocalSecretUnavailableError,
|
||||
assertLocalSecretKeyId,
|
||||
} from '../../domain/localSecret';
|
||||
import type {
|
||||
LocalSecretKeyMaterial,
|
||||
LocalSecretKeyProvider,
|
||||
} from '../../ports/localSecretKeyProvider';
|
||||
|
||||
const MAX_KEYRING_BYTES = 16 * 1024;
|
||||
const MAX_KEY_COUNT = 16;
|
||||
const BASE64URL_PATTERN = /^[A-Za-z0-9_-]+$/;
|
||||
|
||||
interface LocalSecretKeyringManifest {
|
||||
version: 1;
|
||||
activeKeyId: string;
|
||||
keys: Record<string, string>;
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const actual = Object.keys(value).sort();
|
||||
return (
|
||||
actual.length === expected.length &&
|
||||
actual.every((key, index) => key === expected[index])
|
||||
);
|
||||
}
|
||||
|
||||
function parseManifest(value: Buffer): LocalSecretKeyringManifest {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(value.toString('utf8'));
|
||||
} catch {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
if (
|
||||
!parsed ||
|
||||
typeof parsed !== 'object' ||
|
||||
Array.isArray(parsed) ||
|
||||
!exactKeys(parsed, ['activeKeyId', 'keys', 'version'])
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
const record = parsed as Record<string, unknown>;
|
||||
if (
|
||||
record.version !== 1 ||
|
||||
!record.keys ||
|
||||
typeof record.keys !== 'object' ||
|
||||
Array.isArray(record.keys)
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
assertLocalSecretKeyId(record.activeKeyId as string);
|
||||
const entries = Object.entries(record.keys as Record<string, unknown>);
|
||||
if (entries.length < 1 || entries.length > MAX_KEY_COUNT) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
const keys: Record<string, string> = Object.create(null);
|
||||
for (const [keyId, encoded] of entries) {
|
||||
assertLocalSecretKeyId(keyId);
|
||||
const decoded =
|
||||
typeof encoded === 'string'
|
||||
? Buffer.from(encoded, 'base64url')
|
||||
: Buffer.alloc(0);
|
||||
if (
|
||||
typeof encoded !== 'string' ||
|
||||
!BASE64URL_PATTERN.test(encoded) ||
|
||||
decoded.length !== 32 ||
|
||||
decoded.toString('base64url') !== encoded
|
||||
) {
|
||||
decoded.fill(0);
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
decoded.fill(0);
|
||||
keys[keyId] = encoded;
|
||||
}
|
||||
if (!keys[record.activeKeyId as string]) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
return {
|
||||
version: 1,
|
||||
activeKeyId: record.activeKeyId as string,
|
||||
keys,
|
||||
};
|
||||
}
|
||||
|
||||
export class LocalSecretKeyringFileProvider implements LocalSecretKeyProvider {
|
||||
private readonly filePath: string;
|
||||
|
||||
constructor(filePath: string) {
|
||||
if (!path.isAbsolute(filePath) || filePath.includes('\0')) {
|
||||
throw new TypeError('Local Secret keyring path must be absolute');
|
||||
}
|
||||
this.filePath = path.resolve(filePath);
|
||||
}
|
||||
|
||||
async active(): Promise<LocalSecretKeyMaterial> {
|
||||
const manifest = await this.read();
|
||||
return this.material(
|
||||
manifest,
|
||||
manifest.activeKeyId,
|
||||
) as LocalSecretKeyMaterial;
|
||||
}
|
||||
|
||||
async resolve(keyId: string): Promise<LocalSecretKeyMaterial | null> {
|
||||
try {
|
||||
assertLocalSecretKeyId(keyId);
|
||||
} catch {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
const manifest = await this.read();
|
||||
return this.material(manifest, keyId);
|
||||
}
|
||||
|
||||
private material(
|
||||
manifest: LocalSecretKeyringManifest,
|
||||
keyId: string,
|
||||
): LocalSecretKeyMaterial | null {
|
||||
const encoded = manifest.keys[keyId];
|
||||
return encoded
|
||||
? Object.freeze({
|
||||
keyId,
|
||||
key: Uint8Array.from(Buffer.from(encoded, 'base64url')),
|
||||
})
|
||||
: null;
|
||||
}
|
||||
|
||||
private async read(): Promise<LocalSecretKeyringManifest> {
|
||||
let file: fs.FileHandle | undefined;
|
||||
let contents: Buffer | undefined;
|
||||
try {
|
||||
file = await fs.open(
|
||||
this.filePath,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
const stat = await file.stat();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.isSymbolicLink() ||
|
||||
(stat.mode & 0o077) !== 0 ||
|
||||
stat.size < 1 ||
|
||||
stat.size > MAX_KEYRING_BYTES
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
contents = await file.readFile();
|
||||
if (contents.length !== stat.size) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
return parseManifest(contents);
|
||||
} catch {
|
||||
throw new LocalSecretUnavailableError();
|
||||
} finally {
|
||||
contents?.fill(0);
|
||||
await file?.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { createHash } from 'crypto';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import {
|
||||
defaultOffRuntimeRolloutPolicy,
|
||||
parseRuntimeRolloutManifest,
|
||||
} from '../../domain/runtimeRolloutManifest';
|
||||
import type {
|
||||
RuntimeRolloutLoadAudit,
|
||||
RuntimeRolloutLoadResult,
|
||||
} from '../../ports/runtimeRolloutLoader';
|
||||
|
||||
export type {
|
||||
RuntimeRolloutLoadAudit,
|
||||
RuntimeRolloutLoadResult,
|
||||
RuntimeRolloutLoadStatus,
|
||||
} from '../../ports/runtimeRolloutLoader';
|
||||
|
||||
export const MAX_RUNTIME_ROLLOUT_MANIFEST_BYTES = 64 * 1024;
|
||||
|
||||
export interface RuntimeRolloutManifestLoaderOptions {
|
||||
clock?: { now(): number };
|
||||
maxBytes?: number;
|
||||
}
|
||||
|
||||
function rejected(
|
||||
audit: RuntimeRolloutLoadAudit,
|
||||
reasonCode: NonNullable<RuntimeRolloutLoadAudit['reasonCode']>,
|
||||
): RuntimeRolloutLoadResult {
|
||||
return {
|
||||
status: 'rejected',
|
||||
policy: defaultOffRuntimeRolloutPolicy(),
|
||||
audit: { ...audit, status: 'rejected', reasonCode },
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadRuntimeRolloutManifest(
|
||||
sourcePath: string,
|
||||
options: RuntimeRolloutManifestLoaderOptions = {},
|
||||
): Promise<RuntimeRolloutLoadResult> {
|
||||
if (!path.isAbsolute(sourcePath)) {
|
||||
throw new TypeError('Runtime rollout manifest path must be absolute');
|
||||
}
|
||||
const evaluatedAtMs = (options.clock ?? { now: Date.now }).now();
|
||||
if (!Number.isSafeInteger(evaluatedAtMs) || evaluatedAtMs < 0) {
|
||||
throw new TypeError('Runtime rollout clock returned an invalid timestamp');
|
||||
}
|
||||
const maxBytes = options.maxBytes ?? MAX_RUNTIME_ROLLOUT_MANIFEST_BYTES;
|
||||
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
|
||||
throw new TypeError('Runtime rollout maxBytes must be a positive integer');
|
||||
}
|
||||
const baseAudit: RuntimeRolloutLoadAudit = {
|
||||
event: 'runtime.rollout_config_evaluated',
|
||||
evaluatedAtMs,
|
||||
sourcePath,
|
||||
status: 'rejected',
|
||||
};
|
||||
|
||||
let bytes: Buffer;
|
||||
try {
|
||||
bytes = await fs.readFile(sourcePath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return {
|
||||
status: 'missing',
|
||||
policy: defaultOffRuntimeRolloutPolicy(),
|
||||
audit: {
|
||||
...baseAudit,
|
||||
status: 'missing',
|
||||
reasonCode: 'FILE_MISSING',
|
||||
},
|
||||
};
|
||||
}
|
||||
return rejected(baseAudit, 'FILE_READ_FAILED');
|
||||
}
|
||||
|
||||
const sourceSha256 = createHash('sha256').update(bytes).digest('hex');
|
||||
const hashedAudit = { ...baseAudit, sourceSha256 };
|
||||
if (bytes.byteLength > maxBytes) {
|
||||
return rejected(hashedAudit, 'FILE_TOO_LARGE');
|
||||
}
|
||||
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(bytes.toString('utf8'));
|
||||
} catch {
|
||||
return rejected(hashedAudit, 'INVALID_JSON');
|
||||
}
|
||||
|
||||
try {
|
||||
const decision = parseRuntimeRolloutManifest(value, evaluatedAtMs);
|
||||
const status = decision.manifest.enabled ? 'accepted' : 'disabled';
|
||||
return {
|
||||
status,
|
||||
policy: decision.policy,
|
||||
manifest: decision.manifest,
|
||||
audit: {
|
||||
...hashedAudit,
|
||||
status,
|
||||
revision: decision.manifest.revision,
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return rejected(hashedAudit, 'INVALID_MANIFEST');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,534 @@
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { randomBytes } from 'crypto';
|
||||
import { lock } from 'proper-lockfile';
|
||||
import {
|
||||
MAX_WORKER_EXECUTION_OFFER_JOURNAL_ENTRIES,
|
||||
MAX_WORKER_EXECUTION_OFFER_JOURNAL_PAGE_SIZE,
|
||||
MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES,
|
||||
cloneWorkerExecutionOfferJournalRecord,
|
||||
parseWorkerExecutionOfferJournalRecord,
|
||||
serializeWorkerExecutionOfferJournalRecord,
|
||||
type WorkerExecutionOfferJournalRecord,
|
||||
} from '../../domain/workerExecutionOffer';
|
||||
import { assertRunDispatchOfferId } from '../../domain/runDispatchOffer';
|
||||
import type {
|
||||
WorkerExecutionOfferJournal,
|
||||
WorkerExecutionOfferJournalCreateResult,
|
||||
WorkerExecutionOfferJournalPage,
|
||||
} from '../../ports/workerExecutionOfferJournal';
|
||||
import type {
|
||||
WorkerExecutionOfferJournalOwnership,
|
||||
WorkerExecutionOfferJournalOwnershipState,
|
||||
} from '../../ports/workerExecutionOfferJournalOwnership';
|
||||
|
||||
const JOURNAL_FILE_PATTERN = /^([0-9a-f]{64})\.json$/;
|
||||
|
||||
export const MIN_WORKER_OFFER_JOURNAL_LOCK_STALE_MS = 5_000;
|
||||
export const MAX_WORKER_OFFER_JOURNAL_LOCK_STALE_MS = 5 * 60_000;
|
||||
|
||||
export interface WorkerExecutionOfferLockProvider {
|
||||
acquire(options: {
|
||||
root: string;
|
||||
lockfilePath: string;
|
||||
staleMs: number;
|
||||
updateMs: number;
|
||||
onCompromised(error: Error): void;
|
||||
}): Promise<() => Promise<void>>;
|
||||
}
|
||||
|
||||
const properLockProvider: WorkerExecutionOfferLockProvider = {
|
||||
acquire(options) {
|
||||
return lock(options.root, {
|
||||
stale: options.staleMs,
|
||||
update: options.updateMs,
|
||||
retries: 0,
|
||||
realpath: true,
|
||||
lockfilePath: options.lockfilePath,
|
||||
onCompromised: options.onCompromised,
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
export class WorkerExecutionOfferJournalCapacityError extends Error {
|
||||
constructor(readonly maximumEntries: number) {
|
||||
super(`Worker execution offer journal reached ${maximumEntries} entries`);
|
||||
this.name = 'WorkerExecutionOfferJournalCapacityError';
|
||||
}
|
||||
}
|
||||
|
||||
export class WorkerExecutionOfferJournalRevisionError extends Error {
|
||||
constructor(readonly offerId: string) {
|
||||
super(`Worker execution offer journal revision changed for ${offerId}`);
|
||||
this.name = 'WorkerExecutionOfferJournalRevisionError';
|
||||
}
|
||||
}
|
||||
|
||||
export class WorkerExecutionOfferJournalNotFoundError extends Error {
|
||||
constructor(readonly offerId: string) {
|
||||
super(`Worker execution offer journal entry ${offerId} was not found`);
|
||||
this.name = 'WorkerExecutionOfferJournalNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
export class WorkerExecutionOfferJournalOwnershipError extends Error {
|
||||
constructor(
|
||||
readonly reason: 'already_owned' | 'not_owned' | 'compromised',
|
||||
readonly cause?: unknown,
|
||||
) {
|
||||
super(`Worker execution offer journal ownership failed: ${reason}`);
|
||||
this.name = 'WorkerExecutionOfferJournalOwnershipError';
|
||||
}
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
(error as { code?: string }).code === code
|
||||
);
|
||||
}
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Worker-local restart journal. One bounded file per offer avoids a database or
|
||||
* sidecar on edge devices. The Worker runtime must exclusively own this root.
|
||||
*/
|
||||
export class WorkerExecutionOfferFileJournal
|
||||
implements WorkerExecutionOfferJournal, WorkerExecutionOfferJournalOwnership
|
||||
{
|
||||
private readonly maximumEntries: number;
|
||||
private readonly ownershipStaleMs: number;
|
||||
private readonly lockProvider: WorkerExecutionOfferLockProvider;
|
||||
private readonly onOwnershipCompromised?: (error: Error) => void;
|
||||
private ownerState: WorkerExecutionOfferJournalOwnershipState = 'unowned';
|
||||
private releaseOwner?: () => Promise<void>;
|
||||
private ownershipError?: Error;
|
||||
private mutationTail: Promise<void> = Promise.resolve();
|
||||
|
||||
constructor(
|
||||
private readonly root: string,
|
||||
options: {
|
||||
maximumEntries?: number;
|
||||
ownershipStaleMs?: number;
|
||||
lockProvider?: WorkerExecutionOfferLockProvider;
|
||||
onOwnershipCompromised?: (error: Error) => void;
|
||||
} = {},
|
||||
) {
|
||||
if (!path.isAbsolute(root)) {
|
||||
throw new RangeError(
|
||||
'Worker execution offer journal root must be absolute',
|
||||
);
|
||||
}
|
||||
this.maximumEntries = options.maximumEntries ?? 64;
|
||||
assertIntegerBetween(
|
||||
'maximumEntries',
|
||||
this.maximumEntries,
|
||||
1,
|
||||
MAX_WORKER_EXECUTION_OFFER_JOURNAL_ENTRIES,
|
||||
);
|
||||
this.ownershipStaleMs = options.ownershipStaleMs ?? 30_000;
|
||||
assertIntegerBetween(
|
||||
'ownershipStaleMs',
|
||||
this.ownershipStaleMs,
|
||||
MIN_WORKER_OFFER_JOURNAL_LOCK_STALE_MS,
|
||||
MAX_WORKER_OFFER_JOURNAL_LOCK_STALE_MS,
|
||||
);
|
||||
this.lockProvider = options.lockProvider ?? properLockProvider;
|
||||
this.onOwnershipCompromised = options.onOwnershipCompromised;
|
||||
}
|
||||
|
||||
ownershipState(): WorkerExecutionOfferJournalOwnershipState {
|
||||
return this.ownerState;
|
||||
}
|
||||
|
||||
async acquireOwnership(): Promise<'acquired' | 'already_owned'> {
|
||||
if (this.ownerState === 'owned') return 'already_owned';
|
||||
if (this.ownerState === 'releasing') {
|
||||
throw new WorkerExecutionOfferJournalOwnershipError('not_owned');
|
||||
}
|
||||
if (this.ownerState === 'compromised') {
|
||||
throw new WorkerExecutionOfferJournalOwnershipError(
|
||||
'compromised',
|
||||
this.ownershipError,
|
||||
);
|
||||
}
|
||||
await this.ensureRoot();
|
||||
try {
|
||||
const release = await this.lockProvider.acquire({
|
||||
root: this.root,
|
||||
lockfilePath: path.join(this.root, '.owner.lock'),
|
||||
staleMs: this.ownershipStaleMs,
|
||||
updateMs: Math.max(1_000, Math.floor(this.ownershipStaleMs / 2)),
|
||||
onCompromised: (error) => this.compromiseOwnership(error),
|
||||
});
|
||||
this.releaseOwner = release;
|
||||
this.ownerState = 'owned';
|
||||
return 'acquired';
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ELOCKED')) {
|
||||
throw new WorkerExecutionOfferJournalOwnershipError(
|
||||
'already_owned',
|
||||
error,
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async releaseOwnership(): Promise<'released' | 'not_owned' | 'compromised'> {
|
||||
if (this.ownerState === 'unowned') return 'not_owned';
|
||||
if (this.ownerState === 'compromised') return 'compromised';
|
||||
if (this.ownerState === 'releasing') return 'not_owned';
|
||||
while (true) {
|
||||
const pending = this.mutationTail;
|
||||
await pending;
|
||||
if (pending === this.mutationTail) break;
|
||||
}
|
||||
const ownerStateAfterMutations = this.ownershipState();
|
||||
if (ownerStateAfterMutations === 'compromised') return 'compromised';
|
||||
if (ownerStateAfterMutations !== 'owned') return 'not_owned';
|
||||
const release = this.releaseOwner;
|
||||
if (!release) {
|
||||
this.compromiseOwnership(
|
||||
new Error('Worker offer journal owner release capability is missing'),
|
||||
);
|
||||
return 'compromised';
|
||||
}
|
||||
this.ownerState = 'releasing';
|
||||
try {
|
||||
await release();
|
||||
this.releaseOwner = undefined;
|
||||
this.ownerState = 'unowned';
|
||||
return 'released';
|
||||
} catch (error) {
|
||||
const compromised =
|
||||
error instanceof Error
|
||||
? error
|
||||
: new Error('Worker offer journal owner release failed');
|
||||
this.compromiseOwnership(compromised);
|
||||
throw new WorkerExecutionOfferJournalOwnershipError('compromised', error);
|
||||
}
|
||||
}
|
||||
|
||||
async create(
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
): Promise<WorkerExecutionOfferJournalCreateResult> {
|
||||
this.assertOwned();
|
||||
const candidate = cloneWorkerExecutionOfferJournalRecord(record);
|
||||
if (candidate.revision !== 0 || candidate.state !== 'accepted') {
|
||||
throw new TypeError(
|
||||
'A new Worker offer journal entry must be accepted at revision zero',
|
||||
);
|
||||
}
|
||||
return this.serializeMutation(async () => {
|
||||
const target = this.target(candidate.offer.offerId);
|
||||
if (await this.exists(target)) return 'exists';
|
||||
const names = await this.entryNames();
|
||||
if (names.length >= this.maximumEntries) {
|
||||
throw new WorkerExecutionOfferJournalCapacityError(this.maximumEntries);
|
||||
}
|
||||
const temporary = this.temporary(candidate.offer.offerId);
|
||||
try {
|
||||
await this.writeTemporary(temporary, candidate);
|
||||
this.assertOwned();
|
||||
await fs.link(temporary, target);
|
||||
await this.bestEffortSyncDirectory();
|
||||
return 'created';
|
||||
} catch (error) {
|
||||
if (isCode(error, 'EEXIST')) return 'exists';
|
||||
throw error;
|
||||
} finally {
|
||||
await this.bestEffortUnlink(temporary);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async read(
|
||||
offerId: string,
|
||||
): Promise<WorkerExecutionOfferJournalRecord | undefined> {
|
||||
this.assertOwned();
|
||||
assertRunDispatchOfferId(offerId);
|
||||
let handle: fs.FileHandle;
|
||||
try {
|
||||
handle = await fs.open(
|
||||
this.target(offerId),
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return undefined;
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (!stat.isFile()) {
|
||||
throw new TypeError(
|
||||
'Worker execution offer journal entry must be a regular file',
|
||||
);
|
||||
}
|
||||
const bytes = Buffer.allocUnsafe(
|
||||
MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES + 1,
|
||||
);
|
||||
const { bytesRead } = await handle.read(bytes, 0, bytes.length, 0);
|
||||
if (bytesRead > MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES) {
|
||||
throw new TypeError(
|
||||
'Worker execution offer journal entry exceeds the byte limit',
|
||||
);
|
||||
}
|
||||
const record = parseWorkerExecutionOfferJournalRecord(
|
||||
bytes.subarray(0, bytesRead),
|
||||
);
|
||||
if (record.offer.offerId !== offerId) {
|
||||
throw new TypeError(
|
||||
'Worker offer journal path and payload do not match',
|
||||
);
|
||||
}
|
||||
return record;
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
async replace(
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
expectedRevision: number,
|
||||
): Promise<void> {
|
||||
this.assertOwned();
|
||||
const candidate = cloneWorkerExecutionOfferJournalRecord(record);
|
||||
assertIntegerBetween(
|
||||
'expectedRevision',
|
||||
expectedRevision,
|
||||
0,
|
||||
Number.MAX_SAFE_INTEGER - 1,
|
||||
);
|
||||
if (candidate.revision !== expectedRevision + 1) {
|
||||
throw new TypeError('Replacement journal revision must increment by one');
|
||||
}
|
||||
await this.serializeMutation(async () => {
|
||||
const current = await this.read(candidate.offer.offerId);
|
||||
if (!current) {
|
||||
throw new WorkerExecutionOfferJournalNotFoundError(
|
||||
candidate.offer.offerId,
|
||||
);
|
||||
}
|
||||
if (current.revision !== expectedRevision) {
|
||||
throw new WorkerExecutionOfferJournalRevisionError(
|
||||
candidate.offer.offerId,
|
||||
);
|
||||
}
|
||||
const temporary = this.temporary(candidate.offer.offerId);
|
||||
try {
|
||||
await this.writeTemporary(temporary, candidate);
|
||||
this.assertOwned();
|
||||
await fs.rename(temporary, this.target(candidate.offer.offerId));
|
||||
await this.bestEffortSyncDirectory();
|
||||
} finally {
|
||||
await this.bestEffortUnlink(temporary);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async remove(offerId: string, expectedRevision?: number): Promise<boolean> {
|
||||
this.assertOwned();
|
||||
assertRunDispatchOfferId(offerId);
|
||||
if (expectedRevision !== undefined) {
|
||||
assertIntegerBetween(
|
||||
'expectedRevision',
|
||||
expectedRevision,
|
||||
0,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
);
|
||||
}
|
||||
return this.serializeMutation(async () => {
|
||||
if (expectedRevision !== undefined) {
|
||||
const current = await this.read(offerId);
|
||||
if (!current) return false;
|
||||
if (current.revision !== expectedRevision) {
|
||||
throw new WorkerExecutionOfferJournalRevisionError(offerId);
|
||||
}
|
||||
}
|
||||
try {
|
||||
this.assertOwned();
|
||||
await fs.unlink(this.target(offerId));
|
||||
await this.bestEffortSyncDirectory();
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async list(
|
||||
options: { afterOfferId?: string; limit?: number } = {},
|
||||
): Promise<WorkerExecutionOfferJournalPage> {
|
||||
this.assertOwned();
|
||||
if (options.afterOfferId !== undefined) {
|
||||
assertRunDispatchOfferId(options.afterOfferId);
|
||||
}
|
||||
const limit = options.limit ?? 32;
|
||||
assertIntegerBetween(
|
||||
'limit',
|
||||
limit,
|
||||
1,
|
||||
MAX_WORKER_EXECUTION_OFFER_JOURNAL_PAGE_SIZE,
|
||||
);
|
||||
const names = await this.entryNames();
|
||||
const offerIds = names
|
||||
.map((name) => JOURNAL_FILE_PATTERN.exec(name)?.[1])
|
||||
.filter((value): value is string => value !== undefined)
|
||||
.filter(
|
||||
(offerId) =>
|
||||
options.afterOfferId === undefined || offerId > options.afterOfferId,
|
||||
)
|
||||
.sort();
|
||||
const selected = offerIds.slice(0, limit + 1);
|
||||
const hasMore = selected.length > limit;
|
||||
const pageIds = selected.slice(0, limit);
|
||||
const records: WorkerExecutionOfferJournalRecord[] = [];
|
||||
for (const offerId of pageIds) {
|
||||
const record = await this.read(offerId);
|
||||
if (!record) {
|
||||
throw new WorkerExecutionOfferJournalRevisionError(offerId);
|
||||
}
|
||||
records.push(record);
|
||||
}
|
||||
return {
|
||||
records,
|
||||
...(hasMore && pageIds.length
|
||||
? { nextAfterOfferId: pageIds[pageIds.length - 1] }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
private target(offerId: string): string {
|
||||
assertRunDispatchOfferId(offerId);
|
||||
return path.join(this.root, `${offerId}.json`);
|
||||
}
|
||||
|
||||
private temporary(offerId: string): string {
|
||||
return path.join(
|
||||
this.root,
|
||||
`.${offerId}.${randomBytes(16).toString('hex')}.tmp`,
|
||||
);
|
||||
}
|
||||
|
||||
private async ensureRoot(): Promise<void> {
|
||||
await fs.mkdir(this.root, { recursive: true, mode: 0o700 });
|
||||
const stat = await fs.lstat(this.root);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new TypeError(
|
||||
'Worker execution offer journal root must be a real directory',
|
||||
);
|
||||
}
|
||||
await fs.chmod(this.root, 0o700);
|
||||
}
|
||||
|
||||
private assertOwned(): void {
|
||||
if (this.ownerState === 'owned') return;
|
||||
throw new WorkerExecutionOfferJournalOwnershipError(
|
||||
this.ownerState === 'compromised' ? 'compromised' : 'not_owned',
|
||||
this.ownershipError,
|
||||
);
|
||||
}
|
||||
|
||||
private compromiseOwnership(error: Error): void {
|
||||
this.ownershipError = error;
|
||||
this.releaseOwner = undefined;
|
||||
this.ownerState = 'compromised';
|
||||
try {
|
||||
this.onOwnershipCompromised?.(error);
|
||||
} catch {
|
||||
// Ownership loss must remain visible even if diagnostics fail.
|
||||
}
|
||||
}
|
||||
|
||||
private async entryNames(): Promise<string[]> {
|
||||
let names: string[];
|
||||
try {
|
||||
names = await fs.readdir(this.root);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return [];
|
||||
throw error;
|
||||
}
|
||||
const entries = names.filter((name) => JOURNAL_FILE_PATTERN.test(name));
|
||||
if (entries.length > this.maximumEntries) {
|
||||
throw new WorkerExecutionOfferJournalCapacityError(this.maximumEntries);
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
private async writeTemporary(
|
||||
temporary: string,
|
||||
record: WorkerExecutionOfferJournalRecord,
|
||||
): Promise<void> {
|
||||
const serialized = serializeWorkerExecutionOfferJournalRecord(record);
|
||||
let handle: fs.FileHandle | undefined;
|
||||
try {
|
||||
handle = await fs.open(temporary, 'wx', 0o600);
|
||||
await handle.writeFile(serialized, 'utf8');
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle?.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
private async exists(target: string): Promise<boolean> {
|
||||
try {
|
||||
const stat = await fs.lstat(target);
|
||||
if (!stat.isFile()) {
|
||||
throw new TypeError(
|
||||
'Worker execution offer journal target must be a regular file',
|
||||
);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async bestEffortUnlink(target: string): Promise<void> {
|
||||
try {
|
||||
await fs.unlink(target);
|
||||
} catch (error) {
|
||||
if (!isCode(error, 'ENOENT')) {
|
||||
// Temp cleanup is diagnostic-only; the atomically published record wins.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async bestEffortSyncDirectory(): Promise<void> {
|
||||
try {
|
||||
const handle = await fs.open(this.root, constants.O_RDONLY);
|
||||
try {
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
} catch {
|
||||
// Some supported filesystems cannot fsync directories.
|
||||
}
|
||||
}
|
||||
|
||||
private serializeMutation<T>(operation: () => Promise<T>): Promise<T> {
|
||||
const result = this.mutationTail.then(operation, operation);
|
||||
this.mutationTail = result.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,925 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
APPROVAL_REQUEST_TABLE,
|
||||
APPROVED_ACTION_DISPATCH_TABLE,
|
||||
} from '../../../migrations/0020-approval-requests';
|
||||
import { APPROVED_ACTION_DISPATCH_EXECUTION_TABLE } from '../../../migrations/0021-approved-action-dispatch-executions';
|
||||
import { DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS } from '../../domain/approvedActionDispatchExecution';
|
||||
import {
|
||||
ApprovalMutationConflictError,
|
||||
ApprovalPolicyFenceConflictError,
|
||||
ApprovalRequestExpiredError,
|
||||
ApprovalRequestNotFoundError,
|
||||
ApprovalRequestStateConflictError,
|
||||
ApprovalRequestVersionConflictError,
|
||||
ApprovalUnavailableError,
|
||||
InvalidApprovalValueError,
|
||||
normalizeApprovalActionBinding,
|
||||
normalizeApprovalPolicyFence,
|
||||
normalizeApprovalRequestRecord,
|
||||
normalizeApprovedActionDispatchRecord,
|
||||
sameApprovalAction,
|
||||
sameApprovalSubject,
|
||||
type ApprovalRequestRecord,
|
||||
type ApprovedActionDispatchRecord,
|
||||
} from '../../domain/approvalRequest';
|
||||
import {
|
||||
normalizePolicySubject,
|
||||
type ProjectPolicyFence,
|
||||
} from '../../domain/projectPolicy';
|
||||
import type {
|
||||
ApprovalRequestRepository,
|
||||
ConsumeApprovalRequestCommand,
|
||||
ConsumeApprovalRequestResult,
|
||||
CreateApprovalRequestCommand,
|
||||
CreateApprovalRequestResult,
|
||||
DecideApprovalRequestCommand,
|
||||
DecideApprovalRequestResult,
|
||||
} from '../../ports/approvalRequestRepository';
|
||||
import {
|
||||
PROJECT_ROLE_BINDING_TABLE,
|
||||
PROJECT_TABLE,
|
||||
} from '../../../migrations/0017-project-policy';
|
||||
|
||||
const RETRY_ATTEMPTS = 5;
|
||||
|
||||
interface ApprovalRequestRow {
|
||||
id: string;
|
||||
projectId: string;
|
||||
version: number;
|
||||
state: string;
|
||||
permission: string;
|
||||
actionType: string;
|
||||
actionRef: string;
|
||||
actionDigest: string;
|
||||
previewDigest: string;
|
||||
risk: string;
|
||||
requestedByType: string;
|
||||
requestedById: string;
|
||||
requestedAtMs: number | string;
|
||||
expiresAtMs: number | string;
|
||||
decisionId: string | null;
|
||||
decision: string | null;
|
||||
decisionReasonCode: string | null;
|
||||
decidedByType: string | null;
|
||||
decidedById: string | null;
|
||||
decidedAtMs: number | string | null;
|
||||
consumptionId: string | null;
|
||||
dispatchId: string | null;
|
||||
consumedByType: string | null;
|
||||
consumedById: string | null;
|
||||
consumedAtMs: number | string | null;
|
||||
}
|
||||
|
||||
interface ApprovalRequestInstance
|
||||
extends Model<ApprovalRequestRow, ApprovalRequestRow>,
|
||||
ApprovalRequestRow {}
|
||||
|
||||
interface ApprovedActionDispatchRow {
|
||||
id: string;
|
||||
approvalRequestId: string;
|
||||
approvalRequestVersion: number;
|
||||
projectId: string;
|
||||
state: string;
|
||||
permission: string;
|
||||
actionType: string;
|
||||
actionRef: string;
|
||||
actionDigest: string;
|
||||
previewDigest: string;
|
||||
requestedByType: string;
|
||||
requestedById: string;
|
||||
consumedByType: string;
|
||||
consumedById: string;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface ApprovedActionDispatchInstance
|
||||
extends Model<ApprovedActionDispatchRow, ApprovedActionDispatchRow>,
|
||||
ApprovedActionDispatchRow {}
|
||||
|
||||
interface PolicyFenceRow {
|
||||
project_version: number | string;
|
||||
binding_version: number | string | null;
|
||||
}
|
||||
|
||||
function defineApprovalRequestModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<ApprovalRequestInstance> {
|
||||
return database.define<ApprovalRequestInstance>(
|
||||
'Ql3ApprovalRequest',
|
||||
{
|
||||
id: { type: DataTypes.STRING(64), allowNull: false, primaryKey: true },
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
},
|
||||
version: { type: DataTypes.INTEGER, allowNull: false },
|
||||
state: { type: DataTypes.STRING(16), allowNull: false },
|
||||
permission: { type: DataTypes.STRING(255), allowNull: false },
|
||||
actionType: {
|
||||
field: 'action_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
actionRef: {
|
||||
field: 'action_ref',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
actionDigest: {
|
||||
field: 'action_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
previewDigest: {
|
||||
field: 'preview_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
risk: { type: DataTypes.STRING(16), allowNull: false },
|
||||
requestedByType: {
|
||||
field: 'requested_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
requestedById: {
|
||||
field: 'requested_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
requestedAtMs: {
|
||||
field: 'requested_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
expiresAtMs: {
|
||||
field: 'expires_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
decisionId: {
|
||||
field: 'decision_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: true,
|
||||
},
|
||||
decision: { type: DataTypes.STRING(16), allowNull: true },
|
||||
decisionReasonCode: {
|
||||
field: 'decision_reason_code',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: true,
|
||||
},
|
||||
decidedByType: {
|
||||
field: 'decided_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: true,
|
||||
},
|
||||
decidedById: {
|
||||
field: 'decided_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: true,
|
||||
},
|
||||
decidedAtMs: {
|
||||
field: 'decided_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
consumptionId: {
|
||||
field: 'consumption_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: true,
|
||||
},
|
||||
dispatchId: {
|
||||
field: 'dispatch_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: true,
|
||||
},
|
||||
consumedByType: {
|
||||
field: 'consumed_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: true,
|
||||
},
|
||||
consumedById: {
|
||||
field: 'consumed_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: true,
|
||||
},
|
||||
consumedAtMs: {
|
||||
field: 'consumed_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: APPROVAL_REQUEST_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function defineApprovedActionDispatchModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<ApprovedActionDispatchInstance> {
|
||||
return database.define<ApprovedActionDispatchInstance>(
|
||||
'Ql3ApprovedActionDispatch',
|
||||
{
|
||||
id: { type: DataTypes.STRING(64), allowNull: false, primaryKey: true },
|
||||
approvalRequestId: {
|
||||
field: 'approval_request_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
approvalRequestVersion: {
|
||||
field: 'approval_request_version',
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
},
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
},
|
||||
state: { type: DataTypes.STRING(16), allowNull: false },
|
||||
permission: { type: DataTypes.STRING(255), allowNull: false },
|
||||
actionType: {
|
||||
field: 'action_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
actionRef: {
|
||||
field: 'action_ref',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
actionDigest: {
|
||||
field: 'action_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
previewDigest: {
|
||||
field: 'preview_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
requestedByType: {
|
||||
field: 'requested_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
requestedById: {
|
||||
field: 'requested_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
consumedByType: {
|
||||
field: 'consumed_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
consumedById: {
|
||||
field: 'consumed_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: APPROVED_ACTION_DISPATCH_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function rowToRequest(
|
||||
row: ApprovalRequestRow,
|
||||
): Readonly<ApprovalRequestRecord> {
|
||||
try {
|
||||
return normalizeApprovalRequestRecord({
|
||||
id: row.id,
|
||||
projectId: row.projectId,
|
||||
version: Number(row.version),
|
||||
state: row.state as ApprovalRequestRecord['state'],
|
||||
action: {
|
||||
permission:
|
||||
row.permission as ApprovalRequestRecord['action']['permission'],
|
||||
actionType: row.actionType,
|
||||
actionRef: row.actionRef,
|
||||
actionDigest: row.actionDigest,
|
||||
previewDigest: row.previewDigest,
|
||||
},
|
||||
risk: row.risk as ApprovalRequestRecord['risk'],
|
||||
requestedBy: {
|
||||
type: row.requestedByType as ApprovalRequestRecord['requestedBy']['type'],
|
||||
id: row.requestedById,
|
||||
},
|
||||
requestedAtMs: Number(row.requestedAtMs),
|
||||
expiresAtMs: Number(row.expiresAtMs),
|
||||
decisionId: row.decisionId,
|
||||
decision: row.decision as ApprovalRequestRecord['decision'],
|
||||
decisionReasonCode: row.decisionReasonCode,
|
||||
decidedBy:
|
||||
row.decidedByType === null || row.decidedById === null
|
||||
? null
|
||||
: {
|
||||
type: row.decidedByType as NonNullable<
|
||||
ApprovalRequestRecord['decidedBy']
|
||||
>['type'],
|
||||
id: row.decidedById,
|
||||
},
|
||||
decidedAtMs: row.decidedAtMs === null ? null : Number(row.decidedAtMs),
|
||||
consumptionId: row.consumptionId,
|
||||
dispatchId: row.dispatchId,
|
||||
consumedBy:
|
||||
row.consumedByType === null || row.consumedById === null
|
||||
? null
|
||||
: {
|
||||
type: row.consumedByType as NonNullable<
|
||||
ApprovalRequestRecord['consumedBy']
|
||||
>['type'],
|
||||
id: row.consumedById,
|
||||
},
|
||||
consumedAtMs: row.consumedAtMs === null ? null : Number(row.consumedAtMs),
|
||||
});
|
||||
} catch {
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function requestToRow(
|
||||
request: Readonly<ApprovalRequestRecord>,
|
||||
): ApprovalRequestRow {
|
||||
return {
|
||||
id: request.id,
|
||||
projectId: request.projectId,
|
||||
version: request.version,
|
||||
state: request.state,
|
||||
permission: request.action.permission,
|
||||
actionType: request.action.actionType,
|
||||
actionRef: request.action.actionRef,
|
||||
actionDigest: request.action.actionDigest,
|
||||
previewDigest: request.action.previewDigest,
|
||||
risk: request.risk,
|
||||
requestedByType: request.requestedBy.type,
|
||||
requestedById: request.requestedBy.id,
|
||||
requestedAtMs: request.requestedAtMs,
|
||||
expiresAtMs: request.expiresAtMs,
|
||||
decisionId: request.decisionId,
|
||||
decision: request.decision,
|
||||
decisionReasonCode: request.decisionReasonCode,
|
||||
decidedByType: request.decidedBy?.type ?? null,
|
||||
decidedById: request.decidedBy?.id ?? null,
|
||||
decidedAtMs: request.decidedAtMs,
|
||||
consumptionId: request.consumptionId,
|
||||
dispatchId: request.dispatchId,
|
||||
consumedByType: request.consumedBy?.type ?? null,
|
||||
consumedById: request.consumedBy?.id ?? null,
|
||||
consumedAtMs: request.consumedAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function rowToDispatch(
|
||||
row: ApprovedActionDispatchRow,
|
||||
): Readonly<ApprovedActionDispatchRecord> {
|
||||
try {
|
||||
return normalizeApprovedActionDispatchRecord({
|
||||
id: row.id,
|
||||
approvalRequestId: row.approvalRequestId,
|
||||
approvalRequestVersion: Number(row.approvalRequestVersion),
|
||||
projectId: row.projectId,
|
||||
state: row.state as ApprovedActionDispatchRecord['state'],
|
||||
action: {
|
||||
permission:
|
||||
row.permission as ApprovedActionDispatchRecord['action']['permission'],
|
||||
actionType: row.actionType,
|
||||
actionRef: row.actionRef,
|
||||
actionDigest: row.actionDigest,
|
||||
previewDigest: row.previewDigest,
|
||||
},
|
||||
requestedBy: {
|
||||
type: row.requestedByType as ApprovedActionDispatchRecord['requestedBy']['type'],
|
||||
id: row.requestedById,
|
||||
},
|
||||
consumedBy: {
|
||||
type: row.consumedByType as ApprovedActionDispatchRecord['consumedBy']['type'],
|
||||
id: row.consumedById,
|
||||
},
|
||||
createdAtMs: Number(row.createdAtMs),
|
||||
});
|
||||
} catch {
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function dispatchToRow(
|
||||
dispatch: Readonly<ApprovedActionDispatchRecord>,
|
||||
): ApprovedActionDispatchRow {
|
||||
return {
|
||||
id: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
approvalRequestVersion: dispatch.approvalRequestVersion,
|
||||
projectId: dispatch.projectId,
|
||||
state: dispatch.state,
|
||||
permission: dispatch.action.permission,
|
||||
actionType: dispatch.action.actionType,
|
||||
actionRef: dispatch.action.actionRef,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
previewDigest: dispatch.action.previewDigest,
|
||||
requestedByType: dispatch.requestedBy.type,
|
||||
requestedById: dispatch.requestedBy.id,
|
||||
consumedByType: dispatch.consumedBy.type,
|
||||
consumedById: dispatch.consumedBy.id,
|
||||
createdAtMs: dispatch.createdAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function sameRequestCreation(
|
||||
left: Readonly<ApprovalRequestRecord>,
|
||||
right: Readonly<ApprovalRequestRecord>,
|
||||
): boolean {
|
||||
return (
|
||||
left.id === right.id &&
|
||||
left.projectId === right.projectId &&
|
||||
sameApprovalAction(left.action, right.action) &&
|
||||
left.risk === right.risk &&
|
||||
sameApprovalSubject(left.requestedBy, right.requestedBy) &&
|
||||
left.requestedAtMs === right.requestedAtMs &&
|
||||
left.expiresAtMs === right.expiresAtMs
|
||||
);
|
||||
}
|
||||
|
||||
function sameDispatch(
|
||||
left: Readonly<ApprovedActionDispatchRecord>,
|
||||
right: Readonly<ApprovedActionDispatchRecord>,
|
||||
): boolean {
|
||||
return (
|
||||
left.id === right.id &&
|
||||
left.approvalRequestId === right.approvalRequestId &&
|
||||
left.approvalRequestVersion === right.approvalRequestVersion &&
|
||||
left.projectId === right.projectId &&
|
||||
left.state === right.state &&
|
||||
sameApprovalAction(left.action, right.action) &&
|
||||
sameApprovalSubject(left.requestedBy, right.requestedBy) &&
|
||||
sameApprovalSubject(left.consumedBy, right.consumedBy) &&
|
||||
left.createdAtMs === right.createdAtMs
|
||||
);
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
function isApprovalError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof ApprovalMutationConflictError ||
|
||||
error instanceof InvalidApprovalValueError ||
|
||||
error instanceof ApprovalPolicyFenceConflictError ||
|
||||
error instanceof ApprovalRequestExpiredError ||
|
||||
error instanceof ApprovalRequestNotFoundError ||
|
||||
error instanceof ApprovalRequestStateConflictError ||
|
||||
error instanceof ApprovalRequestVersionConflictError ||
|
||||
error instanceof ApprovalUnavailableError
|
||||
);
|
||||
}
|
||||
|
||||
export class LegacySequelizeApprovalRequestRepository
|
||||
implements ApprovalRequestRepository
|
||||
{
|
||||
private readonly requests: ModelStatic<ApprovalRequestInstance>;
|
||||
private readonly dispatches: ModelStatic<ApprovedActionDispatchInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Approval request repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.requests = defineApprovalRequestModel(database);
|
||||
this.dispatches = defineApprovedActionDispatchModel(database);
|
||||
}
|
||||
|
||||
async findById(id: string): Promise<Readonly<ApprovalRequestRecord> | null> {
|
||||
const row = await this.requests.findByPk(id, { raw: true });
|
||||
return row ? rowToRequest(row) : null;
|
||||
}
|
||||
|
||||
private async assertFence(
|
||||
projectId: string,
|
||||
subject: Readonly<ApprovalRequestRecord['requestedBy']>,
|
||||
requestedFence: Readonly<ProjectPolicyFence>,
|
||||
transaction: Transaction,
|
||||
): Promise<void> {
|
||||
const normalizedSubject = normalizePolicySubject(subject);
|
||||
const fence = normalizeApprovalPolicyFence(requestedFence);
|
||||
const rows = await this.database.query<PolicyFenceRow>(
|
||||
`SELECT project.version AS project_version,
|
||||
(SELECT MAX(binding.version)
|
||||
FROM "${PROJECT_ROLE_BINDING_TABLE}" AS binding
|
||||
WHERE binding.project_id = project.id
|
||||
AND binding.subject_type = :subjectType
|
||||
AND binding.subject_id = :subjectId) AS binding_version
|
||||
FROM "${PROJECT_TABLE}" AS project
|
||||
WHERE project.id = :projectId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
projectId,
|
||||
subjectType: normalizedSubject.type,
|
||||
subjectId: normalizedSubject.id,
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (rows.length !== 1) throw new ApprovalPolicyFenceConflictError();
|
||||
const currentProjectVersion = Number(rows[0].project_version);
|
||||
const currentBindingVersion =
|
||||
rows[0].binding_version === null ? null : Number(rows[0].binding_version);
|
||||
if (
|
||||
currentProjectVersion !== fence.projectVersion ||
|
||||
currentBindingVersion !== fence.bindingVersion
|
||||
) {
|
||||
throw new ApprovalPolicyFenceConflictError();
|
||||
}
|
||||
}
|
||||
|
||||
private async findDecisionReplay(
|
||||
decisionId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<Readonly<ApprovalRequestRecord> | null> {
|
||||
const row = await this.requests.findOne({
|
||||
where: { decisionId },
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
return row ? rowToRequest(row) : null;
|
||||
}
|
||||
|
||||
private async findConsumptionReplay(
|
||||
consumptionId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<{
|
||||
request: Readonly<ApprovalRequestRecord>;
|
||||
dispatch: Readonly<ApprovedActionDispatchRecord>;
|
||||
} | null> {
|
||||
const row = await this.requests.findOne({
|
||||
where: { consumptionId },
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!row) return null;
|
||||
const request = rowToRequest(row);
|
||||
if (!request.dispatchId) throw new ApprovalUnavailableError();
|
||||
const dispatchRow = await this.dispatches.findByPk(request.dispatchId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!dispatchRow) throw new ApprovalUnavailableError();
|
||||
const executionRows = await this.database.query<{
|
||||
dispatch_id: string;
|
||||
project_id: string;
|
||||
}>(
|
||||
`SELECT dispatch_id, project_id
|
||||
FROM "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
WHERE dispatch_id = :dispatchId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { dispatchId: request.dispatchId },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (
|
||||
executionRows.length !== 1 ||
|
||||
executionRows[0].project_id !== request.projectId
|
||||
) {
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
return { request, dispatch: rowToDispatch(dispatchRow) };
|
||||
}
|
||||
|
||||
async create(
|
||||
command: CreateApprovalRequestCommand,
|
||||
): Promise<CreateApprovalRequestResult> {
|
||||
const request = normalizeApprovalRequestRecord(command.request);
|
||||
const fence = normalizeApprovalPolicyFence(command.authorizationFence);
|
||||
if (request.state !== 'pending' || request.version !== 1) {
|
||||
throw new ApprovalRequestStateConflictError();
|
||||
}
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const existing = await this.requests.findByPk(request.id, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (existing) {
|
||||
const previous = rowToRequest(existing);
|
||||
if (!sameRequestCreation(previous, request)) {
|
||||
throw new ApprovalMutationConflictError();
|
||||
}
|
||||
return { status: 'existing', request: previous };
|
||||
}
|
||||
await this.assertFence(
|
||||
request.projectId,
|
||||
request.requestedBy,
|
||||
fence,
|
||||
transaction,
|
||||
);
|
||||
await this.requests.create(requestToRow(request), { transaction });
|
||||
return { status: 'created', request };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (isApprovalError(error)) throw error;
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
|
||||
async decide(
|
||||
command: DecideApprovalRequestCommand,
|
||||
): Promise<DecideApprovalRequestResult> {
|
||||
const decidedBy = normalizePolicySubject(command.decidedBy);
|
||||
const fence = normalizeApprovalPolicyFence(command.authorizationFence);
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const replay = await this.findDecisionReplay(
|
||||
command.decisionId,
|
||||
transaction,
|
||||
);
|
||||
if (replay) {
|
||||
if (
|
||||
replay.id !== command.requestId ||
|
||||
command.expectedVersion !== 1 ||
|
||||
replay.decisionId !== command.decisionId ||
|
||||
replay.decision !== command.decision ||
|
||||
replay.decisionReasonCode !== command.reasonCode ||
|
||||
!replay.decidedBy ||
|
||||
!sameApprovalSubject(replay.decidedBy, decidedBy) ||
|
||||
replay.decidedAtMs !== command.decidedAtMs
|
||||
) {
|
||||
throw new ApprovalMutationConflictError();
|
||||
}
|
||||
return { status: 'existing', request: replay };
|
||||
}
|
||||
const row = await this.requests.findByPk(command.requestId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!row) throw new ApprovalRequestNotFoundError();
|
||||
const current = rowToRequest(row);
|
||||
if (command.decidedAtMs >= current.expiresAtMs) {
|
||||
throw new ApprovalRequestExpiredError();
|
||||
}
|
||||
if (current.version !== command.expectedVersion) {
|
||||
throw new ApprovalRequestVersionConflictError();
|
||||
}
|
||||
if (current.state !== 'pending') {
|
||||
throw new ApprovalRequestStateConflictError();
|
||||
}
|
||||
await this.assertFence(
|
||||
current.projectId,
|
||||
decidedBy,
|
||||
fence,
|
||||
transaction,
|
||||
);
|
||||
const decided = normalizeApprovalRequestRecord({
|
||||
...current,
|
||||
version: 2,
|
||||
state: command.decision,
|
||||
decisionId: command.decisionId,
|
||||
decision: command.decision,
|
||||
decisionReasonCode: command.reasonCode,
|
||||
decidedBy,
|
||||
decidedAtMs: command.decidedAtMs,
|
||||
});
|
||||
const [updated] = await this.requests.update(
|
||||
requestToRow(decided),
|
||||
{
|
||||
where: {
|
||||
id: current.id,
|
||||
version: command.expectedVersion,
|
||||
state: 'pending',
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (updated !== 1) {
|
||||
throw new ApprovalRequestVersionConflictError();
|
||||
}
|
||||
return { status: 'decided', request: decided };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (isApprovalError(error)) throw error;
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
|
||||
async consume(
|
||||
command: ConsumeApprovalRequestCommand,
|
||||
): Promise<ConsumeApprovalRequestResult> {
|
||||
const action = normalizeApprovalActionBinding(command.action);
|
||||
const requestedBy = normalizePolicySubject(command.requestedBy);
|
||||
const consumedBy = normalizePolicySubject(command.consumedBy);
|
||||
const fence = normalizeApprovalPolicyFence(command.authorizationFence);
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const replay = await this.findConsumptionReplay(
|
||||
command.consumptionId,
|
||||
transaction,
|
||||
);
|
||||
if (replay) {
|
||||
const expectedDispatch = normalizeApprovedActionDispatchRecord({
|
||||
id: command.dispatchId,
|
||||
approvalRequestId: command.requestId,
|
||||
approvalRequestVersion: 3,
|
||||
projectId: replay.request.projectId,
|
||||
state: 'pending',
|
||||
action,
|
||||
requestedBy,
|
||||
consumedBy,
|
||||
createdAtMs: command.consumedAtMs,
|
||||
});
|
||||
if (
|
||||
command.expectedVersion !== 2 ||
|
||||
replay.request.id !== command.requestId ||
|
||||
replay.request.consumptionId !== command.consumptionId ||
|
||||
!sameDispatch(replay.dispatch, expectedDispatch)
|
||||
) {
|
||||
throw new ApprovalMutationConflictError();
|
||||
}
|
||||
return {
|
||||
status: 'existing',
|
||||
request: replay.request,
|
||||
dispatch: replay.dispatch,
|
||||
};
|
||||
}
|
||||
const row = await this.requests.findByPk(command.requestId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!row) throw new ApprovalRequestNotFoundError();
|
||||
const current = rowToRequest(row);
|
||||
if (command.consumedAtMs >= current.expiresAtMs) {
|
||||
throw new ApprovalRequestExpiredError();
|
||||
}
|
||||
if (current.version !== command.expectedVersion) {
|
||||
throw new ApprovalRequestVersionConflictError();
|
||||
}
|
||||
if (current.state !== 'approved') {
|
||||
throw new ApprovalRequestStateConflictError();
|
||||
}
|
||||
if (
|
||||
!sameApprovalAction(current.action, action) ||
|
||||
!sameApprovalSubject(current.requestedBy, requestedBy)
|
||||
) {
|
||||
throw new ApprovalMutationConflictError();
|
||||
}
|
||||
await this.assertFence(
|
||||
current.projectId,
|
||||
requestedBy,
|
||||
fence,
|
||||
transaction,
|
||||
);
|
||||
const dispatch = normalizeApprovedActionDispatchRecord({
|
||||
id: command.dispatchId,
|
||||
approvalRequestId: current.id,
|
||||
approvalRequestVersion: 3,
|
||||
projectId: current.projectId,
|
||||
state: 'pending',
|
||||
action,
|
||||
requestedBy,
|
||||
consumedBy,
|
||||
createdAtMs: command.consumedAtMs,
|
||||
});
|
||||
const dispatchCollision = await this.dispatches.findByPk(
|
||||
dispatch.id,
|
||||
{ raw: true, transaction },
|
||||
);
|
||||
if (dispatchCollision) throw new ApprovalMutationConflictError();
|
||||
await this.dispatches.create(dispatchToRow(dispatch), {
|
||||
transaction,
|
||||
});
|
||||
await this.database.query(
|
||||
`INSERT INTO "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
(dispatch_id, project_id, status, version, attempt_count,
|
||||
max_attempts, eligible_at_ms, next_attempt_at_ms,
|
||||
lease_owner, lease_token, lease_expires_at_ms, started_at_ms,
|
||||
last_result_code, completed_at_ms, created_at_ms, updated_at_ms)
|
||||
VALUES
|
||||
(:dispatchId, :projectId, 'pending', 0, 0, :maxAttempts,
|
||||
:createdAtMs, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
|
||||
:createdAtMs, :createdAtMs)`,
|
||||
{
|
||||
replacements: {
|
||||
dispatchId: dispatch.id,
|
||||
projectId: dispatch.projectId,
|
||||
maxAttempts: DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS,
|
||||
createdAtMs: dispatch.createdAtMs,
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
const consumed = normalizeApprovalRequestRecord({
|
||||
...current,
|
||||
version: 3,
|
||||
state: 'consumed',
|
||||
consumptionId: command.consumptionId,
|
||||
dispatchId: command.dispatchId,
|
||||
consumedBy,
|
||||
consumedAtMs: command.consumedAtMs,
|
||||
});
|
||||
const [updated] = await this.requests.update(
|
||||
requestToRow(consumed),
|
||||
{
|
||||
where: {
|
||||
id: current.id,
|
||||
version: command.expectedVersion,
|
||||
state: 'approved',
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (updated !== 1) {
|
||||
throw new ApprovalRequestVersionConflictError();
|
||||
}
|
||||
return { status: 'consumed', request: consumed, dispatch };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (isApprovalError(error)) throw error;
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,427 @@
|
||||
import {
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
type Transaction as SequelizeTransaction,
|
||||
} from 'sequelize';
|
||||
import { APPROVED_RUN_ACTION_RECEIPT_TABLE } from '../../../migrations/0023-approved-run-action-receipts';
|
||||
import { APPROVED_ACTION_DISPATCH_EXECUTION_TABLE } from '../../../migrations/0021-approved-action-dispatch-executions';
|
||||
import {
|
||||
APPROVED_RUN_ACTION_TYPE,
|
||||
APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
ApprovedRunActionBindingConflictError,
|
||||
ApprovedRunActionRepositoryError,
|
||||
InvalidApprovedRunActionError,
|
||||
digestApprovedRunCreationPlan,
|
||||
digestApprovedRunCreationReceipt,
|
||||
normalizeApprovedRunCreationPlan,
|
||||
normalizeApprovedRunCreationReceipt,
|
||||
type ApprovedRunCreationReceipt,
|
||||
} from '../../domain/approvedRunAction';
|
||||
import {
|
||||
normalizeApprovedActionDispatchExecutionRecord,
|
||||
type ApprovedActionDispatchExecutionSnapshot,
|
||||
} from '../../domain/approvedActionDispatchExecution';
|
||||
import { normalizeApprovedActionDispatchRecord } from '../../domain/approvalRequest';
|
||||
import { DuplicateIdempotencyKeyError } from '../../domain/repositoryErrors';
|
||||
import type { RunRecord } from '../../domain/run';
|
||||
import {
|
||||
PrimaryRunCreator,
|
||||
type PrimaryRunIdFactory,
|
||||
} from '../../application/primaryRunCreator';
|
||||
import type {
|
||||
ApprovedRunActionRepository,
|
||||
ApprovedRunReference,
|
||||
CreateApprovedRunCommand,
|
||||
} from '../../ports/approvedRunActionRepository';
|
||||
import type { RunRepositoryTransaction } from '../../ports/runRepository';
|
||||
import {
|
||||
LegacySequelizeRunRepository,
|
||||
LegacySequelizeRunTransaction,
|
||||
} from './runRepository';
|
||||
|
||||
interface ApprovedRunReceiptRow {
|
||||
schema_version: number;
|
||||
dispatch_id: string;
|
||||
approval_request_id: string;
|
||||
project_id: string;
|
||||
action_type: string;
|
||||
action_digest: string;
|
||||
execution_attempt: number;
|
||||
execution_version: number;
|
||||
started_at_ms: number;
|
||||
idempotency_key: string;
|
||||
outcome: string;
|
||||
result_code: string;
|
||||
resource_type: string;
|
||||
resource_id: string;
|
||||
finished_at_ms: number;
|
||||
evidence_digest: string;
|
||||
created_at_ms: number;
|
||||
}
|
||||
|
||||
interface ReceiptBinding {
|
||||
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>;
|
||||
clock: () => number;
|
||||
}
|
||||
|
||||
interface ExecutionFenceRow {
|
||||
project_id: string;
|
||||
status: string;
|
||||
version: number;
|
||||
attempt_count: number;
|
||||
lease_owner: string | null;
|
||||
lease_token: string | null;
|
||||
started_at_ms: number | null;
|
||||
}
|
||||
|
||||
function rowToReceipt(
|
||||
row: ApprovedRunReceiptRow,
|
||||
): Readonly<ApprovedRunCreationReceipt> {
|
||||
return normalizeApprovedRunCreationReceipt({
|
||||
schemaVersion: row.schema_version as 1,
|
||||
dispatchId: row.dispatch_id,
|
||||
approvalRequestId: row.approval_request_id,
|
||||
projectId: row.project_id,
|
||||
actionType: row.action_type as typeof APPROVED_RUN_ACTION_TYPE,
|
||||
actionDigest: row.action_digest,
|
||||
executionAttempt: row.execution_attempt,
|
||||
executionVersion: row.execution_version,
|
||||
startedAtMs: row.started_at_ms,
|
||||
idempotencyKey: row.idempotency_key,
|
||||
outcome: row.outcome as 'succeeded',
|
||||
resultCode: row.result_code as typeof APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
resourceType: row.resource_type as 'run',
|
||||
resourceId: row.resource_id,
|
||||
finishedAtMs: row.finished_at_ms,
|
||||
evidenceDigest: row.evidence_digest,
|
||||
createdAtMs: row.created_at_ms,
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeSnapshot(
|
||||
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
|
||||
): Readonly<ApprovedActionDispatchExecutionSnapshot> {
|
||||
if (!snapshot || typeof snapshot !== 'object' || Array.isArray(snapshot)) {
|
||||
throw new InvalidApprovedRunActionError('execution snapshot is invalid');
|
||||
}
|
||||
const dispatch = normalizeApprovedActionDispatchRecord(snapshot.dispatch);
|
||||
const execution = normalizeApprovedActionDispatchExecutionRecord(
|
||||
snapshot.execution,
|
||||
);
|
||||
if (
|
||||
execution.dispatchId !== dispatch.id ||
|
||||
execution.projectId !== dispatch.projectId ||
|
||||
execution.status !== 'executing' ||
|
||||
execution.startedAtMs === null
|
||||
) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
return Object.freeze({ dispatch, execution });
|
||||
}
|
||||
|
||||
function receiptMatches(
|
||||
receipt: Readonly<ApprovedRunCreationReceipt>,
|
||||
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
|
||||
): boolean {
|
||||
return (
|
||||
receipt.dispatchId === snapshot.dispatch.id &&
|
||||
receipt.approvalRequestId === snapshot.dispatch.approvalRequestId &&
|
||||
receipt.projectId === snapshot.dispatch.projectId &&
|
||||
receipt.actionType === snapshot.dispatch.action.actionType &&
|
||||
receipt.actionDigest === snapshot.dispatch.action.actionDigest &&
|
||||
receipt.executionAttempt === snapshot.execution.attemptCount &&
|
||||
receipt.startedAtMs === snapshot.execution.startedAtMs &&
|
||||
receipt.idempotencyKey === snapshot.dispatch.id
|
||||
);
|
||||
}
|
||||
|
||||
class AtomicApprovedRunRepository extends LegacySequelizeRunRepository {
|
||||
constructor(
|
||||
private readonly approvedDatabase: Sequelize,
|
||||
private readonly binding: Readonly<ReceiptBinding>,
|
||||
) {
|
||||
super(approvedDatabase);
|
||||
}
|
||||
|
||||
override async transaction<T>(
|
||||
work: (transaction: RunRepositoryTransaction) => Promise<T>,
|
||||
): Promise<T> {
|
||||
return this.approvedDatabase.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const executionVersion = await this.requireCurrentExecutionFence(
|
||||
transaction,
|
||||
);
|
||||
const result = await work(
|
||||
new LegacySequelizeRunTransaction(this.models, transaction),
|
||||
);
|
||||
const run = this.requireCreatedRun(result);
|
||||
await this.insertReceipt(run, executionVersion, transaction);
|
||||
return result;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async requireCurrentExecutionFence(
|
||||
transaction: SequelizeTransaction,
|
||||
): Promise<number> {
|
||||
const { dispatch, execution } = this.binding.snapshot;
|
||||
const rows = await this.approvedDatabase.query<ExecutionFenceRow>(
|
||||
`SELECT project_id, status, version, attempt_count, lease_owner,
|
||||
lease_token, started_at_ms
|
||||
FROM "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
WHERE dispatch_id = :dispatchId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { dispatchId: dispatch.id },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
const current = rows[0];
|
||||
if (
|
||||
rows.length !== 1 ||
|
||||
current.project_id !== dispatch.projectId ||
|
||||
current.status !== 'executing' ||
|
||||
current.version < execution.version ||
|
||||
current.attempt_count !== execution.attemptCount ||
|
||||
current.lease_owner !== execution.leaseOwner ||
|
||||
current.lease_token !== execution.leaseToken ||
|
||||
current.started_at_ms !== execution.startedAtMs
|
||||
) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
return current.version;
|
||||
}
|
||||
|
||||
private requireCreatedRun(value: unknown): Readonly<RunRecord> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!('id' in value) ||
|
||||
typeof value.id !== 'string' ||
|
||||
!('projectId' in value) ||
|
||||
value.projectId !== this.binding.snapshot.dispatch.projectId ||
|
||||
!('idempotencyKey' in value) ||
|
||||
value.idempotencyKey !== this.binding.snapshot.dispatch.id ||
|
||||
!('requestId' in value) ||
|
||||
value.requestId !== this.binding.snapshot.dispatch.approvalRequestId ||
|
||||
!('status' in value) ||
|
||||
value.status !== 'queued'
|
||||
) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
return value as Readonly<RunRecord>;
|
||||
}
|
||||
|
||||
private async insertReceipt(
|
||||
run: Readonly<RunRecord>,
|
||||
executionVersion: number,
|
||||
transaction: SequelizeTransaction,
|
||||
): Promise<void> {
|
||||
const { dispatch, execution } = this.binding.snapshot;
|
||||
const finishedAtMs = this.nowAtOrAfter(execution.startedAtMs!);
|
||||
const unsigned: Omit<ApprovedRunCreationReceipt, 'evidenceDigest'> = {
|
||||
schemaVersion: 1,
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
projectId: dispatch.projectId,
|
||||
actionType: APPROVED_RUN_ACTION_TYPE,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
executionAttempt: execution.attemptCount,
|
||||
executionVersion,
|
||||
startedAtMs: execution.startedAtMs!,
|
||||
idempotencyKey: dispatch.id,
|
||||
outcome: 'succeeded',
|
||||
resultCode: APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
resourceType: 'run',
|
||||
resourceId: run.id,
|
||||
finishedAtMs,
|
||||
createdAtMs: finishedAtMs,
|
||||
};
|
||||
const receipt = normalizeApprovedRunCreationReceipt({
|
||||
...unsigned,
|
||||
evidenceDigest: digestApprovedRunCreationReceipt(unsigned),
|
||||
});
|
||||
await this.approvedDatabase.query(
|
||||
`INSERT INTO "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
(dispatch_id, approval_request_id, project_id, schema_version,
|
||||
action_type, action_digest, execution_attempt, execution_version,
|
||||
started_at_ms, idempotency_key, outcome, result_code, resource_type,
|
||||
resource_id, finished_at_ms, evidence_digest, created_at_ms)
|
||||
VALUES
|
||||
(:dispatchId, :approvalRequestId, :projectId, :schemaVersion,
|
||||
:actionType, :actionDigest, :executionAttempt, :executionVersion,
|
||||
:startedAtMs, :idempotencyKey, :outcome, :resultCode, :resourceType,
|
||||
:resourceId, :finishedAtMs, :evidenceDigest, :createdAtMs)`,
|
||||
{ replacements: receipt, transaction },
|
||||
);
|
||||
}
|
||||
|
||||
private nowAtOrAfter(minimum: number): number {
|
||||
const nowMs = this.binding.clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < minimum) {
|
||||
throw new RangeError('clock must not precede the action start barrier');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
|
||||
export interface LegacySequelizeApprovedRunActionRepositoryOptions {
|
||||
clock?: () => number;
|
||||
createId?: PrimaryRunIdFactory;
|
||||
}
|
||||
|
||||
export class LegacySequelizeApprovedRunActionRepository
|
||||
implements ApprovedRunActionRepository
|
||||
{
|
||||
private readonly runs: LegacySequelizeRunRepository;
|
||||
private readonly clock: () => number;
|
||||
private readonly createId?: PrimaryRunIdFactory;
|
||||
|
||||
constructor(
|
||||
private readonly database: Sequelize,
|
||||
options: LegacySequelizeApprovedRunActionRepositoryOptions = {},
|
||||
) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Approved Run action repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.runs = new LegacySequelizeRunRepository(database);
|
||||
this.clock = options.clock ?? Date.now;
|
||||
this.createId = options.createId;
|
||||
}
|
||||
|
||||
async create(
|
||||
command: Readonly<CreateApprovedRunCommand>,
|
||||
): Promise<Readonly<ApprovedRunReference>> {
|
||||
try {
|
||||
const snapshot = normalizeSnapshot(command.snapshot);
|
||||
const plan = normalizeApprovedRunCreationPlan(command.plan);
|
||||
this.assertPlanBinding(snapshot, plan);
|
||||
const replay = await this.findReplay(snapshot);
|
||||
if (replay) return replay;
|
||||
|
||||
const atomic = new AtomicApprovedRunRepository(this.database, {
|
||||
snapshot,
|
||||
clock: this.clock,
|
||||
});
|
||||
const creator = new PrimaryRunCreator(atomic, this.createId);
|
||||
try {
|
||||
return await creator.create(
|
||||
{
|
||||
projectId: plan.projectId,
|
||||
taskId: plan.taskId,
|
||||
taskRevision: plan.taskRevision,
|
||||
...(plan.taskName === undefined ? {} : { taskName: plan.taskName }),
|
||||
...(plan.taskSnapshotRef === undefined
|
||||
? {}
|
||||
: { taskSnapshotRef: plan.taskSnapshotRef }),
|
||||
triggerType: 'approved_action',
|
||||
executionOrigin: 'system',
|
||||
triggeredBy: `approved-action:${snapshot.dispatch.id}`,
|
||||
requestId: snapshot.dispatch.approvalRequestId,
|
||||
priority: plan.priority,
|
||||
idempotencyKey: snapshot.dispatch.id,
|
||||
...(plan.inputRef === undefined ? {} : { inputRef: plan.inputRef }),
|
||||
acceptedAtMs: snapshot.execution.startedAtMs!,
|
||||
actor: { type: 'system', id: 'approved-action-dispatcher' },
|
||||
},
|
||||
plan.executorType,
|
||||
);
|
||||
} catch (error) {
|
||||
if (!(error instanceof DuplicateIdempotencyKeyError)) throw error;
|
||||
const raced = await this.findReplay(snapshot);
|
||||
if (raced) return raced;
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof ApprovedRunActionBindingConflictError ||
|
||||
error instanceof InvalidApprovedRunActionError ||
|
||||
error instanceof RangeError ||
|
||||
error instanceof TypeError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new ApprovedRunActionRepositoryError();
|
||||
}
|
||||
}
|
||||
|
||||
private assertPlanBinding(
|
||||
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
|
||||
plan: Readonly<ReturnType<typeof normalizeApprovedRunCreationPlan>>,
|
||||
): void {
|
||||
if (
|
||||
snapshot.dispatch.action.actionType !== APPROVED_RUN_ACTION_TYPE ||
|
||||
snapshot.dispatch.action.actionRef !== plan.actionRef ||
|
||||
snapshot.dispatch.projectId !== plan.projectId ||
|
||||
snapshot.dispatch.action.actionDigest !==
|
||||
digestApprovedRunCreationPlan(plan)
|
||||
) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
}
|
||||
|
||||
private async findReplay(
|
||||
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
|
||||
): Promise<Readonly<ApprovedRunReference> | null> {
|
||||
const rows = await this.database.query<ApprovedRunReceiptRow>(
|
||||
`SELECT schema_version, dispatch_id, approval_request_id, project_id,
|
||||
action_type, action_digest, execution_attempt, execution_version,
|
||||
started_at_ms, idempotency_key, outcome, result_code,
|
||||
resource_type, resource_id, finished_at_ms, evidence_digest,
|
||||
created_at_ms
|
||||
FROM "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
WHERE dispatch_id = :dispatchId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { dispatchId: snapshot.dispatch.id },
|
||||
},
|
||||
);
|
||||
if (rows.length > 1) throw new ApprovedRunActionBindingConflictError();
|
||||
if (rows.length === 0) {
|
||||
const collisions = await this.database.query<{ id: string }>(
|
||||
`SELECT id FROM "Runs"
|
||||
WHERE project_id = :projectId AND idempotency_key = :idempotencyKey
|
||||
LIMIT 1`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
projectId: snapshot.dispatch.projectId,
|
||||
idempotencyKey: snapshot.dispatch.id,
|
||||
},
|
||||
},
|
||||
);
|
||||
if (collisions.length > 0) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const receipt = rowToReceipt(rows[0]);
|
||||
if (!receiptMatches(receipt, snapshot)) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
const run = await this.runs.findRunById(receipt.resourceId);
|
||||
const attempt = await this.runs.findLatestAttemptByRunId(
|
||||
receipt.resourceId,
|
||||
);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
run.projectId !== receipt.projectId ||
|
||||
run.idempotencyKey !== receipt.idempotencyKey ||
|
||||
run.requestId !== receipt.approvalRequestId ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
run.executionOrigin !== 'system' ||
|
||||
run.triggerType !== 'approved_action'
|
||||
) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
return Object.freeze({ run, attempt });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import { APPROVED_RUN_ACTION_RECEIPT_TABLE } from '../../../migrations/0023-approved-run-action-receipts';
|
||||
import {
|
||||
APPROVED_RUN_ACTION_TYPE,
|
||||
APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
InvalidApprovedRunActionError,
|
||||
normalizeApprovedRunCreationReceipt,
|
||||
type ApprovedRunCreationReceipt,
|
||||
} from '../../domain/approvedRunAction';
|
||||
import type {
|
||||
ApprovedActionRecoveryEvidence,
|
||||
ApprovedActionRecoveryEvidenceContext,
|
||||
ApprovedActionRecoveryEvidenceProvider,
|
||||
} from '../../ports/approvedActionRecoveryEvidenceProvider';
|
||||
import { LegacySequelizeRunRepository } from './runRepository';
|
||||
|
||||
interface ReceiptRow {
|
||||
schema_version: number;
|
||||
dispatch_id: string;
|
||||
approval_request_id: string;
|
||||
project_id: string;
|
||||
action_type: string;
|
||||
action_digest: string;
|
||||
execution_attempt: number;
|
||||
execution_version: number;
|
||||
started_at_ms: number;
|
||||
idempotency_key: string;
|
||||
outcome: string;
|
||||
result_code: string;
|
||||
resource_type: string;
|
||||
resource_id: string;
|
||||
finished_at_ms: number;
|
||||
evidence_digest: string;
|
||||
created_at_ms: number;
|
||||
}
|
||||
|
||||
function normalizeRow(row: ReceiptRow): Readonly<ApprovedRunCreationReceipt> {
|
||||
return normalizeApprovedRunCreationReceipt({
|
||||
schemaVersion: row.schema_version as 1,
|
||||
dispatchId: row.dispatch_id,
|
||||
approvalRequestId: row.approval_request_id,
|
||||
projectId: row.project_id,
|
||||
actionType: row.action_type as typeof APPROVED_RUN_ACTION_TYPE,
|
||||
actionDigest: row.action_digest,
|
||||
executionAttempt: row.execution_attempt,
|
||||
executionVersion: row.execution_version,
|
||||
startedAtMs: row.started_at_ms,
|
||||
idempotencyKey: row.idempotency_key,
|
||||
outcome: row.outcome as 'succeeded',
|
||||
resultCode: row.result_code as typeof APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
resourceType: row.resource_type as 'run',
|
||||
resourceId: row.resource_id,
|
||||
finishedAtMs: row.finished_at_ms,
|
||||
evidenceDigest: row.evidence_digest,
|
||||
createdAtMs: row.created_at_ms,
|
||||
});
|
||||
}
|
||||
|
||||
const CONFLICT: ApprovedActionRecoveryEvidence = Object.freeze({
|
||||
finding: 'conflict',
|
||||
resultCode: 'approved_run_receipt_conflict',
|
||||
});
|
||||
|
||||
export class LegacySequelizeApprovedRunRecoveryEvidenceProvider
|
||||
implements ApprovedActionRecoveryEvidenceProvider
|
||||
{
|
||||
readonly actionType = APPROVED_RUN_ACTION_TYPE;
|
||||
readonly capability = 'automatic' as const;
|
||||
private readonly runs: LegacySequelizeRunRepository;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Approved Run recovery provider is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.runs = new LegacySequelizeRunRepository(database);
|
||||
}
|
||||
|
||||
async inspect(
|
||||
context: Readonly<ApprovedActionRecoveryEvidenceContext>,
|
||||
): Promise<ApprovedActionRecoveryEvidence> {
|
||||
const snapshot = context.snapshot;
|
||||
const dispatch = snapshot.action.dispatch;
|
||||
const execution = snapshot.action.execution;
|
||||
if (
|
||||
dispatch.action.actionType !== this.actionType ||
|
||||
context.idempotencyKey !== dispatch.id ||
|
||||
execution.dispatchId !== dispatch.id ||
|
||||
execution.projectId !== dispatch.projectId ||
|
||||
execution.startedAtMs === null
|
||||
) {
|
||||
return CONFLICT;
|
||||
}
|
||||
const rows = await this.database.query<ReceiptRow>(
|
||||
`SELECT schema_version, dispatch_id, approval_request_id, project_id,
|
||||
action_type, action_digest, execution_attempt, execution_version,
|
||||
started_at_ms, idempotency_key, outcome, result_code,
|
||||
resource_type, resource_id, finished_at_ms, evidence_digest,
|
||||
created_at_ms
|
||||
FROM "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
WHERE dispatch_id = :dispatchId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { dispatchId: dispatch.id },
|
||||
},
|
||||
);
|
||||
if (rows.length > 1) return CONFLICT;
|
||||
if (rows.length === 0) {
|
||||
const collisions = await this.database.query<{ id: string }>(
|
||||
`SELECT id FROM "Runs"
|
||||
WHERE project_id = :projectId AND idempotency_key = :idempotencyKey
|
||||
LIMIT 1`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
projectId: dispatch.projectId,
|
||||
idempotencyKey: dispatch.id,
|
||||
},
|
||||
},
|
||||
);
|
||||
return collisions.length === 0
|
||||
? {
|
||||
finding: 'missing',
|
||||
resultCode: 'approved_run_receipt_missing',
|
||||
}
|
||||
: CONFLICT;
|
||||
}
|
||||
|
||||
let receipt: Readonly<ApprovedRunCreationReceipt>;
|
||||
try {
|
||||
receipt = normalizeRow(rows[0]);
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidApprovedRunActionError) return CONFLICT;
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
receipt.dispatchId !== dispatch.id ||
|
||||
receipt.approvalRequestId !== dispatch.approvalRequestId ||
|
||||
receipt.projectId !== dispatch.projectId ||
|
||||
receipt.actionType !== dispatch.action.actionType ||
|
||||
receipt.actionDigest !== dispatch.action.actionDigest ||
|
||||
receipt.executionAttempt !== execution.attemptCount ||
|
||||
receipt.executionVersion > execution.version ||
|
||||
receipt.startedAtMs !== execution.startedAtMs ||
|
||||
receipt.idempotencyKey !== context.idempotencyKey
|
||||
) {
|
||||
return CONFLICT;
|
||||
}
|
||||
const run = await this.runs.findRunById(receipt.resourceId);
|
||||
const attempt = await this.runs.findLatestAttemptByRunId(
|
||||
receipt.resourceId,
|
||||
);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
run.projectId !== receipt.projectId ||
|
||||
run.idempotencyKey !== receipt.idempotencyKey ||
|
||||
run.requestId !== receipt.approvalRequestId ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
run.executionOrigin !== 'system' ||
|
||||
run.triggerType !== 'approved_action'
|
||||
) {
|
||||
return CONFLICT;
|
||||
}
|
||||
return {
|
||||
finding: 'verified_succeeded',
|
||||
resultCode: 'approved_run_receipt_verified',
|
||||
evidenceDigest: receipt.evidenceDigest,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,735 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
RUN_EVENT_TABLE,
|
||||
RUN_TABLE,
|
||||
RUN_ATTEMPT_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { RUN_CANCELLATION_DISPATCH_TABLE } from '../../../migrations/0005-run-cancellation-dispatch';
|
||||
import {
|
||||
CANCELLATION_DISPATCH_RESULTS,
|
||||
CANCELLATION_DISPATCH_STATUSES,
|
||||
type CancellationDispatchRecord,
|
||||
type CancellationDispatchResult,
|
||||
type CancellationDispatchStatus,
|
||||
} from '../../domain/cancellationDispatch';
|
||||
import {
|
||||
CancellationDispatchBindingConflictError,
|
||||
CancellationDispatchFenceRejectedError,
|
||||
CancellationDispatchRepositoryError,
|
||||
InvalidCancellationDispatchCommandError,
|
||||
} from '../../domain/cancellationDispatchErrors';
|
||||
import type { RunEventRecord, RunStatus } from '../../domain/run';
|
||||
import type {
|
||||
CancellationDispatchRepository,
|
||||
ClaimCancellationDispatchCommand,
|
||||
ClaimCancellationDispatchResult,
|
||||
RecordCancellationDispatchResult,
|
||||
RecordCancellationDispatchResultCommand,
|
||||
} from '../../ports/cancellationDispatchRepository';
|
||||
|
||||
const ACTIVE_RUN_STATUSES: readonly RunStatus[] = [
|
||||
'created',
|
||||
'queued',
|
||||
'dispatching',
|
||||
'running',
|
||||
'waiting_approval',
|
||||
'retry_wait',
|
||||
'lost',
|
||||
];
|
||||
const ACTIVE_ATTEMPT_STATUSES = ['claimed', 'starting', 'running'] as const;
|
||||
const RETRYABLE_RESULTS: readonly CancellationDispatchResult[] = [
|
||||
'controller_missing',
|
||||
'handle_missing',
|
||||
'dispatch_error',
|
||||
];
|
||||
const BLOCKING_RESULTS: readonly CancellationDispatchResult[] = [
|
||||
'identity_mismatch',
|
||||
'pid_mismatch',
|
||||
'unsupported',
|
||||
'invalid',
|
||||
];
|
||||
|
||||
interface CancellationDispatchRow {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
status: string;
|
||||
version: number;
|
||||
dispatchCount: number;
|
||||
nextAttemptAtMs: number | null;
|
||||
leaseOwner: string | null;
|
||||
leaseToken: string | null;
|
||||
leaseExpiresAtMs: number | null;
|
||||
lastResult: string | null;
|
||||
lastDispatchedAtMs: number | null;
|
||||
createdAtMs: number;
|
||||
updatedAtMs: number;
|
||||
}
|
||||
|
||||
interface CancellationDispatchRunRow {
|
||||
id: string;
|
||||
executionOwner: string;
|
||||
status: string;
|
||||
version: number;
|
||||
eventSequence: number;
|
||||
cancelRequestedAtMs: number | null;
|
||||
}
|
||||
|
||||
interface CancellationDispatchAttemptRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
status: string;
|
||||
}
|
||||
|
||||
interface CancellationDispatchEventRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
sequence: number;
|
||||
type: string;
|
||||
dedupeKey: string;
|
||||
actorType: string;
|
||||
actorId: string;
|
||||
attemptId: string;
|
||||
payload: Readonly<Record<string, unknown>>;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
interface CancellationDispatchInstance
|
||||
extends Model<CancellationDispatchRow, CancellationDispatchRow>,
|
||||
CancellationDispatchRow {}
|
||||
interface CancellationDispatchRunInstance
|
||||
extends Model<CancellationDispatchRunRow, CancellationDispatchRunRow>,
|
||||
CancellationDispatchRunRow {}
|
||||
interface CancellationDispatchAttemptInstance
|
||||
extends Model<CancellationDispatchAttemptRow, CancellationDispatchAttemptRow>,
|
||||
CancellationDispatchAttemptRow {}
|
||||
interface CancellationDispatchEventInstance
|
||||
extends Model<CancellationDispatchEventRow, CancellationDispatchEventRow>,
|
||||
CancellationDispatchEventRow {}
|
||||
|
||||
function defineDispatchModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationDispatchInstance> {
|
||||
return database.define<CancellationDispatchInstance>(
|
||||
'Ql3CancellationDispatch',
|
||||
{
|
||||
runId: { field: 'run_id', type: DataTypes.STRING(36), primaryKey: true },
|
||||
attemptId: {
|
||||
field: 'attempt_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
version: { type: DataTypes.INTEGER, allowNull: false },
|
||||
dispatchCount: {
|
||||
field: 'dispatch_count',
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
},
|
||||
nextAttemptAtMs: {
|
||||
field: 'next_attempt_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
leaseOwner: {
|
||||
field: 'lease_owner',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: true,
|
||||
},
|
||||
leaseToken: {
|
||||
field: 'lease_token',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: true,
|
||||
},
|
||||
leaseExpiresAtMs: {
|
||||
field: 'lease_expires_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
lastResult: {
|
||||
field: 'last_result',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: true,
|
||||
},
|
||||
lastDispatchedAtMs: {
|
||||
field: 'last_dispatched_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
updatedAtMs: {
|
||||
field: 'updated_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: RUN_CANCELLATION_DISPATCH_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function defineRunModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationDispatchRunInstance> {
|
||||
return database.define<CancellationDispatchRunInstance>(
|
||||
'Ql3CancellationDispatchRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
executionOwner: {
|
||||
field: 'execution_owner',
|
||||
type: DataTypes.STRING(16),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
version: { type: DataTypes.INTEGER, allowNull: false },
|
||||
eventSequence: {
|
||||
field: 'event_sequence',
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
},
|
||||
cancelRequestedAtMs: {
|
||||
field: 'cancel_requested_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function defineAttemptModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationDispatchAttemptInstance> {
|
||||
return database.define<CancellationDispatchAttemptInstance>(
|
||||
'Ql3CancellationDispatchAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
runId: { field: 'run_id', type: DataTypes.STRING(36), allowNull: false },
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
},
|
||||
{ tableName: RUN_ATTEMPT_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function defineEventModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationDispatchEventInstance> {
|
||||
return database.define<CancellationDispatchEventInstance>(
|
||||
'Ql3CancellationDispatchEvent',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
runId: { field: 'run_id', type: DataTypes.STRING(36), allowNull: false },
|
||||
sequence: { type: DataTypes.INTEGER, allowNull: false },
|
||||
type: { type: DataTypes.STRING(128), allowNull: false },
|
||||
dedupeKey: {
|
||||
field: 'dedupe_key',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
actorType: {
|
||||
field: 'actor_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
actorId: {
|
||||
field: 'actor_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
attemptId: {
|
||||
field: 'attempt_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
payload: { type: DataTypes.JSON, allowNull: false },
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_EVENT_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function assertId(name: string, value: string, maxLength = 36): void {
|
||||
if (!value || value.length > maxLength) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
`${name} must be between 1 and ${maxLength} characters`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertTimestamp(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
`${name} must be a non-negative safe integer`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertClaim(command: ClaimCancellationDispatchCommand): void {
|
||||
assertId('runId', command.runId);
|
||||
assertId('attemptId', command.attemptId);
|
||||
assertId('owner', command.owner, 128);
|
||||
assertId('leaseToken', command.leaseToken, 128);
|
||||
assertTimestamp('requestedAtMs', command.requestedAtMs);
|
||||
assertTimestamp('nowMs', command.nowMs);
|
||||
if (
|
||||
!Number.isSafeInteger(command.leaseDurationMs) ||
|
||||
command.leaseDurationMs < 1
|
||||
) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'leaseDurationMs must be a positive safe integer',
|
||||
);
|
||||
}
|
||||
if (!Number.isSafeInteger(command.nowMs + command.leaseDurationMs)) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'lease expiry exceeds the supported range',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertRecordResult(
|
||||
command: RecordCancellationDispatchResultCommand,
|
||||
): void {
|
||||
assertId('runId', command.runId);
|
||||
assertId('attemptId', command.attemptId);
|
||||
assertId('owner', command.owner, 128);
|
||||
assertId('leaseToken', command.leaseToken, 128);
|
||||
assertId('eventId', command.eventId);
|
||||
assertTimestamp('atMs', command.atMs);
|
||||
if (
|
||||
!Number.isSafeInteger(command.expectedVersion) ||
|
||||
command.expectedVersion < 1
|
||||
) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'expectedVersion must be a positive safe integer',
|
||||
);
|
||||
}
|
||||
if (!CANCELLATION_DISPATCH_RESULTS.includes(command.result)) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'result is not supported',
|
||||
);
|
||||
}
|
||||
if (RETRYABLE_RESULTS.includes(command.result)) {
|
||||
if (
|
||||
command.nextAttemptAtMs === undefined ||
|
||||
!Number.isSafeInteger(command.nextAttemptAtMs) ||
|
||||
command.nextAttemptAtMs <= command.atMs
|
||||
) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'retryable results require nextAttemptAtMs greater than atMs',
|
||||
);
|
||||
}
|
||||
} else if (command.nextAttemptAtMs !== undefined) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'terminal results must not include nextAttemptAtMs',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function rowToDispatch(
|
||||
row: CancellationDispatchRow,
|
||||
): CancellationDispatchRecord {
|
||||
if (
|
||||
!CANCELLATION_DISPATCH_STATUSES.includes(
|
||||
row.status as CancellationDispatchStatus,
|
||||
)
|
||||
) {
|
||||
throw new CancellationDispatchRepositoryError(
|
||||
new Error(`Unsupported cancellation dispatch status: ${row.status}`),
|
||||
);
|
||||
}
|
||||
if (
|
||||
row.lastResult !== null &&
|
||||
!CANCELLATION_DISPATCH_RESULTS.includes(
|
||||
row.lastResult as CancellationDispatchResult,
|
||||
)
|
||||
) {
|
||||
throw new CancellationDispatchRepositoryError(
|
||||
new Error(`Unsupported cancellation dispatch result: ${row.lastResult}`),
|
||||
);
|
||||
}
|
||||
for (const [name, value] of [
|
||||
['version', row.version],
|
||||
['dispatchCount', row.dispatchCount],
|
||||
['createdAtMs', row.createdAtMs],
|
||||
['updatedAtMs', row.updatedAtMs],
|
||||
['nextAttemptAtMs', row.nextAttemptAtMs],
|
||||
['leaseExpiresAtMs', row.leaseExpiresAtMs],
|
||||
['lastDispatchedAtMs', row.lastDispatchedAtMs],
|
||||
] as const) {
|
||||
if (
|
||||
value !== null &&
|
||||
(!Number.isSafeInteger(Number(value)) || Number(value) < 0)
|
||||
) {
|
||||
throw new CancellationDispatchRepositoryError(
|
||||
new Error(`Invalid cancellation dispatch ${name}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
const status = row.status as CancellationDispatchStatus;
|
||||
const hasCompleteLease =
|
||||
row.leaseOwner !== null &&
|
||||
row.leaseToken !== null &&
|
||||
row.leaseExpiresAtMs !== null;
|
||||
const hasAnyLease =
|
||||
row.leaseOwner !== null ||
|
||||
row.leaseToken !== null ||
|
||||
row.leaseExpiresAtMs !== null;
|
||||
if (
|
||||
(status === 'leased' && !hasCompleteLease) ||
|
||||
(status !== 'leased' && hasAnyLease) ||
|
||||
((status === 'pending' || status === 'retry_wait') &&
|
||||
row.nextAttemptAtMs === null) ||
|
||||
((status === 'dispatched' || status === 'blocked' || status === 'leased') &&
|
||||
row.nextAttemptAtMs !== null) ||
|
||||
((status === 'dispatched' || status === 'blocked') &&
|
||||
row.lastResult === null)
|
||||
) {
|
||||
throw new CancellationDispatchRepositoryError(
|
||||
new Error('Cancellation dispatch lease/status fields are inconsistent'),
|
||||
);
|
||||
}
|
||||
return {
|
||||
runId: row.runId,
|
||||
attemptId: row.attemptId,
|
||||
status,
|
||||
version: Number(row.version),
|
||||
dispatchCount: Number(row.dispatchCount),
|
||||
createdAtMs: Number(row.createdAtMs),
|
||||
updatedAtMs: Number(row.updatedAtMs),
|
||||
...(row.nextAttemptAtMs === null
|
||||
? {}
|
||||
: { nextAttemptAtMs: Number(row.nextAttemptAtMs) }),
|
||||
...(row.leaseOwner === null ? {} : { leaseOwner: row.leaseOwner }),
|
||||
...(row.leaseToken === null ? {} : { leaseToken: row.leaseToken }),
|
||||
...(row.leaseExpiresAtMs === null
|
||||
? {}
|
||||
: { leaseExpiresAtMs: Number(row.leaseExpiresAtMs) }),
|
||||
...(row.lastResult === null
|
||||
? {}
|
||||
: { lastResult: row.lastResult as CancellationDispatchResult }),
|
||||
...(row.lastDispatchedAtMs === null
|
||||
? {}
|
||||
: { lastDispatchedAtMs: Number(row.lastDispatchedAtMs) }),
|
||||
};
|
||||
}
|
||||
|
||||
function resultState(result: CancellationDispatchResult): {
|
||||
status: CancellationDispatchStatus;
|
||||
eventType: string;
|
||||
} {
|
||||
if (RETRYABLE_RESULTS.includes(result)) {
|
||||
return { status: 'retry_wait', eventType: 'run.cancel_dispatch_failed' };
|
||||
}
|
||||
if (BLOCKING_RESULTS.includes(result)) {
|
||||
return { status: 'blocked', eventType: 'run.cancel_dispatch_blocked' };
|
||||
}
|
||||
return { status: 'dispatched', eventType: 'run.cancel_dispatched' };
|
||||
}
|
||||
|
||||
function withoutScheduleAndLease(
|
||||
dispatch: CancellationDispatchRecord,
|
||||
): Omit<
|
||||
CancellationDispatchRecord,
|
||||
'nextAttemptAtMs' | 'leaseOwner' | 'leaseToken' | 'leaseExpiresAtMs'
|
||||
> {
|
||||
const {
|
||||
nextAttemptAtMs: _nextAttemptAtMs,
|
||||
leaseOwner: _leaseOwner,
|
||||
leaseToken: _leaseToken,
|
||||
leaseExpiresAtMs: _leaseExpiresAtMs,
|
||||
...rest
|
||||
} = dispatch;
|
||||
return rest;
|
||||
}
|
||||
|
||||
export class LegacySequelizeCancellationDispatchRepository
|
||||
implements CancellationDispatchRepository
|
||||
{
|
||||
private readonly dispatch: ModelStatic<CancellationDispatchInstance>;
|
||||
private readonly run: ModelStatic<CancellationDispatchRunInstance>;
|
||||
private readonly attempt: ModelStatic<CancellationDispatchAttemptInstance>;
|
||||
private readonly event: ModelStatic<CancellationDispatchEventInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
this.dispatch = defineDispatchModel(database);
|
||||
this.run = defineRunModel(database);
|
||||
this.attempt = defineAttemptModel(database);
|
||||
this.event = defineEventModel(database);
|
||||
}
|
||||
|
||||
async findByRunId(runId: string): Promise<CancellationDispatchRecord | null> {
|
||||
assertId('runId', runId);
|
||||
const row = (await this.dispatch.findByPk(runId, {
|
||||
raw: true,
|
||||
})) as unknown as CancellationDispatchRow | null;
|
||||
return row === null ? null : rowToDispatch(row);
|
||||
}
|
||||
|
||||
async claim(
|
||||
command: ClaimCancellationDispatchCommand,
|
||||
): Promise<ClaimCancellationDispatchResult> {
|
||||
assertClaim(command);
|
||||
return this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const [run, attempt] = await Promise.all([
|
||||
this.run.findByPk(command.runId, { raw: true, transaction }),
|
||||
this.attempt.findByPk(command.attemptId, { raw: true, transaction }),
|
||||
]);
|
||||
const runRow = run as unknown as CancellationDispatchRunRow | null;
|
||||
const attemptRow =
|
||||
attempt as unknown as CancellationDispatchAttemptRow | null;
|
||||
if (
|
||||
runRow === null ||
|
||||
attemptRow === null ||
|
||||
runRow.executionOwner !== 'runtime' ||
|
||||
!ACTIVE_RUN_STATUSES.includes(runRow.status as RunStatus) ||
|
||||
runRow.cancelRequestedAtMs === null ||
|
||||
Number(runRow.cancelRequestedAtMs) !== command.requestedAtMs ||
|
||||
attemptRow.runId !== command.runId ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.includes(
|
||||
attemptRow.status as (typeof ACTIVE_ATTEMPT_STATUSES)[number],
|
||||
)
|
||||
) {
|
||||
return { status: 'not_eligible' as const };
|
||||
}
|
||||
|
||||
let row = (await this.dispatch.findByPk(command.runId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
})) as unknown as CancellationDispatchRow | null;
|
||||
if (row === null) {
|
||||
try {
|
||||
const created = await this.dispatch.create(
|
||||
{
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
status: 'pending',
|
||||
version: 0,
|
||||
dispatchCount: 0,
|
||||
nextAttemptAtMs: command.requestedAtMs,
|
||||
leaseOwner: null,
|
||||
leaseToken: null,
|
||||
leaseExpiresAtMs: null,
|
||||
lastResult: null,
|
||||
lastDispatchedAtMs: null,
|
||||
createdAtMs: command.nowMs,
|
||||
updatedAtMs: command.nowMs,
|
||||
},
|
||||
{ transaction },
|
||||
);
|
||||
row = created.get({ plain: true }) as CancellationDispatchRow;
|
||||
} catch (error) {
|
||||
if (error instanceof UniqueConstraintError) {
|
||||
throw new CancellationDispatchRepositoryError(error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (row.attemptId !== command.attemptId) {
|
||||
throw new CancellationDispatchBindingConflictError(
|
||||
command.runId,
|
||||
command.attemptId,
|
||||
);
|
||||
}
|
||||
const dispatch = rowToDispatch(row);
|
||||
if (dispatch.status === 'dispatched' || dispatch.status === 'blocked') {
|
||||
return { status: dispatch.status, dispatch };
|
||||
}
|
||||
if (
|
||||
dispatch.status === 'leased' &&
|
||||
dispatch.leaseExpiresAtMs !== undefined &&
|
||||
dispatch.leaseExpiresAtMs > command.nowMs
|
||||
) {
|
||||
return { status: 'leased', dispatch };
|
||||
}
|
||||
if (
|
||||
dispatch.status !== 'leased' &&
|
||||
dispatch.nextAttemptAtMs !== undefined &&
|
||||
dispatch.nextAttemptAtMs > command.nowMs
|
||||
) {
|
||||
return { status: 'not_due', dispatch };
|
||||
}
|
||||
|
||||
const nextVersion = dispatch.version + 1;
|
||||
const nextCount = dispatch.dispatchCount + 1;
|
||||
const leaseExpiresAtMs = command.nowMs + command.leaseDurationMs;
|
||||
const [affected] = await this.dispatch.update(
|
||||
{
|
||||
status: 'leased',
|
||||
version: nextVersion,
|
||||
dispatchCount: nextCount,
|
||||
nextAttemptAtMs: null,
|
||||
leaseOwner: command.owner,
|
||||
leaseToken: command.leaseToken,
|
||||
leaseExpiresAtMs,
|
||||
updatedAtMs: command.nowMs,
|
||||
},
|
||||
{
|
||||
where: { runId: command.runId, version: dispatch.version },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (affected !== 1) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
return {
|
||||
status: 'claimed',
|
||||
dispatch: {
|
||||
...withoutScheduleAndLease(dispatch),
|
||||
status: 'leased',
|
||||
version: nextVersion,
|
||||
dispatchCount: nextCount,
|
||||
leaseOwner: command.owner,
|
||||
leaseToken: command.leaseToken,
|
||||
leaseExpiresAtMs,
|
||||
updatedAtMs: command.nowMs,
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async recordResult(
|
||||
command: RecordCancellationDispatchResultCommand,
|
||||
): Promise<RecordCancellationDispatchResult> {
|
||||
assertRecordResult(command);
|
||||
return this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const row = (await this.dispatch.findByPk(command.runId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
})) as unknown as CancellationDispatchRow | null;
|
||||
if (
|
||||
row === null ||
|
||||
row.attemptId !== command.attemptId ||
|
||||
row.status !== 'leased' ||
|
||||
row.version !== command.expectedVersion ||
|
||||
row.leaseOwner !== command.owner ||
|
||||
row.leaseToken !== command.leaseToken
|
||||
) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
const run = (await this.run.findByPk(command.runId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
})) as unknown as CancellationDispatchRunRow | null;
|
||||
if (run === null) {
|
||||
throw new CancellationDispatchRepositoryError(
|
||||
new Error('Run disappeared while recording cancellation result'),
|
||||
);
|
||||
}
|
||||
const state = resultState(command.result);
|
||||
const controllerInvoked = ![
|
||||
'controller_missing',
|
||||
'handle_missing',
|
||||
].includes(command.result);
|
||||
const nextVersion = row.version + 1;
|
||||
const nextSequence = Number(run.eventSequence) + 1;
|
||||
const [runAffected] = await this.run.update(
|
||||
{ version: Number(run.version) + 1, eventSequence: nextSequence },
|
||||
{ where: { id: command.runId, version: run.version }, transaction },
|
||||
);
|
||||
if (runAffected !== 1) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
const [dispatchAffected] = await this.dispatch.update(
|
||||
{
|
||||
status: state.status,
|
||||
version: nextVersion,
|
||||
nextAttemptAtMs: command.nextAttemptAtMs ?? null,
|
||||
leaseOwner: null,
|
||||
leaseToken: null,
|
||||
leaseExpiresAtMs: null,
|
||||
lastResult: command.result,
|
||||
lastDispatchedAtMs: controllerInvoked
|
||||
? command.atMs
|
||||
: row.lastDispatchedAtMs,
|
||||
updatedAtMs: command.atMs,
|
||||
},
|
||||
{
|
||||
where: {
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
status: 'leased',
|
||||
version: command.expectedVersion,
|
||||
leaseOwner: command.owner,
|
||||
leaseToken: command.leaseToken,
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (dispatchAffected !== 1) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
const event: RunEventRecord = {
|
||||
id: command.eventId,
|
||||
runId: command.runId,
|
||||
sequence: nextSequence,
|
||||
type: state.eventType,
|
||||
dedupeKey: `cancel-dispatch:${command.attemptId}:${row.dispatchCount}`,
|
||||
actorType: 'worker',
|
||||
actorId: command.owner,
|
||||
attemptId: command.attemptId,
|
||||
payload: {
|
||||
attempt_id: command.attemptId,
|
||||
dispatch_count: row.dispatchCount,
|
||||
result: command.result,
|
||||
},
|
||||
createdAtMs: command.atMs,
|
||||
};
|
||||
await this.event.create(
|
||||
{
|
||||
id: event.id,
|
||||
runId: event.runId,
|
||||
sequence: event.sequence,
|
||||
type: event.type,
|
||||
dedupeKey: event.dedupeKey!,
|
||||
actorType: event.actorType,
|
||||
actorId: event.actorId!,
|
||||
attemptId: event.attemptId!,
|
||||
payload: event.payload,
|
||||
createdAtMs: event.createdAtMs,
|
||||
},
|
||||
{ transaction },
|
||||
);
|
||||
return {
|
||||
dispatch: {
|
||||
...withoutScheduleAndLease(rowToDispatch(row)),
|
||||
status: state.status,
|
||||
version: nextVersion,
|
||||
...(command.nextAttemptAtMs === undefined
|
||||
? {}
|
||||
: { nextAttemptAtMs: command.nextAttemptAtMs }),
|
||||
lastResult: command.result,
|
||||
...(controllerInvoked
|
||||
? { lastDispatchedAtMs: command.atMs }
|
||||
: row.lastDispatchedAtMs === null
|
||||
? {}
|
||||
: { lastDispatchedAtMs: Number(row.lastDispatchedAtMs) }),
|
||||
updatedAtMs: command.atMs,
|
||||
},
|
||||
event,
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Op,
|
||||
Sequelize,
|
||||
UniqueConstraintError,
|
||||
type WhereOptions,
|
||||
} from 'sequelize';
|
||||
import { RUN_ATTEMPT_TABLE } from '../../../migrations/0002-run-schema';
|
||||
import { COMPLETION_RECEIPT_JOURNAL_TABLE } from '../../../migrations/0007-completion-receipt-journal';
|
||||
import {
|
||||
COMPLETION_RECEIPT_JOURNAL_STATES,
|
||||
type CompletionReceiptJournalCandidate,
|
||||
type CompletionReceiptJournalCursor,
|
||||
type CompletionReceiptJournalRecord,
|
||||
type CompletionReceiptJournalState,
|
||||
} from '../../domain/completionReceiptJournal';
|
||||
import { assertCompletionReceiptId } from '../../domain/completionReceipt';
|
||||
import type { RunAttemptStatus } from '../../domain/run';
|
||||
import {
|
||||
MAX_COMPLETION_RECEIPT_JOURNAL_BATCH_SIZE,
|
||||
type CompletionReceiptJournal,
|
||||
type QuarantineCompletionReceiptCommand,
|
||||
type RegisterCompletionReceiptCommand,
|
||||
} from '../../ports/completionReceiptJournal';
|
||||
|
||||
interface JournalRow {
|
||||
attemptId: string;
|
||||
runId: string;
|
||||
state: string;
|
||||
quarantineRef: string | null;
|
||||
purgeAfterMs: number | null;
|
||||
registeredAtMs: number;
|
||||
updatedAtMs: number;
|
||||
}
|
||||
|
||||
interface AttemptRow {
|
||||
id: string;
|
||||
status: string;
|
||||
executorType: string;
|
||||
finishedAtMs: number | null;
|
||||
}
|
||||
|
||||
interface JournalInstance extends Model<JournalRow, JournalRow>, JournalRow {}
|
||||
interface AttemptInstance extends Model<AttemptRow, AttemptRow>, AttemptRow {}
|
||||
|
||||
function defineJournalModel(database: Sequelize): ModelStatic<JournalInstance> {
|
||||
return database.define<JournalInstance>(
|
||||
'Ql3CompletionReceiptJournal',
|
||||
{
|
||||
attemptId: {
|
||||
field: 'attempt_id',
|
||||
type: DataTypes.STRING(36),
|
||||
primaryKey: true,
|
||||
},
|
||||
runId: {
|
||||
field: 'run_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
state: { type: DataTypes.STRING(16), allowNull: false },
|
||||
quarantineRef: {
|
||||
field: 'quarantine_ref',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: true,
|
||||
},
|
||||
purgeAfterMs: {
|
||||
field: 'purge_after_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
registeredAtMs: {
|
||||
field: 'registered_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
updatedAtMs: {
|
||||
field: 'updated_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: COMPLETION_RECEIPT_JOURNAL_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function defineAttemptModel(database: Sequelize): ModelStatic<AttemptInstance> {
|
||||
return database.define<AttemptInstance>(
|
||||
'Ql3CompletionReceiptJournalAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
executorType: {
|
||||
field: 'executor_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
finishedAtMs: {
|
||||
field: 'finished_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_ATTEMPT_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function assertNonNegativeTimestamp(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new RangeError(`${name} must be a non-negative safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertCursor(cursor: CompletionReceiptJournalCursor): void {
|
||||
assertNonNegativeTimestamp('cursor.updatedAtMs', cursor.updatedAtMs);
|
||||
assertCompletionReceiptId(cursor.attemptId, 'attemptId');
|
||||
}
|
||||
|
||||
function assertQuarantineRef(value: string): void {
|
||||
if (
|
||||
value.length < 1 ||
|
||||
value.length > 255 ||
|
||||
!value.startsWith('.quarantine/') ||
|
||||
value.includes('..') ||
|
||||
value.includes('\\') ||
|
||||
value.includes('\0')
|
||||
) {
|
||||
throw new TypeError('quarantineRef is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function toRecord(row: JournalRow): CompletionReceiptJournalRecord {
|
||||
if (!COMPLETION_RECEIPT_JOURNAL_STATES.includes(row.state as never)) {
|
||||
throw new Error('Completion receipt journal state is corrupt');
|
||||
}
|
||||
return {
|
||||
attemptId: row.attemptId,
|
||||
runId: row.runId,
|
||||
state: row.state as CompletionReceiptJournalState,
|
||||
registeredAtMs: Number(row.registeredAtMs),
|
||||
updatedAtMs: Number(row.updatedAtMs),
|
||||
...(row.quarantineRef === null ? {} : { quarantineRef: row.quarantineRef }),
|
||||
...(row.purgeAfterMs === null
|
||||
? {}
|
||||
: { purgeAfterMs: Number(row.purgeAfterMs) }),
|
||||
};
|
||||
}
|
||||
|
||||
export class LegacySequelizeCompletionReceiptJournal
|
||||
implements CompletionReceiptJournal
|
||||
{
|
||||
private readonly journal: ModelStatic<JournalInstance>;
|
||||
private readonly attempt: ModelStatic<AttemptInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.journal = defineJournalModel(database);
|
||||
this.attempt = defineAttemptModel(database);
|
||||
}
|
||||
|
||||
async register(command: RegisterCompletionReceiptCommand): Promise<void> {
|
||||
assertCompletionReceiptId(command.attemptId, 'attemptId');
|
||||
assertCompletionReceiptId(command.runId, 'runId');
|
||||
assertNonNegativeTimestamp('registeredAtMs', command.registeredAtMs);
|
||||
const values: JournalRow = {
|
||||
attemptId: command.attemptId,
|
||||
runId: command.runId,
|
||||
state: 'pending',
|
||||
quarantineRef: null,
|
||||
purgeAfterMs: null,
|
||||
registeredAtMs: command.registeredAtMs,
|
||||
updatedAtMs: command.registeredAtMs,
|
||||
};
|
||||
try {
|
||||
await this.journal.create(values);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!(error instanceof UniqueConstraintError)) throw error;
|
||||
}
|
||||
const current = (await this.journal.findByPk(command.attemptId, {
|
||||
raw: true,
|
||||
})) as unknown as JournalRow | null;
|
||||
if (
|
||||
!current ||
|
||||
current.runId !== command.runId ||
|
||||
Number(current.registeredAtMs) !== command.registeredAtMs
|
||||
) {
|
||||
throw new Error('Completion receipt journal registration conflicts');
|
||||
}
|
||||
}
|
||||
|
||||
async markQuarantined(
|
||||
command: QuarantineCompletionReceiptCommand,
|
||||
): Promise<void> {
|
||||
assertCompletionReceiptId(command.attemptId, 'attemptId');
|
||||
assertQuarantineRef(command.quarantineRef);
|
||||
assertNonNegativeTimestamp('updatedAtMs', command.updatedAtMs);
|
||||
assertNonNegativeTimestamp('purgeAfterMs', command.purgeAfterMs);
|
||||
if (command.purgeAfterMs < command.updatedAtMs) {
|
||||
throw new RangeError('purgeAfterMs must not precede updatedAtMs');
|
||||
}
|
||||
const [updated] = await this.journal.update(
|
||||
{
|
||||
state: 'quarantined',
|
||||
quarantineRef: command.quarantineRef,
|
||||
purgeAfterMs: command.purgeAfterMs,
|
||||
updatedAtMs: command.updatedAtMs,
|
||||
},
|
||||
{ where: { attemptId: command.attemptId, state: 'pending' } },
|
||||
);
|
||||
if (updated === 1) return;
|
||||
const current = (await this.journal.findByPk(command.attemptId, {
|
||||
raw: true,
|
||||
})) as unknown as JournalRow | null;
|
||||
if (
|
||||
current?.state === 'quarantined' &&
|
||||
current.quarantineRef === command.quarantineRef &&
|
||||
Number(current.purgeAfterMs) === command.purgeAfterMs
|
||||
) {
|
||||
return;
|
||||
}
|
||||
throw new Error('Completion receipt journal quarantine transition failed');
|
||||
}
|
||||
|
||||
async resolve(attemptId: string): Promise<boolean> {
|
||||
assertCompletionReceiptId(attemptId, 'attemptId');
|
||||
return (await this.journal.destroy({ where: { attemptId } })) === 1;
|
||||
}
|
||||
|
||||
async listCandidates({
|
||||
observedAtMs,
|
||||
cursor,
|
||||
limit = 32,
|
||||
}: {
|
||||
observedAtMs: number;
|
||||
cursor?: CompletionReceiptJournalCursor;
|
||||
limit?: number;
|
||||
}) {
|
||||
assertNonNegativeTimestamp('observedAtMs', observedAtMs);
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_COMPLETION_RECEIPT_JOURNAL_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_COMPLETION_RECEIPT_JOURNAL_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (cursor) assertCursor(cursor);
|
||||
|
||||
const eligible: WhereOptions<JournalRow> = {
|
||||
[Op.or]: [
|
||||
{ state: 'pending' },
|
||||
{
|
||||
state: 'quarantined',
|
||||
purgeAfterMs: { [Op.lte]: observedAtMs },
|
||||
},
|
||||
],
|
||||
};
|
||||
const afterCursor: WhereOptions<JournalRow> | undefined = cursor
|
||||
? {
|
||||
[Op.or]: [
|
||||
{ updatedAtMs: { [Op.gt]: cursor.updatedAtMs } },
|
||||
{
|
||||
updatedAtMs: cursor.updatedAtMs,
|
||||
attemptId: { [Op.gt]: cursor.attemptId },
|
||||
},
|
||||
],
|
||||
}
|
||||
: undefined;
|
||||
const where: WhereOptions<JournalRow> = afterCursor
|
||||
? { [Op.and]: [eligible, afterCursor] }
|
||||
: eligible;
|
||||
const rows = (await this.journal.findAll({
|
||||
where,
|
||||
order: [
|
||||
['updatedAtMs', 'ASC'],
|
||||
['attemptId', 'ASC'],
|
||||
],
|
||||
limit: limit + 1,
|
||||
raw: true,
|
||||
})) as unknown as JournalRow[];
|
||||
const truncated = rows.length > limit;
|
||||
const bounded = rows.slice(0, limit);
|
||||
if (bounded.length === 0) return { candidates: [], truncated: false };
|
||||
|
||||
const attempts = (await this.attempt.findAll({
|
||||
where: { id: { [Op.in]: bounded.map((row) => row.attemptId) } },
|
||||
raw: true,
|
||||
})) as unknown as AttemptRow[];
|
||||
const attemptById = new Map(attempts.map((row) => [row.id, row]));
|
||||
const candidates: CompletionReceiptJournalCandidate[] = bounded.map(
|
||||
(row) => {
|
||||
const attempt = attemptById.get(row.attemptId);
|
||||
if (!attempt) {
|
||||
throw new Error('Completion receipt journal Attempt is missing');
|
||||
}
|
||||
return {
|
||||
...toRecord(row),
|
||||
attemptStatus: attempt.status as RunAttemptStatus,
|
||||
executorType: attempt.executorType,
|
||||
...(attempt.finishedAtMs === null
|
||||
? {}
|
||||
: { finishedAtMs: Number(attempt.finishedAtMs) }),
|
||||
};
|
||||
},
|
||||
);
|
||||
const last = bounded[bounded.length - 1];
|
||||
return {
|
||||
candidates,
|
||||
truncated,
|
||||
nextCursor: {
|
||||
updatedAtMs: Number(last.updatedAtMs),
|
||||
attemptId: last.attemptId,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import {
|
||||
IDENTITY_AUTHENTICATION_BINDING_TABLE,
|
||||
IDENTITY_SUBJECT_TABLE,
|
||||
} from '../../../migrations/0019-identity-directory';
|
||||
import {
|
||||
IdentityDirectoryUnavailableError,
|
||||
assertIdentityProvider,
|
||||
assertIdentityProviderSubject,
|
||||
normalizeIdentityAuthenticationBindingRecord,
|
||||
normalizeIdentitySubjectRecord,
|
||||
} from '../../domain/identityDirectory';
|
||||
import type { PolicySubject } from '../../domain/projectPolicy';
|
||||
import type { IdentityDirectoryRepository } from '../../ports/identityDirectoryRepository';
|
||||
|
||||
interface IdentityAuthenticationRow {
|
||||
provider: string;
|
||||
provider_subject: string;
|
||||
binding_version: number;
|
||||
binding_state: string;
|
||||
binding_subject_id: string;
|
||||
binding_created_at_ms: number | string;
|
||||
subject_id: string | null;
|
||||
subject_type: string | null;
|
||||
subject_status: string | null;
|
||||
subject_version: number | null;
|
||||
subject_created_at_ms: number | string | null;
|
||||
subject_updated_at_ms: number | string | null;
|
||||
}
|
||||
|
||||
export class LegacySequelizeIdentityDirectoryRepository
|
||||
implements IdentityDirectoryRepository
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Identity directory repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async resolveAuthenticationSubject(
|
||||
provider: string,
|
||||
providerSubject: string,
|
||||
): Promise<Readonly<PolicySubject> | null> {
|
||||
assertIdentityProvider(provider);
|
||||
assertIdentityProviderSubject(providerSubject);
|
||||
try {
|
||||
const rows = await this.database.query<IdentityAuthenticationRow>(
|
||||
`SELECT binding.provider AS provider,
|
||||
binding.provider_subject AS provider_subject,
|
||||
binding.version AS binding_version,
|
||||
binding.state AS binding_state,
|
||||
binding.subject_id AS binding_subject_id,
|
||||
binding.created_at_ms AS binding_created_at_ms,
|
||||
subject.id AS subject_id,
|
||||
subject.type AS subject_type,
|
||||
subject.status AS subject_status,
|
||||
subject.version AS subject_version,
|
||||
subject.created_at_ms AS subject_created_at_ms,
|
||||
subject.updated_at_ms AS subject_updated_at_ms
|
||||
FROM "${IDENTITY_AUTHENTICATION_BINDING_TABLE}" AS binding
|
||||
LEFT JOIN "${IDENTITY_SUBJECT_TABLE}" AS subject
|
||||
ON subject.id = binding.subject_id
|
||||
WHERE binding.provider = :provider
|
||||
AND binding.provider_subject = :providerSubject
|
||||
AND binding.version = (
|
||||
SELECT MAX(current.version)
|
||||
FROM "${IDENTITY_AUTHENTICATION_BINDING_TABLE}" AS current
|
||||
WHERE current.provider = binding.provider
|
||||
AND current.provider_subject = binding.provider_subject
|
||||
)
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { provider, providerSubject },
|
||||
},
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length !== 1) throw new IdentityDirectoryUnavailableError();
|
||||
const row = rows[0];
|
||||
const binding = normalizeIdentityAuthenticationBindingRecord({
|
||||
provider: row.provider,
|
||||
providerSubject: row.provider_subject,
|
||||
version: Number(row.binding_version),
|
||||
state: row.binding_state as 'active' | 'revoked',
|
||||
subjectId: row.binding_subject_id,
|
||||
createdAtMs: Number(row.binding_created_at_ms),
|
||||
});
|
||||
const subject = normalizeIdentitySubjectRecord({
|
||||
subject: {
|
||||
type: row.subject_type as PolicySubject['type'],
|
||||
id: row.subject_id!,
|
||||
},
|
||||
status: row.subject_status as 'active' | 'disabled',
|
||||
version: Number(row.subject_version),
|
||||
createdAtMs: Number(row.subject_created_at_ms),
|
||||
updatedAtMs: Number(row.subject_updated_at_ms),
|
||||
});
|
||||
if (binding.subjectId !== subject.subject.id) {
|
||||
throw new IdentityDirectoryUnavailableError();
|
||||
}
|
||||
if (
|
||||
binding.state !== 'active' ||
|
||||
subject.status !== 'active' ||
|
||||
subject.subject.type !== 'user'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return subject.subject;
|
||||
} catch (error) {
|
||||
if (error instanceof IdentityDirectoryUnavailableError) throw error;
|
||||
throw new IdentityDirectoryUnavailableError();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { LOCAL_ARTIFACT_RETENTION_TABLE } from '../../../migrations/0015-local-artifact-retention';
|
||||
import {
|
||||
normalizeLocalArtifactReadMetadata,
|
||||
type LocalArtifactReadMetadata,
|
||||
} from '../../domain/artifactRead';
|
||||
import type { LocalArtifactReadMetadataRepository } from '../../ports/localArtifactReadMetadataRepository';
|
||||
|
||||
interface ArtifactMetadataRow {
|
||||
project_id: string;
|
||||
run_id: string;
|
||||
attempt_id: string;
|
||||
attempt_finished_at_ms: number | string | null;
|
||||
log_artifact_id: string;
|
||||
retention_log_artifact_id: string | null;
|
||||
retention_disposition: string | null;
|
||||
retention_finished_at_ms: number | string | null;
|
||||
retention_eligible_at_ms: number | string | null;
|
||||
retention_bytes_reclaimed: number | string | null;
|
||||
retention_recorded_at_ms: number | string | null;
|
||||
}
|
||||
|
||||
export class CorruptLocalArtifactReadMetadataError extends Error {
|
||||
constructor() {
|
||||
super('Local Artifact read metadata is corrupt or ambiguous');
|
||||
this.name = 'CorruptLocalArtifactReadMetadataError';
|
||||
}
|
||||
}
|
||||
|
||||
function rowToMetadata(
|
||||
row: ArtifactMetadataRow,
|
||||
): Readonly<LocalArtifactReadMetadata> {
|
||||
const retentionValues = [
|
||||
row.retention_log_artifact_id,
|
||||
row.retention_disposition,
|
||||
row.retention_finished_at_ms,
|
||||
row.retention_eligible_at_ms,
|
||||
row.retention_bytes_reclaimed,
|
||||
row.retention_recorded_at_ms,
|
||||
];
|
||||
const hasRetention = retentionValues.every((value) => value !== null);
|
||||
if (!hasRetention && retentionValues.some((value) => value !== null)) {
|
||||
throw new CorruptLocalArtifactReadMetadataError();
|
||||
}
|
||||
if (hasRetention && row.retention_log_artifact_id !== row.log_artifact_id) {
|
||||
throw new CorruptLocalArtifactReadMetadataError();
|
||||
}
|
||||
if (
|
||||
hasRetention &&
|
||||
(row.attempt_finished_at_ms === null ||
|
||||
Number(row.retention_finished_at_ms) !==
|
||||
Number(row.attempt_finished_at_ms))
|
||||
) {
|
||||
throw new CorruptLocalArtifactReadMetadataError();
|
||||
}
|
||||
try {
|
||||
return normalizeLocalArtifactReadMetadata({
|
||||
projectId: row.project_id,
|
||||
runId: row.run_id,
|
||||
attemptId: row.attempt_id,
|
||||
logArtifactId: row.log_artifact_id,
|
||||
...(hasRetention
|
||||
? {
|
||||
retention: {
|
||||
disposition: row.retention_disposition as NonNullable<
|
||||
LocalArtifactReadMetadata['retention']
|
||||
>['disposition'],
|
||||
finishedAtMs: Number(row.retention_finished_at_ms),
|
||||
eligibleAtMs: Number(row.retention_eligible_at_ms),
|
||||
bytesReclaimed: Number(row.retention_bytes_reclaimed),
|
||||
recordedAtMs: Number(row.retention_recorded_at_ms),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof CorruptLocalArtifactReadMetadataError) throw error;
|
||||
throw new CorruptLocalArtifactReadMetadataError();
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeLocalArtifactReadMetadataRepository
|
||||
implements LocalArtifactReadMetadataRepository
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Local Artifact read metadata repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async find({
|
||||
projectId,
|
||||
runId,
|
||||
logArtifactId,
|
||||
}: Parameters<
|
||||
LocalArtifactReadMetadataRepository['find']
|
||||
>[0]): Promise<Readonly<LocalArtifactReadMetadata> | null> {
|
||||
const rows = await this.database.query<ArtifactMetadataRow>(
|
||||
`SELECT run.project_id,
|
||||
run.id AS run_id,
|
||||
attempt.id AS attempt_id,
|
||||
attempt.finished_at_ms AS attempt_finished_at_ms,
|
||||
attempt.log_artifact_id,
|
||||
retained.log_artifact_id AS retention_log_artifact_id,
|
||||
retained.disposition AS retention_disposition,
|
||||
retained.finished_at_ms AS retention_finished_at_ms,
|
||||
retained.eligible_at_ms AS retention_eligible_at_ms,
|
||||
retained.bytes_reclaimed AS retention_bytes_reclaimed,
|
||||
retained.recorded_at_ms AS retention_recorded_at_ms
|
||||
FROM "${RUN_TABLE}" AS run
|
||||
JOIN "${RUN_ATTEMPT_TABLE}" AS attempt ON attempt.run_id = run.id
|
||||
LEFT JOIN "${LOCAL_ARTIFACT_RETENTION_TABLE}" AS retained
|
||||
ON retained.attempt_id = attempt.id
|
||||
WHERE run.project_id = :projectId
|
||||
AND run.id = :runId
|
||||
AND run.execution_owner = 'runtime'
|
||||
AND attempt.executor_type = 'local_process'
|
||||
AND attempt.log_artifact_id = :logArtifactId
|
||||
AND attempt.log_artifact_id LIKE 'local-%'
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId, runId, logArtifactId },
|
||||
},
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length !== 1) throw new CorruptLocalArtifactReadMetadataError();
|
||||
return rowToMetadata(rows[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
type ModelStatic,
|
||||
Sequelize,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE } from '../../../migrations/0016-local-artifact-maintenance-cursor';
|
||||
import {
|
||||
normalizeLocalArtifactRetentionCursor,
|
||||
assertLocalArtifactRetentionTimestamp,
|
||||
} from '../../domain/localArtifactRetention';
|
||||
import {
|
||||
normalizeLocalArtifactRetentionCheckpoint,
|
||||
type LocalArtifactRetentionCheckpoint,
|
||||
} from '../../domain/localArtifactRetentionCheckpoint';
|
||||
import type { LocalArtifactRetentionCheckpointStore } from '../../ports/localArtifactRetentionCheckpointStore';
|
||||
|
||||
const RETENTION_SCOPE = 'retention';
|
||||
|
||||
interface CursorRow {
|
||||
scope: string;
|
||||
cursorFinishedAtMs: number | string | null;
|
||||
cursorAttemptId: string | null;
|
||||
version: number | string;
|
||||
updatedAtMs: number | string;
|
||||
}
|
||||
|
||||
interface CursorInstance extends Model<CursorRow, CursorRow>, CursorRow {}
|
||||
|
||||
function defineCursorModel(database: Sequelize): ModelStatic<CursorInstance> {
|
||||
return database.define<CursorInstance>(
|
||||
'Ql3LocalArtifactMaintenanceCursor',
|
||||
{
|
||||
scope: { type: DataTypes.STRING(32), allowNull: false, primaryKey: true },
|
||||
cursorFinishedAtMs: {
|
||||
field: 'cursor_finished_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
cursorAttemptId: {
|
||||
field: 'cursor_attempt_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: true,
|
||||
},
|
||||
version: { type: DataTypes.BIGINT, allowNull: false },
|
||||
updatedAtMs: {
|
||||
field: 'updated_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function rowToCheckpoint(
|
||||
row: CursorRow,
|
||||
): Readonly<LocalArtifactRetentionCheckpoint> {
|
||||
const finishedAtMs =
|
||||
row.cursorFinishedAtMs === null ? null : Number(row.cursorFinishedAtMs);
|
||||
const attemptId = row.cursorAttemptId;
|
||||
if ((finishedAtMs === null) !== (attemptId === null)) {
|
||||
throw new TypeError('Local Artifact retention cursor row is corrupt');
|
||||
}
|
||||
return normalizeLocalArtifactRetentionCheckpoint({
|
||||
version: Number(row.version),
|
||||
...(finishedAtMs === null || attemptId === null
|
||||
? {}
|
||||
: { cursor: { finishedAtMs, attemptId } }),
|
||||
});
|
||||
}
|
||||
|
||||
export class LegacySequelizeLocalArtifactRetentionCheckpointStore
|
||||
implements LocalArtifactRetentionCheckpointStore
|
||||
{
|
||||
private readonly cursors: ModelStatic<CursorInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Local Artifact retention checkpoint store is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.cursors = defineCursorModel(database);
|
||||
}
|
||||
|
||||
async load(): Promise<Readonly<LocalArtifactRetentionCheckpoint>> {
|
||||
const row = await this.cursors.findByPk(RETENTION_SCOPE, { raw: true });
|
||||
return row
|
||||
? rowToCheckpoint(row)
|
||||
: normalizeLocalArtifactRetentionCheckpoint({ version: 0 });
|
||||
}
|
||||
|
||||
async compareAndSet({
|
||||
expectedVersion,
|
||||
cursor,
|
||||
updatedAtMs,
|
||||
}: Parameters<
|
||||
LocalArtifactRetentionCheckpointStore['compareAndSet']
|
||||
>[0]): Promise<boolean> {
|
||||
const checkpoint = normalizeLocalArtifactRetentionCheckpoint({
|
||||
version: expectedVersion,
|
||||
...(cursor ? { cursor } : {}),
|
||||
});
|
||||
assertLocalArtifactRetentionTimestamp('updatedAtMs', updatedAtMs);
|
||||
const next = {
|
||||
scope: RETENTION_SCOPE,
|
||||
cursorFinishedAtMs: checkpoint.cursor?.finishedAtMs ?? null,
|
||||
cursorAttemptId: checkpoint.cursor?.attemptId ?? null,
|
||||
version: checkpoint.version + 1,
|
||||
updatedAtMs,
|
||||
};
|
||||
if (checkpoint.version === 0) {
|
||||
try {
|
||||
await this.cursors.create(next);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error instanceof UniqueConstraintError) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const [updated] = await this.cursors.update(next, {
|
||||
where: { scope: RETENTION_SCOPE, version: checkpoint.version },
|
||||
});
|
||||
return updated === 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { COMPLETION_RECEIPT_JOURNAL_TABLE } from '../../../migrations/0007-completion-receipt-journal';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { LOCAL_ARTIFACT_RETENTION_TABLE } from '../../../migrations/0015-local-artifact-retention';
|
||||
import {
|
||||
normalizeLocalArtifactRetentionCandidate,
|
||||
normalizeLocalArtifactRetentionCursor,
|
||||
normalizeLocalArtifactRetentionRecord,
|
||||
assertLocalArtifactRetentionTimestamp,
|
||||
type LocalArtifactRetentionCandidate,
|
||||
type LocalArtifactRetentionRecord,
|
||||
} from '../../domain/localArtifactRetention';
|
||||
import type {
|
||||
LocalArtifactRetentionPage,
|
||||
LocalArtifactRetentionRepository,
|
||||
} from '../../ports/localArtifactRetentionRepository';
|
||||
import { MAX_LOCAL_ARTIFACT_RETENTION_PAGE_SIZE } from '../../ports/localArtifactRetentionRepository';
|
||||
|
||||
interface LocalArtifactRetentionRow {
|
||||
attemptId: string;
|
||||
logArtifactId: string;
|
||||
finishedAtMs: number | string;
|
||||
eligibleAtMs: number | string;
|
||||
disposition: string;
|
||||
bytesReclaimed: number | string;
|
||||
recordedAtMs: number | string;
|
||||
}
|
||||
|
||||
interface LocalArtifactRetentionInstance
|
||||
extends Model<LocalArtifactRetentionRow, LocalArtifactRetentionRow>,
|
||||
LocalArtifactRetentionRow {}
|
||||
|
||||
interface CandidateRow {
|
||||
attempt_id: string;
|
||||
log_artifact_id: string;
|
||||
finished_at_ms: number | string;
|
||||
}
|
||||
|
||||
function defineRetentionModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<LocalArtifactRetentionInstance> {
|
||||
return database.define<LocalArtifactRetentionInstance>(
|
||||
'Ql3LocalArtifactRetention',
|
||||
{
|
||||
attemptId: {
|
||||
field: 'attempt_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
logArtifactId: {
|
||||
field: 'log_artifact_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
finishedAtMs: {
|
||||
field: 'finished_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
eligibleAtMs: {
|
||||
field: 'eligible_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
disposition: { type: DataTypes.STRING(16), allowNull: false },
|
||||
bytesReclaimed: {
|
||||
field: 'bytes_reclaimed',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
recordedAtMs: {
|
||||
field: 'recorded_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: LOCAL_ARTIFACT_RETENTION_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function rowToRecord(
|
||||
row: LocalArtifactRetentionRow,
|
||||
): LocalArtifactRetentionRecord {
|
||||
return normalizeLocalArtifactRetentionRecord({
|
||||
attemptId: row.attemptId,
|
||||
logArtifactId: row.logArtifactId,
|
||||
finishedAtMs: Number(row.finishedAtMs),
|
||||
eligibleAtMs: Number(row.eligibleAtMs),
|
||||
disposition: row.disposition as LocalArtifactRetentionRecord['disposition'],
|
||||
bytesReclaimed: Number(row.bytesReclaimed),
|
||||
recordedAtMs: Number(row.recordedAtMs),
|
||||
});
|
||||
}
|
||||
|
||||
function sameRetirementIdentity(
|
||||
left: LocalArtifactRetentionRecord,
|
||||
right: LocalArtifactRetentionRecord,
|
||||
): boolean {
|
||||
return (
|
||||
left.attemptId === right.attemptId &&
|
||||
left.logArtifactId === right.logArtifactId &&
|
||||
left.finishedAtMs === right.finishedAtMs
|
||||
);
|
||||
}
|
||||
|
||||
export class LocalArtifactRetentionRecordConflictError extends Error {
|
||||
constructor() {
|
||||
super('Local Artifact retention record conflicts with existing evidence');
|
||||
this.name = 'LocalArtifactRetentionRecordConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeLocalArtifactRetentionRepository
|
||||
implements LocalArtifactRetentionRepository
|
||||
{
|
||||
private readonly retention: ModelStatic<LocalArtifactRetentionInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Local Artifact retention repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.retention = defineRetentionModel(database);
|
||||
}
|
||||
|
||||
async list({
|
||||
cutoffMs,
|
||||
cursor,
|
||||
limit,
|
||||
}: Parameters<
|
||||
LocalArtifactRetentionRepository['list']
|
||||
>[0]): Promise<LocalArtifactRetentionPage> {
|
||||
assertLocalArtifactRetentionTimestamp('cutoffMs', cutoffMs);
|
||||
const normalizedCursor = cursor
|
||||
? normalizeLocalArtifactRetentionCursor(cursor)
|
||||
: undefined;
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_LOCAL_ARTIFACT_RETENTION_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError('Local Artifact retention page size is invalid');
|
||||
}
|
||||
const replacements: Record<string, string | number> = {
|
||||
cutoffMs,
|
||||
fetchLimit: limit + 1,
|
||||
...(normalizedCursor
|
||||
? {
|
||||
cursorFinishedAtMs: normalizedCursor.finishedAtMs,
|
||||
cursorAttemptId: normalizedCursor.attemptId,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const cursorPredicate = normalizedCursor
|
||||
? `AND (
|
||||
attempt.finished_at_ms > :cursorFinishedAtMs OR
|
||||
(attempt.finished_at_ms = :cursorFinishedAtMs AND attempt.id > :cursorAttemptId)
|
||||
)`
|
||||
: '';
|
||||
const rows = await this.database.query<CandidateRow>(
|
||||
`SELECT attempt.id AS attempt_id,
|
||||
attempt.log_artifact_id,
|
||||
attempt.finished_at_ms
|
||||
FROM "${RUN_ATTEMPT_TABLE}" AS attempt
|
||||
JOIN "${RUN_TABLE}" AS run ON run.id = attempt.run_id
|
||||
WHERE run.execution_owner = 'runtime'
|
||||
AND run.status IN ('succeeded','failed','cancelled','timed_out')
|
||||
AND attempt.status IN ('succeeded','failed','cancelled','timed_out')
|
||||
AND attempt.executor_type = 'local_process'
|
||||
AND attempt.log_artifact_id LIKE 'local-%'
|
||||
AND attempt.finished_at_ms IS NOT NULL
|
||||
AND attempt.finished_at_ms <= :cutoffMs
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM "${COMPLETION_RECEIPT_JOURNAL_TABLE}" AS receipt
|
||||
WHERE receipt.attempt_id = attempt.id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM "${LOCAL_ARTIFACT_RETENTION_TABLE}" AS retained
|
||||
WHERE retained.attempt_id = attempt.id
|
||||
)
|
||||
${cursorPredicate}
|
||||
ORDER BY attempt.finished_at_ms ASC, attempt.id ASC
|
||||
LIMIT :fetchLimit`,
|
||||
{ type: QueryTypes.SELECT, replacements },
|
||||
);
|
||||
const truncated = rows.length > limit;
|
||||
const selected = truncated ? rows.slice(0, limit) : rows;
|
||||
const candidates: LocalArtifactRetentionCandidate[] = selected.map((row) =>
|
||||
normalizeLocalArtifactRetentionCandidate({
|
||||
attemptId: row.attempt_id,
|
||||
logArtifactId: row.log_artifact_id,
|
||||
finishedAtMs: Number(row.finished_at_ms),
|
||||
}),
|
||||
);
|
||||
const last = candidates[candidates.length - 1];
|
||||
return Object.freeze({
|
||||
candidates: Object.freeze(candidates),
|
||||
truncated,
|
||||
...(truncated && last
|
||||
? {
|
||||
nextCursor: Object.freeze({
|
||||
finishedAtMs: last.finishedAtMs,
|
||||
attemptId: last.attemptId,
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
async record(
|
||||
value: LocalArtifactRetentionRecord,
|
||||
): Promise<'inserted' | 'existing'> {
|
||||
const record = normalizeLocalArtifactRetentionRecord(value);
|
||||
const row: LocalArtifactRetentionRow = {
|
||||
attemptId: record.attemptId,
|
||||
logArtifactId: record.logArtifactId,
|
||||
finishedAtMs: record.finishedAtMs,
|
||||
eligibleAtMs: record.eligibleAtMs,
|
||||
disposition: record.disposition,
|
||||
bytesReclaimed: record.bytesReclaimed,
|
||||
recordedAtMs: record.recordedAtMs,
|
||||
};
|
||||
try {
|
||||
await this.retention.create(row);
|
||||
return 'inserted';
|
||||
} catch (error) {
|
||||
if (!(error instanceof UniqueConstraintError)) throw error;
|
||||
}
|
||||
const existing = await this.retention.findByPk(record.attemptId, {
|
||||
raw: true,
|
||||
});
|
||||
if (existing && sameRetirementIdentity(rowToRecord(existing), record))
|
||||
return 'existing';
|
||||
throw new LocalArtifactRetentionRecordConflictError();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Sequelize,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE } from '../../../migrations/0013-local-execution-context-recipes';
|
||||
import {
|
||||
assertLocalExecutionContextRef,
|
||||
createLocalExecutionContextRecipeRecord,
|
||||
localExecutionContextRecipeDigest,
|
||||
normalizeLocalExecutionContextRecipe,
|
||||
type LocalExecutionContextRecipe,
|
||||
} from '../../domain/localExecutionContextRecipe';
|
||||
import type {
|
||||
InsertLocalExecutionContextRecipeResult,
|
||||
LocalExecutionContextRecipeRepository,
|
||||
} from '../../ports/localExecutionContextRecipeRepository';
|
||||
|
||||
interface LocalExecutionContextRecipeRow {
|
||||
contextRef: string;
|
||||
environmentRecipe: string;
|
||||
contentDigest: string;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface LocalExecutionContextRecipeInstance
|
||||
extends Model<LocalExecutionContextRecipeRow, LocalExecutionContextRecipeRow>,
|
||||
LocalExecutionContextRecipeRow {}
|
||||
|
||||
export class LocalExecutionContextRecipeConflictError extends Error {
|
||||
constructor(readonly contextRef: string) {
|
||||
super(`Local execution context recipe ${contextRef} is immutable`);
|
||||
this.name = 'LocalExecutionContextRecipeConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalExecutionContextRecipeCorruptError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'LocalExecutionContextRecipeCorruptError';
|
||||
}
|
||||
}
|
||||
|
||||
function defineRecipeModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<LocalExecutionContextRecipeInstance> {
|
||||
return database.define<LocalExecutionContextRecipeInstance>(
|
||||
'Ql3LocalExecutionContextRecipe',
|
||||
{
|
||||
contextRef: {
|
||||
field: 'context_ref',
|
||||
type: DataTypes.STRING(512),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
environmentRecipe: {
|
||||
field: 'environment_recipe',
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: false,
|
||||
},
|
||||
contentDigest: {
|
||||
field: 'content_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
function rowToRecipe(
|
||||
row: LocalExecutionContextRecipeRow,
|
||||
): LocalExecutionContextRecipe {
|
||||
let environment: unknown;
|
||||
try {
|
||||
environment = JSON.parse(row.environmentRecipe);
|
||||
} catch {
|
||||
throw new LocalExecutionContextRecipeCorruptError(
|
||||
'Stored local execution context recipe is not valid JSON',
|
||||
);
|
||||
}
|
||||
try {
|
||||
const normalized = normalizeLocalExecutionContextRecipe({
|
||||
contextRef: row.contextRef,
|
||||
environment: environment as LocalExecutionContextRecipe['environment'],
|
||||
});
|
||||
if (JSON.stringify(normalized.environment) !== row.environmentRecipe) {
|
||||
throw new LocalExecutionContextRecipeCorruptError(
|
||||
'Stored local execution context recipe is not canonical',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!/^[0-9a-f]{64}$/.test(row.contentDigest) ||
|
||||
localExecutionContextRecipeDigest(normalized) !== row.contentDigest
|
||||
) {
|
||||
throw new LocalExecutionContextRecipeCorruptError(
|
||||
'Stored local execution context recipe digest does not match',
|
||||
);
|
||||
}
|
||||
return createLocalExecutionContextRecipeRecord(
|
||||
normalized,
|
||||
Number(row.createdAtMs),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalExecutionContextRecipeCorruptError) throw error;
|
||||
throw new LocalExecutionContextRecipeCorruptError(
|
||||
`Stored local execution context recipe is invalid: ${
|
||||
error instanceof Error ? error.message : 'unknown validation error'
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeLocalExecutionContextRecipeRepository
|
||||
implements LocalExecutionContextRecipeRepository
|
||||
{
|
||||
private readonly recipe: ModelStatic<LocalExecutionContextRecipeInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy local context recipe repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.recipe = defineRecipeModel(database);
|
||||
}
|
||||
|
||||
async resolve(
|
||||
contextRef: string,
|
||||
): Promise<LocalExecutionContextRecipe | null> {
|
||||
assertLocalExecutionContextRef(contextRef);
|
||||
const row = (await this.recipe.findByPk(contextRef, {
|
||||
raw: true,
|
||||
})) as unknown as LocalExecutionContextRecipeRow | null;
|
||||
return row ? rowToRecipe(row) : null;
|
||||
}
|
||||
|
||||
async insert(
|
||||
recipe: LocalExecutionContextRecipe,
|
||||
createdAtMs: number,
|
||||
): Promise<InsertLocalExecutionContextRecipeResult> {
|
||||
const record = createLocalExecutionContextRecipeRecord(recipe, createdAtMs);
|
||||
const values: LocalExecutionContextRecipeRow = {
|
||||
contextRef: record.contextRef,
|
||||
environmentRecipe: JSON.stringify(record.environment),
|
||||
contentDigest: record.contentDigest,
|
||||
createdAtMs: record.createdAtMs,
|
||||
};
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
try {
|
||||
await this.recipe.create(values);
|
||||
return 'inserted';
|
||||
} catch (error) {
|
||||
if (error instanceof UniqueConstraintError) {
|
||||
const existing = await this.resolve(record.contextRef);
|
||||
if (
|
||||
existing &&
|
||||
localExecutionContextRecipeDigest(existing) === record.contentDigest
|
||||
) {
|
||||
return 'idempotent';
|
||||
}
|
||||
throw new LocalExecutionContextRecipeConflictError(record.contextRef);
|
||||
}
|
||||
if (errorCode(error) === 'SQLITE_BUSY' && attempt < 4) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new Error('Local context recipe insert retry budget exhausted');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { LOCAL_SECRET_ENVELOPE_TABLE } from '../../../migrations/0014-local-secret-envelopes';
|
||||
import {
|
||||
LOCAL_SECRET_ALGORITHM,
|
||||
LocalSecretUnavailableError,
|
||||
LocalSecretVersionConflictError,
|
||||
assertLocalSecretMutationId,
|
||||
assertLocalSecretName,
|
||||
assertLocalSecretProjectId,
|
||||
createLocalSecretRef,
|
||||
normalizeLocalSecretEnvelope,
|
||||
type LocalSecretEnvelope,
|
||||
type LocalSecretReference,
|
||||
} from '../../domain/localSecret';
|
||||
import type {
|
||||
AppendLocalSecretEnvelopeCommand,
|
||||
AppendLocalSecretEnvelopeResult,
|
||||
LocalSecretEnvelopeRepository,
|
||||
} from '../../ports/localSecretEnvelopeRepository';
|
||||
|
||||
const MAX_BATCH_SIZE = 64;
|
||||
const RETRY_ATTEMPTS = 5;
|
||||
|
||||
interface LocalSecretEnvelopeRow {
|
||||
projectId: string;
|
||||
name: string;
|
||||
version: number;
|
||||
mutationId: string;
|
||||
keyId: string;
|
||||
algorithm: string;
|
||||
nonce: Buffer;
|
||||
ciphertext: Buffer;
|
||||
authTag: Buffer;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface LocalSecretEnvelopeInstance
|
||||
extends Model<LocalSecretEnvelopeRow, LocalSecretEnvelopeRow>,
|
||||
LocalSecretEnvelopeRow {}
|
||||
|
||||
interface ResolvedSecretRow {
|
||||
position: number;
|
||||
project_id: string | null;
|
||||
secret_name: string | null;
|
||||
version: number | null;
|
||||
mutation_id: string | null;
|
||||
key_id: string | null;
|
||||
algorithm: string | null;
|
||||
nonce: Buffer | null;
|
||||
ciphertext: Buffer | null;
|
||||
auth_tag: Buffer | null;
|
||||
created_at_ms: number | string | null;
|
||||
}
|
||||
|
||||
function defineLocalSecretEnvelopeModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<LocalSecretEnvelopeInstance> {
|
||||
return database.define<LocalSecretEnvelopeInstance>(
|
||||
'Ql3LocalSecretEnvelope',
|
||||
{
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
name: {
|
||||
field: 'secret_name',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
version: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
mutationId: {
|
||||
field: 'mutation_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
keyId: {
|
||||
field: 'key_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
},
|
||||
algorithm: {
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
nonce: { type: DataTypes.BLOB, allowNull: false },
|
||||
ciphertext: { type: DataTypes.BLOB, allowNull: false },
|
||||
authTag: {
|
||||
field: 'auth_tag',
|
||||
type: DataTypes.BLOB,
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: LOCAL_SECRET_ENVELOPE_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function rowToEnvelope(row: LocalSecretEnvelopeRow): LocalSecretEnvelope {
|
||||
try {
|
||||
return normalizeLocalSecretEnvelope({
|
||||
projectId: row.projectId,
|
||||
name: row.name,
|
||||
version: Number(row.version),
|
||||
mutationId: row.mutationId,
|
||||
keyId: row.keyId,
|
||||
algorithm: row.algorithm as typeof LOCAL_SECRET_ALGORITHM,
|
||||
nonce: Buffer.from(row.nonce).toString('base64url'),
|
||||
ciphertext: Buffer.from(row.ciphertext).toString('base64url'),
|
||||
authTag: Buffer.from(row.authTag).toString('base64url'),
|
||||
createdAtMs: Number(row.createdAtMs),
|
||||
});
|
||||
} catch {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function resolvedRowToEnvelope(
|
||||
row: ResolvedSecretRow,
|
||||
): LocalSecretEnvelope | null {
|
||||
if (row.version === null) return null;
|
||||
if (
|
||||
row.project_id === null ||
|
||||
row.secret_name === null ||
|
||||
row.mutation_id === null ||
|
||||
row.key_id === null ||
|
||||
row.algorithm === null ||
|
||||
row.nonce === null ||
|
||||
row.ciphertext === null ||
|
||||
row.auth_tag === null ||
|
||||
row.created_at_ms === null
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
return rowToEnvelope({
|
||||
projectId: row.project_id,
|
||||
name: row.secret_name,
|
||||
version: row.version,
|
||||
mutationId: row.mutation_id,
|
||||
keyId: row.key_id,
|
||||
algorithm: row.algorithm,
|
||||
nonce: row.nonce,
|
||||
ciphertext: row.ciphertext,
|
||||
authTag: row.auth_tag,
|
||||
createdAtMs: row.created_at_ms,
|
||||
});
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
function assertExpectedVersion(value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0 || value >= 2_147_483_647) {
|
||||
throw new TypeError('Local Secret expected current version is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeLocalSecretEnvelopeRepository
|
||||
implements LocalSecretEnvelopeRepository
|
||||
{
|
||||
private readonly envelope: ModelStatic<LocalSecretEnvelopeInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Local Secret envelope repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.envelope = defineLocalSecretEnvelopeModel(database);
|
||||
}
|
||||
|
||||
async append(
|
||||
command: AppendLocalSecretEnvelopeCommand,
|
||||
): Promise<AppendLocalSecretEnvelopeResult> {
|
||||
assertExpectedVersion(command.expectedCurrentVersion);
|
||||
const envelope = normalizeLocalSecretEnvelope(command.envelope);
|
||||
if (envelope.version !== command.expectedCurrentVersion + 1) {
|
||||
throw new LocalSecretVersionConflictError();
|
||||
}
|
||||
const values = this.values(envelope);
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const replay = await this.envelope.findOne({
|
||||
where: {
|
||||
projectId: envelope.projectId,
|
||||
name: envelope.name,
|
||||
mutationId: envelope.mutationId,
|
||||
},
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (replay) {
|
||||
return { status: 'existing', envelope: rowToEnvelope(replay) };
|
||||
}
|
||||
const current = await this.envelope.findOne({
|
||||
where: { projectId: envelope.projectId, name: envelope.name },
|
||||
order: [['version', 'DESC']],
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
const currentVersion = current ? Number(current.version) : 0;
|
||||
if (currentVersion !== command.expectedCurrentVersion) {
|
||||
throw new LocalSecretVersionConflictError();
|
||||
}
|
||||
await this.envelope.create(values, { transaction });
|
||||
return { status: 'inserted', envelope };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalSecretVersionConflictError) throw error;
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
|
||||
async findByMutation(
|
||||
projectId: string,
|
||||
name: string,
|
||||
mutationId: string,
|
||||
): Promise<LocalSecretEnvelope | null> {
|
||||
assertLocalSecretProjectId(projectId);
|
||||
assertLocalSecretName(name);
|
||||
assertLocalSecretMutationId(mutationId);
|
||||
const row = await this.envelope.findOne({
|
||||
where: { projectId, name, mutationId },
|
||||
raw: true,
|
||||
});
|
||||
return row ? rowToEnvelope(row) : null;
|
||||
}
|
||||
|
||||
async resolveMany(
|
||||
references: readonly LocalSecretReference[],
|
||||
): Promise<readonly (LocalSecretEnvelope | null)[]> {
|
||||
if (!Array.isArray(references) || references.length > MAX_BATCH_SIZE) {
|
||||
throw new RangeError('Local Secret batch is too large');
|
||||
}
|
||||
if (references.length === 0) return Object.freeze([]);
|
||||
const replacements: Record<string, string | number | null> = {};
|
||||
const requestedValues = references.map((reference, position) => {
|
||||
createLocalSecretRef(reference);
|
||||
replacements[`position${position}`] = position;
|
||||
replacements[`project${position}`] = reference.projectId;
|
||||
replacements[`name${position}`] = reference.name;
|
||||
replacements[`version${position}`] = reference.version ?? null;
|
||||
return `(:position${position}, :project${position}, :name${position}, :version${position})`;
|
||||
});
|
||||
const rows = await this.database.query<ResolvedSecretRow>(
|
||||
`WITH requested(position, project_id, secret_name, requested_version) AS (
|
||||
VALUES ${requestedValues.join(', ')}
|
||||
)
|
||||
SELECT requested.position,
|
||||
envelope.project_id,
|
||||
envelope.secret_name,
|
||||
envelope.version,
|
||||
envelope.mutation_id,
|
||||
envelope.key_id,
|
||||
envelope.algorithm,
|
||||
envelope.nonce,
|
||||
envelope.ciphertext,
|
||||
envelope.auth_tag,
|
||||
envelope.created_at_ms
|
||||
FROM requested
|
||||
LEFT JOIN "${LOCAL_SECRET_ENVELOPE_TABLE}" AS envelope
|
||||
ON envelope.project_id = requested.project_id
|
||||
AND envelope.secret_name = requested.secret_name
|
||||
AND envelope.version = COALESCE(
|
||||
requested.requested_version,
|
||||
(SELECT MAX(current.version)
|
||||
FROM "${LOCAL_SECRET_ENVELOPE_TABLE}" AS current
|
||||
WHERE current.project_id = requested.project_id
|
||||
AND current.secret_name = requested.secret_name)
|
||||
)
|
||||
ORDER BY requested.position ASC`,
|
||||
{ type: QueryTypes.SELECT, replacements },
|
||||
);
|
||||
if (rows.length !== references.length) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
return Object.freeze(rows.map(resolvedRowToEnvelope));
|
||||
}
|
||||
|
||||
private values(envelope: LocalSecretEnvelope): LocalSecretEnvelopeRow {
|
||||
return {
|
||||
projectId: envelope.projectId,
|
||||
name: envelope.name,
|
||||
version: envelope.version,
|
||||
mutationId: envelope.mutationId,
|
||||
keyId: envelope.keyId,
|
||||
algorithm: envelope.algorithm,
|
||||
nonce: Buffer.from(envelope.nonce, 'base64url'),
|
||||
ciphertext: Buffer.from(envelope.ciphertext, 'base64url'),
|
||||
authTag: Buffer.from(envelope.authTag, 'base64url'),
|
||||
createdAtMs: envelope.createdAtMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Op,
|
||||
Sequelize,
|
||||
type WhereOptions,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import type { ExecutionStopKind, ExecutorType } from '../../domain/execution';
|
||||
import type {
|
||||
PrimaryCancellationAttemptReference,
|
||||
PrimaryCancellationCandidate,
|
||||
PrimaryCancellationCursor,
|
||||
PrimaryCancellationPage,
|
||||
PrimaryCancellationSource,
|
||||
} from '../../ports/primaryCancellationSource';
|
||||
import { MAX_PRIMARY_CANCELLATION_BATCH_SIZE } from '../../ports/primaryCancellationSource';
|
||||
|
||||
interface CancellationRunRow {
|
||||
id: string;
|
||||
executionOwner: string;
|
||||
status: string;
|
||||
cancelRequestedAtMs: number | null;
|
||||
cancelReason: string | null;
|
||||
}
|
||||
|
||||
interface CancellationRequestedRunRow extends CancellationRunRow {
|
||||
cancelRequestedAtMs: number;
|
||||
cancelReason: string;
|
||||
}
|
||||
|
||||
interface CancellationAttemptRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
attempt: number;
|
||||
status: string;
|
||||
executorType: string;
|
||||
executorHandle: string | null;
|
||||
pid: number | null;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
interface CancellationRunInstance
|
||||
extends Model<CancellationRunRow, CancellationRunRow>,
|
||||
CancellationRunRow {}
|
||||
interface CancellationAttemptInstance
|
||||
extends Model<CancellationAttemptRow, CancellationAttemptRow>,
|
||||
CancellationAttemptRow {}
|
||||
|
||||
function defineCancellationRunModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationRunInstance> {
|
||||
return database.define<CancellationRunInstance>(
|
||||
'Ql3PrimaryCancellationRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
executionOwner: {
|
||||
field: 'execution_owner',
|
||||
type: DataTypes.STRING(16),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
cancelRequestedAtMs: {
|
||||
field: 'cancel_requested_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
cancelReason: {
|
||||
field: 'cancel_reason',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function defineCancellationAttemptModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationAttemptInstance> {
|
||||
return database.define<CancellationAttemptInstance>(
|
||||
'Ql3PrimaryCancellationAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
runId: {
|
||||
field: 'run_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
attempt: { type: DataTypes.INTEGER, allowNull: false },
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
executorType: {
|
||||
field: 'executor_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
executorHandle: {
|
||||
field: 'executor_handle',
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
},
|
||||
pid: { type: DataTypes.INTEGER, allowNull: true },
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: RUN_ATTEMPT_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function assertCursor(cursor: PrimaryCancellationCursor): void {
|
||||
if (!Number.isSafeInteger(cursor.requestedAtMs) || cursor.requestedAtMs < 0) {
|
||||
throw new RangeError('cursor.requestedAtMs must be a non-negative integer');
|
||||
}
|
||||
if (!cursor.runId || cursor.runId.length > 36) {
|
||||
throw new RangeError('cursor.runId must be between 1 and 36 characters');
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizePrimaryCancellationSource
|
||||
implements PrimaryCancellationSource
|
||||
{
|
||||
private readonly run: ModelStatic<CancellationRunInstance>;
|
||||
private readonly attempt: ModelStatic<CancellationAttemptInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.run = defineCancellationRunModel(database);
|
||||
this.attempt = defineCancellationAttemptModel(database);
|
||||
}
|
||||
|
||||
async listCandidates({
|
||||
cursor,
|
||||
limit = 32,
|
||||
}: {
|
||||
cursor?: PrimaryCancellationCursor;
|
||||
limit?: number;
|
||||
} = {}): Promise<PrimaryCancellationPage> {
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_PRIMARY_CANCELLATION_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_PRIMARY_CANCELLATION_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (cursor) assertCursor(cursor);
|
||||
|
||||
const where: WhereOptions<CancellationRunRow> = {
|
||||
executionOwner: 'runtime',
|
||||
status: {
|
||||
[Op.in]: [
|
||||
'created',
|
||||
'queued',
|
||||
'dispatching',
|
||||
'running',
|
||||
'waiting_approval',
|
||||
'retry_wait',
|
||||
'lost',
|
||||
],
|
||||
},
|
||||
cancelRequestedAtMs: {
|
||||
[Op.ne]: null,
|
||||
},
|
||||
cancelReason: { [Op.ne]: null },
|
||||
...(cursor === undefined
|
||||
? {}
|
||||
: {
|
||||
[Op.or]: [
|
||||
{ cancelRequestedAtMs: { [Op.gt]: cursor.requestedAtMs } },
|
||||
{
|
||||
cancelRequestedAtMs: cursor.requestedAtMs,
|
||||
id: { [Op.gt]: cursor.runId },
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
const runRows = (await this.run.findAll({
|
||||
attributes: ['id', 'cancelRequestedAtMs', 'cancelReason'],
|
||||
where,
|
||||
order: [
|
||||
['cancelRequestedAtMs', 'ASC'],
|
||||
['id', 'ASC'],
|
||||
],
|
||||
limit: limit + 1,
|
||||
raw: true,
|
||||
})) as unknown as CancellationRunRow[];
|
||||
const truncated = runRows.length > limit;
|
||||
const boundedRuns = runRows
|
||||
.filter(
|
||||
(run): run is CancellationRequestedRunRow =>
|
||||
run.cancelRequestedAtMs !== null && run.cancelReason !== null,
|
||||
)
|
||||
.slice(0, limit);
|
||||
if (boundedRuns.length === 0) {
|
||||
return {
|
||||
candidates: [],
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
};
|
||||
}
|
||||
|
||||
const maxAttemptRows = limit * 2;
|
||||
const attemptRows = (await this.attempt.findAll({
|
||||
attributes: [
|
||||
'id',
|
||||
'runId',
|
||||
'attempt',
|
||||
'executorType',
|
||||
'executorHandle',
|
||||
'pid',
|
||||
],
|
||||
where: {
|
||||
runId: { [Op.in]: boundedRuns.map((run) => run.id) },
|
||||
status: { [Op.in]: ['claimed', 'starting', 'running'] },
|
||||
},
|
||||
order: [
|
||||
['runId', 'ASC'],
|
||||
['attempt', 'DESC'],
|
||||
['createdAtMs', 'DESC'],
|
||||
['id', 'DESC'],
|
||||
],
|
||||
limit: maxAttemptRows + 1,
|
||||
raw: true,
|
||||
})) as unknown as CancellationAttemptRow[];
|
||||
if (attemptRows.length > maxAttemptRows) {
|
||||
return {
|
||||
candidates: [],
|
||||
truncated,
|
||||
unsafeAttemptOverflow: true,
|
||||
};
|
||||
}
|
||||
|
||||
const attemptsByRun = new Map<
|
||||
string,
|
||||
PrimaryCancellationAttemptReference[]
|
||||
>();
|
||||
for (const attempt of attemptRows) {
|
||||
const references = attemptsByRun.get(attempt.runId) ?? [];
|
||||
references.push({
|
||||
attemptId: attempt.id,
|
||||
executorType: attempt.executorType as ExecutorType,
|
||||
...(attempt.executorHandle === null
|
||||
? {}
|
||||
: { executorHandle: attempt.executorHandle }),
|
||||
...(attempt.pid === null ? {} : { pid: attempt.pid }),
|
||||
});
|
||||
attemptsByRun.set(attempt.runId, references);
|
||||
}
|
||||
|
||||
const candidates: PrimaryCancellationCandidate[] = boundedRuns.map(
|
||||
(run) => ({
|
||||
runId: run.id,
|
||||
requestedAtMs: run.cancelRequestedAtMs,
|
||||
reason: run.cancelReason as ExecutionStopKind,
|
||||
attempts: attemptsByRun.get(run.id) ?? [],
|
||||
}),
|
||||
);
|
||||
const last = boundedRuns[boundedRuns.length - 1];
|
||||
return {
|
||||
candidates,
|
||||
truncated,
|
||||
unsafeAttemptOverflow: false,
|
||||
nextCursor: {
|
||||
requestedAtMs: last.cancelRequestedAtMs,
|
||||
runId: last.id,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
type ModelStatic,
|
||||
Op,
|
||||
type Sequelize,
|
||||
type Transaction,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { RUNNING_INSTANCE_TABLE } from '../../../migrations/0003-running-instance-run-reference';
|
||||
import type { RunAttemptStatus, RunStatus } from '../../domain/run';
|
||||
import { parseLegacyLogOutputRef } from '../../compatibility/legacyLogOutputRef';
|
||||
import type {
|
||||
SequelizeRunProjectionContext,
|
||||
SequelizeRunProjectionParticipant,
|
||||
} from './projectedRunRepository';
|
||||
|
||||
const CRONTAB_TABLE = 'Crontabs';
|
||||
const CRONTAB_STATUS_RUNNING = 0;
|
||||
const CRONTAB_STATUS_IDLE = 1;
|
||||
const CRONTAB_STATUS_QUEUED = 3;
|
||||
const INSTANCE_STATUS_RUNNING = 0;
|
||||
const INSTANCE_STATUS_FINISHED = 1;
|
||||
const INSTANCE_STATUS_STOPPED = 2;
|
||||
const INSTANCE_STATUS_ERROR = 3;
|
||||
|
||||
const RUNNING_RUN_STATUSES: readonly RunStatus[] = [
|
||||
'running',
|
||||
'waiting_approval',
|
||||
];
|
||||
const QUEUED_RUN_STATUSES: readonly RunStatus[] = [
|
||||
'created',
|
||||
'queued',
|
||||
'dispatching',
|
||||
'retry_wait',
|
||||
];
|
||||
const TERMINAL_RUN_STATUSES: readonly RunStatus[] = [
|
||||
'lost',
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
];
|
||||
|
||||
interface ProjectionRunRow {
|
||||
id: string;
|
||||
legacyCronId: number | null;
|
||||
executionOwner: string;
|
||||
status: RunStatus;
|
||||
outputRef: string | null;
|
||||
createdAtMs: number;
|
||||
startedAtMs: number | null;
|
||||
finishedAtMs: number | null;
|
||||
}
|
||||
|
||||
interface ProjectionAttemptRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
attempt: number;
|
||||
status: RunAttemptStatus;
|
||||
pid: number | null;
|
||||
createdAtMs: number;
|
||||
startedAtMs: number | null;
|
||||
finishedAtMs: number | null;
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
interface ProjectionCrontabRow {
|
||||
id: number;
|
||||
status: number | null;
|
||||
pid: number | null;
|
||||
logPath: string | null;
|
||||
lastRunningTime: number | null;
|
||||
lastExecutionTime: number | null;
|
||||
}
|
||||
|
||||
interface ProjectionInstanceRow {
|
||||
id?: number;
|
||||
cronId: number;
|
||||
runId: string | null;
|
||||
attemptId: string | null;
|
||||
pid: number | null;
|
||||
logPath: string | null;
|
||||
startedAt: number;
|
||||
finishedAt: number | null;
|
||||
status: number;
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
interface ProjectionRunInstance
|
||||
extends Model<ProjectionRunRow, ProjectionRunRow>,
|
||||
ProjectionRunRow {}
|
||||
interface ProjectionAttemptInstance
|
||||
extends Model<ProjectionAttemptRow, ProjectionAttemptRow>,
|
||||
ProjectionAttemptRow {}
|
||||
interface ProjectionCrontabInstance
|
||||
extends Model<ProjectionCrontabRow, ProjectionCrontabRow>,
|
||||
ProjectionCrontabRow {}
|
||||
interface ProjectionInstanceInstance
|
||||
extends Model<ProjectionInstanceRow, ProjectionInstanceRow>,
|
||||
ProjectionInstanceRow {}
|
||||
|
||||
interface ProjectionModels {
|
||||
run: ModelStatic<ProjectionRunInstance>;
|
||||
attempt: ModelStatic<ProjectionAttemptInstance>;
|
||||
crontab: ModelStatic<ProjectionCrontabInstance>;
|
||||
instance: ModelStatic<ProjectionInstanceInstance>;
|
||||
}
|
||||
|
||||
interface SelectedRun {
|
||||
run: ProjectionRunRow;
|
||||
attempt: ProjectionAttemptRow | null;
|
||||
}
|
||||
|
||||
function defineProjectionModels(database: Sequelize): ProjectionModels {
|
||||
const common = { timestamps: false, freezeTableName: true } as const;
|
||||
const run = database.define<ProjectionRunInstance>(
|
||||
'Ql3PrimaryCronProjectionRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
legacyCronId: { field: 'legacy_cron_id', type: DataTypes.INTEGER },
|
||||
executionOwner: {
|
||||
field: 'execution_owner',
|
||||
type: DataTypes.STRING(16),
|
||||
},
|
||||
status: { type: DataTypes.STRING(32) },
|
||||
outputRef: { field: 'output_ref', type: DataTypes.STRING(512) },
|
||||
createdAtMs: { field: 'created_at_ms', type: DataTypes.BIGINT },
|
||||
startedAtMs: { field: 'started_at_ms', type: DataTypes.BIGINT },
|
||||
finishedAtMs: { field: 'finished_at_ms', type: DataTypes.BIGINT },
|
||||
},
|
||||
{ ...common, tableName: RUN_TABLE },
|
||||
);
|
||||
const attempt = database.define<ProjectionAttemptInstance>(
|
||||
'Ql3PrimaryCronProjectionAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
runId: { field: 'run_id', type: DataTypes.STRING(36) },
|
||||
attempt: { type: DataTypes.INTEGER },
|
||||
status: { type: DataTypes.STRING(32) },
|
||||
pid: { type: DataTypes.INTEGER },
|
||||
createdAtMs: { field: 'created_at_ms', type: DataTypes.BIGINT },
|
||||
startedAtMs: { field: 'started_at_ms', type: DataTypes.BIGINT },
|
||||
finishedAtMs: { field: 'finished_at_ms', type: DataTypes.BIGINT },
|
||||
exitCode: { field: 'exit_code', type: DataTypes.INTEGER },
|
||||
},
|
||||
{ ...common, tableName: RUN_ATTEMPT_TABLE },
|
||||
);
|
||||
const crontab = database.define<ProjectionCrontabInstance>(
|
||||
'Ql3PrimaryCronProjectionCrontab',
|
||||
{
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true },
|
||||
status: { type: DataTypes.INTEGER },
|
||||
pid: { type: DataTypes.INTEGER },
|
||||
logPath: { field: 'log_path', type: DataTypes.STRING },
|
||||
lastRunningTime: {
|
||||
field: 'last_running_time',
|
||||
type: DataTypes.INTEGER,
|
||||
},
|
||||
lastExecutionTime: {
|
||||
field: 'last_execution_time',
|
||||
type: DataTypes.INTEGER,
|
||||
},
|
||||
},
|
||||
{ ...common, tableName: CRONTAB_TABLE },
|
||||
);
|
||||
const instance = database.define<ProjectionInstanceInstance>(
|
||||
'Ql3PrimaryCronProjectionInstance',
|
||||
{
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true },
|
||||
cronId: { field: 'cron_id', type: DataTypes.INTEGER },
|
||||
runId: { field: 'run_id', type: DataTypes.STRING(36) },
|
||||
attemptId: { field: 'attempt_id', type: DataTypes.STRING(36) },
|
||||
pid: { type: DataTypes.INTEGER },
|
||||
logPath: { field: 'log_path', type: DataTypes.STRING },
|
||||
startedAt: { field: 'started_at', type: DataTypes.INTEGER },
|
||||
finishedAt: { field: 'finished_at', type: DataTypes.INTEGER },
|
||||
status: { type: DataTypes.INTEGER },
|
||||
exitCode: { field: 'exit_code', type: DataTypes.INTEGER },
|
||||
},
|
||||
{ ...common, tableName: RUNNING_INSTANCE_TABLE },
|
||||
);
|
||||
return { run, attempt, crontab, instance };
|
||||
}
|
||||
|
||||
function toUnixSeconds(milliseconds: number): number {
|
||||
return Math.floor(milliseconds / 1000);
|
||||
}
|
||||
|
||||
function instanceStatus(status: RunAttemptStatus): number | null {
|
||||
switch (status) {
|
||||
case 'claimed':
|
||||
return null;
|
||||
case 'starting':
|
||||
case 'running':
|
||||
return INSTANCE_STATUS_RUNNING;
|
||||
case 'succeeded':
|
||||
return INSTANCE_STATUS_FINISHED;
|
||||
case 'cancelled':
|
||||
return INSTANCE_STATUS_STOPPED;
|
||||
case 'failed':
|
||||
case 'timed_out':
|
||||
case 'lost':
|
||||
return INSTANCE_STATUS_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
function isAttemptActive(status: RunAttemptStatus): boolean {
|
||||
return status === 'starting' || status === 'running';
|
||||
}
|
||||
|
||||
/**
|
||||
* Projects runtime-owned Run state into the legacy UI tables before the same
|
||||
* SQLite transaction commits. It never projects legacy-owned Shadow Runs.
|
||||
*/
|
||||
export class PrimaryCronProjection
|
||||
implements SequelizeRunProjectionParticipant
|
||||
{
|
||||
private readonly models: ProjectionModels;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.models = defineProjectionModels(database);
|
||||
}
|
||||
|
||||
async apply(context: SequelizeRunProjectionContext): Promise<void> {
|
||||
const cronIds = new Set<number>();
|
||||
for (const attemptId of context.changedAttemptIds) {
|
||||
const cronId = await this.projectAttempt(attemptId, context.transaction);
|
||||
if (cronId !== null) cronIds.add(cronId);
|
||||
}
|
||||
for (const runId of context.changedRunIds) {
|
||||
const run = await context.runs.findRunById(runId);
|
||||
if (run?.executionOwner === 'runtime' && run.legacyCronId !== undefined) {
|
||||
cronIds.add(run.legacyCronId);
|
||||
}
|
||||
}
|
||||
for (const cronId of cronIds) {
|
||||
await this.projectCrontab(cronId, context.transaction);
|
||||
}
|
||||
}
|
||||
|
||||
private async projectAttempt(
|
||||
attemptId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<number | null> {
|
||||
const attempt = await this.models.attempt.findByPk(attemptId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!attempt) return null;
|
||||
const run = await this.models.run.findByPk(attempt.runId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!run || run.executionOwner !== 'runtime' || run.legacyCronId === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const status = instanceStatus(attempt.status);
|
||||
if (status === null) return run.legacyCronId;
|
||||
const logPath = parseLegacyLogOutputRef(run.outputRef ?? undefined);
|
||||
const values: ProjectionInstanceRow = {
|
||||
cronId: run.legacyCronId,
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
pid: attempt.pid,
|
||||
logPath,
|
||||
startedAt: toUnixSeconds(
|
||||
attempt.startedAtMs ?? run.startedAtMs ?? attempt.createdAtMs,
|
||||
),
|
||||
finishedAt:
|
||||
attempt.finishedAtMs === null
|
||||
? null
|
||||
: toUnixSeconds(attempt.finishedAtMs),
|
||||
status,
|
||||
exitCode: attempt.exitCode,
|
||||
};
|
||||
const existing = await this.models.instance.findOne({
|
||||
where: { attemptId: attempt.id },
|
||||
transaction,
|
||||
});
|
||||
if (existing) {
|
||||
await existing.update(values, { transaction });
|
||||
} else {
|
||||
await this.models.instance.create(values, { transaction });
|
||||
}
|
||||
return run.legacyCronId;
|
||||
}
|
||||
|
||||
private async projectCrontab(
|
||||
cronId: number,
|
||||
transaction: Transaction,
|
||||
): Promise<void> {
|
||||
const running = await this.findSelectedRun(
|
||||
cronId,
|
||||
RUNNING_RUN_STATUSES,
|
||||
transaction,
|
||||
);
|
||||
if (running) {
|
||||
await this.updateCrontab(
|
||||
cronId,
|
||||
CRONTAB_STATUS_RUNNING,
|
||||
running,
|
||||
transaction,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const queued = await this.findSelectedRun(
|
||||
cronId,
|
||||
QUEUED_RUN_STATUSES,
|
||||
transaction,
|
||||
);
|
||||
if (queued) {
|
||||
await this.updateCrontab(
|
||||
cronId,
|
||||
CRONTAB_STATUS_QUEUED,
|
||||
queued,
|
||||
transaction,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const terminal = await this.findSelectedRun(
|
||||
cronId,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
transaction,
|
||||
);
|
||||
await this.updateCrontab(
|
||||
cronId,
|
||||
CRONTAB_STATUS_IDLE,
|
||||
terminal,
|
||||
transaction,
|
||||
);
|
||||
}
|
||||
|
||||
private async findSelectedRun(
|
||||
cronId: number,
|
||||
statuses: readonly RunStatus[],
|
||||
transaction: Transaction,
|
||||
): Promise<SelectedRun | null> {
|
||||
const run = await this.models.run.findOne({
|
||||
where: {
|
||||
legacyCronId: cronId,
|
||||
executionOwner: 'runtime',
|
||||
status: { [Op.in]: [...statuses] },
|
||||
},
|
||||
order: [
|
||||
['createdAtMs', 'DESC'],
|
||||
['id', 'DESC'],
|
||||
],
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!run) return null;
|
||||
const attempt = await this.models.attempt.findOne({
|
||||
where: { runId: run.id },
|
||||
order: [
|
||||
['attempt', 'DESC'],
|
||||
['id', 'DESC'],
|
||||
],
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async updateCrontab(
|
||||
cronId: number,
|
||||
status: number,
|
||||
selected: SelectedRun | null,
|
||||
transaction: Transaction,
|
||||
): Promise<void> {
|
||||
const run = selected?.run;
|
||||
const attempt = selected?.attempt;
|
||||
const startedAtMs = attempt?.startedAtMs ?? run?.startedAtMs ?? null;
|
||||
const finishedAtMs = attempt?.finishedAtMs ?? run?.finishedAtMs ?? null;
|
||||
const values: Partial<ProjectionCrontabRow> = {
|
||||
status,
|
||||
pid: attempt && isAttemptActive(attempt.status) ? attempt.pid : null,
|
||||
logPath: parseLegacyLogOutputRef(run?.outputRef ?? undefined),
|
||||
};
|
||||
if (startedAtMs !== null) {
|
||||
values.lastExecutionTime = toUnixSeconds(startedAtMs);
|
||||
}
|
||||
if (startedAtMs !== null && finishedAtMs !== null) {
|
||||
values.lastRunningTime = Math.max(
|
||||
0,
|
||||
Math.floor((finishedAtMs - startedAtMs) / 1000),
|
||||
);
|
||||
}
|
||||
await this.models.crontab.update(values, {
|
||||
where: { id: cronId },
|
||||
transaction,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { DataTypes, Model, ModelStatic, Sequelize } from 'sequelize';
|
||||
import { RUN_TABLE } from '../../../migrations/0002-run-schema';
|
||||
import type { PrimaryRunIdempotencyLookup } from '../../ports/primaryRunIdempotencyLookup';
|
||||
|
||||
interface IdempotentRunRow {
|
||||
id: string;
|
||||
projectId: string;
|
||||
idempotencyKey: string | null;
|
||||
}
|
||||
|
||||
interface IdempotentRunInstance
|
||||
extends Model<IdempotentRunRow, IdempotentRunRow>,
|
||||
IdempotentRunRow {}
|
||||
|
||||
function defineIdempotentRunModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<IdempotentRunInstance> {
|
||||
return database.define<IdempotentRunInstance>(
|
||||
'Ql3PrimaryIdempotentRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
},
|
||||
idempotencyKey: {
|
||||
field: 'idempotency_key',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: RUN_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export class LegacySequelizePrimaryRunIdempotencyLookup
|
||||
implements PrimaryRunIdempotencyLookup
|
||||
{
|
||||
private readonly run: ModelStatic<IdempotentRunInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.run = defineIdempotentRunModel(database);
|
||||
}
|
||||
|
||||
async findRunId(
|
||||
projectId: string,
|
||||
idempotencyKey: string,
|
||||
): Promise<string | null> {
|
||||
if (!projectId || projectId.length > 128) {
|
||||
throw new RangeError('projectId must be between 1 and 128 characters');
|
||||
}
|
||||
if (!idempotencyKey || idempotencyKey.length > 255) {
|
||||
throw new RangeError(
|
||||
'idempotencyKey must be between 1 and 255 characters',
|
||||
);
|
||||
}
|
||||
const row = await this.run.findOne({
|
||||
attributes: ['id'],
|
||||
where: { projectId, idempotencyKey },
|
||||
raw: true,
|
||||
});
|
||||
return row?.id ?? null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Op,
|
||||
Sequelize,
|
||||
type WhereOptions,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import type { ExecutorType } from '../../domain/execution';
|
||||
import type {
|
||||
PrimaryRunRecoveryAttemptReference,
|
||||
PrimaryRunRecoveryCandidate,
|
||||
PrimaryRunRecoveryCursor,
|
||||
PrimaryRunRecoveryPage,
|
||||
PrimaryRunRecoverySource,
|
||||
} from '../../ports/primaryRunRecoverySource';
|
||||
import { MAX_PRIMARY_RECOVERY_BATCH_SIZE } from '../../ports/primaryRunRecoverySource';
|
||||
|
||||
interface RecoveryRunRow {
|
||||
id: string;
|
||||
executionOwner: string;
|
||||
status: string;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
interface RecoveryAttemptRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
attempt: number;
|
||||
status: string;
|
||||
executorType: string;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
interface RecoveryRunInstance
|
||||
extends Model<RecoveryRunRow, RecoveryRunRow>,
|
||||
RecoveryRunRow {}
|
||||
interface RecoveryAttemptInstance
|
||||
extends Model<RecoveryAttemptRow, RecoveryAttemptRow>,
|
||||
RecoveryAttemptRow {}
|
||||
|
||||
function defineRecoveryRunModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<RecoveryRunInstance> {
|
||||
return database.define<RecoveryRunInstance>(
|
||||
'Ql3PrimaryRecoveryRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
executionOwner: {
|
||||
field: 'execution_owner',
|
||||
type: DataTypes.STRING(16),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function defineRecoveryAttemptModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<RecoveryAttemptInstance> {
|
||||
return database.define<RecoveryAttemptInstance>(
|
||||
'Ql3PrimaryRecoveryAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
runId: {
|
||||
field: 'run_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
attempt: { type: DataTypes.INTEGER, allowNull: false },
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
executorType: {
|
||||
field: 'executor_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: RUN_ATTEMPT_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function assertCursor(cursor: PrimaryRunRecoveryCursor): void {
|
||||
if (!Number.isSafeInteger(cursor.createdAtMs) || cursor.createdAtMs < 0) {
|
||||
throw new RangeError('cursor.createdAtMs must be a non-negative integer');
|
||||
}
|
||||
if (!cursor.runId || cursor.runId.length > 36) {
|
||||
throw new RangeError('cursor.runId must be between 1 and 36 characters');
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizePrimaryRunRecoverySource
|
||||
implements PrimaryRunRecoverySource
|
||||
{
|
||||
private readonly run: ModelStatic<RecoveryRunInstance>;
|
||||
private readonly attempt: ModelStatic<RecoveryAttemptInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.run = defineRecoveryRunModel(database);
|
||||
this.attempt = defineRecoveryAttemptModel(database);
|
||||
}
|
||||
|
||||
async listCandidates({
|
||||
cursor,
|
||||
limit = 32,
|
||||
}: {
|
||||
cursor?: PrimaryRunRecoveryCursor;
|
||||
limit?: number;
|
||||
} = {}): Promise<PrimaryRunRecoveryPage> {
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_PRIMARY_RECOVERY_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_PRIMARY_RECOVERY_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (cursor) assertCursor(cursor);
|
||||
|
||||
const where: WhereOptions<RecoveryRunRow> = {
|
||||
executionOwner: 'runtime',
|
||||
status: { [Op.in]: ['dispatching', 'running'] },
|
||||
...(cursor === undefined
|
||||
? {}
|
||||
: {
|
||||
[Op.or]: [
|
||||
{ createdAtMs: { [Op.gt]: cursor.createdAtMs } },
|
||||
{
|
||||
createdAtMs: cursor.createdAtMs,
|
||||
id: { [Op.gt]: cursor.runId },
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
const runRows = (await this.run.findAll({
|
||||
attributes: ['id', 'createdAtMs'],
|
||||
where,
|
||||
order: [
|
||||
['createdAtMs', 'ASC'],
|
||||
['id', 'ASC'],
|
||||
],
|
||||
limit: limit + 1,
|
||||
raw: true,
|
||||
})) as unknown as RecoveryRunRow[];
|
||||
const truncated = runRows.length > limit;
|
||||
const boundedRuns = runRows.slice(0, limit);
|
||||
if (boundedRuns.length === 0) {
|
||||
return {
|
||||
candidates: [],
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
};
|
||||
}
|
||||
|
||||
const maxAttemptRows = limit * 2;
|
||||
const attemptRows = (await this.attempt.findAll({
|
||||
attributes: ['id', 'runId', 'attempt', 'executorType'],
|
||||
where: {
|
||||
runId: { [Op.in]: boundedRuns.map((run) => run.id) },
|
||||
status: { [Op.in]: ['claimed', 'starting', 'running'] },
|
||||
},
|
||||
order: [
|
||||
['runId', 'ASC'],
|
||||
['attempt', 'DESC'],
|
||||
['createdAtMs', 'DESC'],
|
||||
['id', 'DESC'],
|
||||
],
|
||||
limit: maxAttemptRows + 1,
|
||||
raw: true,
|
||||
})) as unknown as RecoveryAttemptRow[];
|
||||
if (attemptRows.length > maxAttemptRows) {
|
||||
return {
|
||||
candidates: [],
|
||||
truncated,
|
||||
unsafeAttemptOverflow: true,
|
||||
};
|
||||
}
|
||||
|
||||
const attemptsByRun = new Map<
|
||||
string,
|
||||
PrimaryRunRecoveryAttemptReference[]
|
||||
>();
|
||||
for (const attempt of attemptRows) {
|
||||
const references = attemptsByRun.get(attempt.runId) ?? [];
|
||||
references.push({
|
||||
attemptId: attempt.id,
|
||||
executorType: attempt.executorType as ExecutorType,
|
||||
});
|
||||
attemptsByRun.set(attempt.runId, references);
|
||||
}
|
||||
const candidates: PrimaryRunRecoveryCandidate[] = boundedRuns.map(
|
||||
(run) => ({
|
||||
runId: run.id,
|
||||
attempts: attemptsByRun.get(run.id) ?? [],
|
||||
}),
|
||||
);
|
||||
const last = boundedRuns[boundedRuns.length - 1];
|
||||
return {
|
||||
candidates,
|
||||
truncated,
|
||||
unsafeAttemptOverflow: false,
|
||||
nextCursor: { createdAtMs: last.createdAtMs, runId: last.id },
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Op,
|
||||
Sequelize,
|
||||
type WhereOptions,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import {
|
||||
MAX_PRIMARY_TIMEOUT_BATCH_SIZE,
|
||||
type PrimaryTimeoutCursor,
|
||||
type PrimaryTimeoutPage,
|
||||
type PrimaryTimeoutSource,
|
||||
} from '../../ports/primaryTimeoutSource';
|
||||
|
||||
interface TimeoutAttemptRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
status: string;
|
||||
deadlineAtMs: number | null;
|
||||
}
|
||||
|
||||
interface TimeoutRunRow {
|
||||
id: string;
|
||||
executionOwner: string;
|
||||
status: string;
|
||||
cancelRequestedAtMs: number | null;
|
||||
}
|
||||
|
||||
interface TimeoutAttemptInstance
|
||||
extends Model<TimeoutAttemptRow, TimeoutAttemptRow>,
|
||||
TimeoutAttemptRow {}
|
||||
interface TimeoutRunInstance
|
||||
extends Model<TimeoutRunRow, TimeoutRunRow>,
|
||||
TimeoutRunRow {}
|
||||
|
||||
function defineTimeoutAttemptModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<TimeoutAttemptInstance> {
|
||||
return database.define<TimeoutAttemptInstance>(
|
||||
'Ql3PrimaryTimeoutAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
runId: {
|
||||
field: 'run_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
deadlineAtMs: {
|
||||
field: 'deadline_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_ATTEMPT_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function defineTimeoutRunModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<TimeoutRunInstance> {
|
||||
return database.define<TimeoutRunInstance>(
|
||||
'Ql3PrimaryTimeoutRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
executionOwner: {
|
||||
field: 'execution_owner',
|
||||
type: DataTypes.STRING(16),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
cancelRequestedAtMs: {
|
||||
field: 'cancel_requested_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function assertCursor(cursor: PrimaryTimeoutCursor): void {
|
||||
if (!Number.isSafeInteger(cursor.deadlineAtMs) || cursor.deadlineAtMs < 0) {
|
||||
throw new RangeError('cursor.deadlineAtMs must be a non-negative integer');
|
||||
}
|
||||
if (!cursor.attemptId || cursor.attemptId.length > 36) {
|
||||
throw new RangeError(
|
||||
'cursor.attemptId must be between 1 and 36 characters',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizePrimaryTimeoutSource
|
||||
implements PrimaryTimeoutSource
|
||||
{
|
||||
private readonly attempt: ModelStatic<TimeoutAttemptInstance>;
|
||||
private readonly run: ModelStatic<TimeoutRunInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.attempt = defineTimeoutAttemptModel(database);
|
||||
this.run = defineTimeoutRunModel(database);
|
||||
this.attempt.belongsTo(this.run, {
|
||||
as: 'timeoutRun',
|
||||
foreignKey: 'runId',
|
||||
targetKey: 'id',
|
||||
constraints: false,
|
||||
});
|
||||
}
|
||||
|
||||
async listOverdue(options: {
|
||||
nowMs: number;
|
||||
cursor?: PrimaryTimeoutCursor;
|
||||
limit?: number;
|
||||
}): Promise<PrimaryTimeoutPage> {
|
||||
if (!Number.isSafeInteger(options.nowMs) || options.nowMs < 0) {
|
||||
throw new RangeError('nowMs must be a non-negative safe integer');
|
||||
}
|
||||
const limit = options.limit ?? 32;
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_PRIMARY_TIMEOUT_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_PRIMARY_TIMEOUT_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (options.cursor) assertCursor(options.cursor);
|
||||
|
||||
const cursorWhere: WhereOptions<TimeoutAttemptRow> = options.cursor
|
||||
? {
|
||||
[Op.or]: [
|
||||
{ deadlineAtMs: { [Op.gt]: options.cursor.deadlineAtMs } },
|
||||
{
|
||||
deadlineAtMs: options.cursor.deadlineAtMs,
|
||||
id: { [Op.gt]: options.cursor.attemptId },
|
||||
},
|
||||
],
|
||||
}
|
||||
: {};
|
||||
const rows = (await this.attempt.findAll({
|
||||
attributes: ['id', 'runId', 'status', 'deadlineAtMs'],
|
||||
where: {
|
||||
status: { [Op.in]: ['starting', 'running'] },
|
||||
deadlineAtMs: { [Op.ne]: null, [Op.lte]: options.nowMs },
|
||||
...cursorWhere,
|
||||
},
|
||||
include: [
|
||||
{
|
||||
model: this.run,
|
||||
as: 'timeoutRun',
|
||||
attributes: [],
|
||||
required: true,
|
||||
where: {
|
||||
executionOwner: 'runtime',
|
||||
status: { [Op.in]: ['dispatching', 'running'] },
|
||||
cancelRequestedAtMs: null,
|
||||
},
|
||||
},
|
||||
],
|
||||
order: [
|
||||
['deadlineAtMs', 'ASC'],
|
||||
['id', 'ASC'],
|
||||
],
|
||||
limit: limit + 1,
|
||||
raw: true,
|
||||
})) as unknown as TimeoutAttemptRow[];
|
||||
|
||||
const truncated = rows.length > limit;
|
||||
const selected = rows.slice(0, limit);
|
||||
const candidates = selected.map((row) => ({
|
||||
runId: row.runId,
|
||||
attemptId: row.id,
|
||||
deadlineAtMs: Number(row.deadlineAtMs),
|
||||
}));
|
||||
const last = candidates[candidates.length - 1];
|
||||
return {
|
||||
candidates,
|
||||
truncated,
|
||||
...(truncated && last
|
||||
? {
|
||||
nextCursor: {
|
||||
deadlineAtMs: last.deadlineAtMs,
|
||||
attemptId: last.attemptId,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,522 @@
|
||||
import { timingSafeEqual } from 'crypto';
|
||||
import {
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
PROJECT_ROLE_BINDING_TABLE,
|
||||
PROJECT_TABLE,
|
||||
} from '../../../migrations/0017-project-policy';
|
||||
import { PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE } from '../../../migrations/0018-project-owner-bootstrap';
|
||||
import {
|
||||
assertProjectPolicyProjectId,
|
||||
normalizePolicySubject,
|
||||
normalizeProjectRoleBindingRecord,
|
||||
type ProjectRoleBindingRecord,
|
||||
} from '../../domain/projectPolicy';
|
||||
import {
|
||||
OWNER_BOOTSTRAP_MAX_VERSION,
|
||||
OWNER_BOOTSTRAP_SYSTEM_SUBJECT,
|
||||
ProjectOwnerBootstrapChallengeActiveError,
|
||||
ProjectOwnerBootstrapClaimRejectedError,
|
||||
ProjectOwnerBootstrapProjectInactiveError,
|
||||
ProjectOwnerBootstrapProjectNotFoundError,
|
||||
ProjectOwnerBootstrapProjectNotPristineError,
|
||||
ProjectOwnerBootstrapUnavailableError,
|
||||
assertProjectOwnerBootstrapChallengeId,
|
||||
assertProjectOwnerBootstrapTokenDigest,
|
||||
normalizeProjectOwnerBootstrapChallengeRecord,
|
||||
type ProjectOwnerBootstrapChallengeRecord,
|
||||
} from '../../domain/projectOwnerBootstrap';
|
||||
import type {
|
||||
ClaimProjectOwnerBootstrapChallengeCommand,
|
||||
ClaimProjectOwnerBootstrapChallengeResult,
|
||||
IssueProjectOwnerBootstrapChallengeCommand,
|
||||
ProjectOwnerBootstrapRepository,
|
||||
} from '../../ports/projectOwnerBootstrapRepository';
|
||||
|
||||
const RETRY_ATTEMPTS = 5;
|
||||
|
||||
interface ProjectStatusRow {
|
||||
status: string;
|
||||
}
|
||||
|
||||
interface BootstrapChallengeRow {
|
||||
project_id: string;
|
||||
version: number;
|
||||
challenge_id: string;
|
||||
token_digest: string;
|
||||
issued_at_ms: number | string;
|
||||
expires_at_ms: number | string;
|
||||
consumed_at_ms: number | string | null;
|
||||
claimed_subject_type: string | null;
|
||||
claimed_subject_id: string | null;
|
||||
}
|
||||
|
||||
interface BootstrapBindingRow {
|
||||
project_id: string;
|
||||
subject_type: string;
|
||||
subject_id: string;
|
||||
version: number;
|
||||
state: string;
|
||||
role: string | null;
|
||||
mutation_id: string;
|
||||
changed_by_type: string;
|
||||
changed_by_id: string;
|
||||
created_at_ms: number | string;
|
||||
}
|
||||
|
||||
function assertExactKeys(value: object, expected: readonly string[]): void {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
keys.length !== canonical.length ||
|
||||
keys.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
throw new TypeError('Project owner bootstrap command shape is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function assertTimestamp(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError(`Project owner bootstrap ${name} is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeIssueCommand(
|
||||
command: IssueProjectOwnerBootstrapChallengeCommand,
|
||||
): Readonly<IssueProjectOwnerBootstrapChallengeCommand> {
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Project owner bootstrap issue command is invalid');
|
||||
}
|
||||
assertExactKeys(command, [
|
||||
'projectId',
|
||||
'challengeId',
|
||||
'tokenDigest',
|
||||
'issuedAtMs',
|
||||
'expiresAtMs',
|
||||
]);
|
||||
assertProjectPolicyProjectId(command.projectId);
|
||||
assertProjectOwnerBootstrapChallengeId(command.challengeId);
|
||||
assertProjectOwnerBootstrapTokenDigest(command.tokenDigest);
|
||||
assertTimestamp('issuedAtMs', command.issuedAtMs);
|
||||
assertTimestamp('expiresAtMs', command.expiresAtMs);
|
||||
if (command.expiresAtMs <= command.issuedAtMs) {
|
||||
throw new TypeError('Project owner bootstrap lifetime is invalid');
|
||||
}
|
||||
return Object.freeze({ ...command });
|
||||
}
|
||||
|
||||
function normalizeClaimCommand(
|
||||
command: ClaimProjectOwnerBootstrapChallengeCommand,
|
||||
): Readonly<ClaimProjectOwnerBootstrapChallengeCommand> {
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Project owner bootstrap claim command is invalid');
|
||||
}
|
||||
assertExactKeys(command, [
|
||||
'projectId',
|
||||
'challengeId',
|
||||
'tokenDigest',
|
||||
'subject',
|
||||
'claimedAtMs',
|
||||
]);
|
||||
assertProjectPolicyProjectId(command.projectId);
|
||||
assertProjectOwnerBootstrapChallengeId(command.challengeId);
|
||||
assertProjectOwnerBootstrapTokenDigest(command.tokenDigest);
|
||||
const subject = normalizePolicySubject(command.subject);
|
||||
if (subject.type !== 'user') {
|
||||
throw new ProjectOwnerBootstrapClaimRejectedError();
|
||||
}
|
||||
assertTimestamp('claimedAtMs', command.claimedAtMs);
|
||||
return Object.freeze({ ...command, subject });
|
||||
}
|
||||
|
||||
function rowToChallenge(
|
||||
row: BootstrapChallengeRow,
|
||||
): Readonly<ProjectOwnerBootstrapChallengeRecord> {
|
||||
return normalizeProjectOwnerBootstrapChallengeRecord({
|
||||
projectId: row.project_id,
|
||||
version: Number(row.version),
|
||||
challengeId: row.challenge_id,
|
||||
tokenDigest: row.token_digest,
|
||||
issuedAtMs: Number(row.issued_at_ms),
|
||||
expiresAtMs: Number(row.expires_at_ms),
|
||||
...(row.consumed_at_ms === null
|
||||
? {}
|
||||
: {
|
||||
consumedAtMs: Number(row.consumed_at_ms),
|
||||
claimedSubject: {
|
||||
type: row.claimed_subject_type as NonNullable<
|
||||
ProjectOwnerBootstrapChallengeRecord['claimedSubject']
|
||||
>['type'],
|
||||
id: row.claimed_subject_id!,
|
||||
},
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function rowToBinding(
|
||||
row: BootstrapBindingRow,
|
||||
): Readonly<ProjectRoleBindingRecord> {
|
||||
return normalizeProjectRoleBindingRecord({
|
||||
projectId: row.project_id,
|
||||
subject: {
|
||||
type: row.subject_type as ProjectRoleBindingRecord['subject']['type'],
|
||||
id: row.subject_id,
|
||||
},
|
||||
version: Number(row.version),
|
||||
state: row.state as ProjectRoleBindingRecord['state'],
|
||||
...(row.role === null
|
||||
? {}
|
||||
: { role: row.role as NonNullable<ProjectRoleBindingRecord['role']> }),
|
||||
mutationId: row.mutation_id,
|
||||
changedBy: {
|
||||
type: row.changed_by_type as ProjectRoleBindingRecord['changedBy']['type'],
|
||||
id: row.changed_by_id,
|
||||
},
|
||||
createdAtMs: Number(row.created_at_ms),
|
||||
});
|
||||
}
|
||||
|
||||
function digestMatches(expected: string, actual: string): boolean {
|
||||
assertProjectOwnerBootstrapTokenDigest(expected);
|
||||
assertProjectOwnerBootstrapTokenDigest(actual);
|
||||
return timingSafeEqual(
|
||||
Buffer.from(expected, 'hex'),
|
||||
Buffer.from(actual, 'hex'),
|
||||
);
|
||||
}
|
||||
|
||||
function mutationId(challengeId: string): string {
|
||||
return `owner-bootstrap:${challengeId}`;
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
function isExpectedError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof ProjectOwnerBootstrapChallengeActiveError ||
|
||||
error instanceof ProjectOwnerBootstrapClaimRejectedError ||
|
||||
error instanceof ProjectOwnerBootstrapProjectInactiveError ||
|
||||
error instanceof ProjectOwnerBootstrapProjectNotFoundError ||
|
||||
error instanceof ProjectOwnerBootstrapProjectNotPristineError ||
|
||||
error instanceof ProjectOwnerBootstrapUnavailableError
|
||||
);
|
||||
}
|
||||
|
||||
export class LegacySequelizeProjectOwnerBootstrapRepository
|
||||
implements ProjectOwnerBootstrapRepository
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Project owner bootstrap repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async issue(
|
||||
rawCommand: IssueProjectOwnerBootstrapChallengeCommand,
|
||||
): Promise<Readonly<ProjectOwnerBootstrapChallengeRecord>> {
|
||||
const command = normalizeIssueCommand(rawCommand);
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
await this.assertActiveProject(command.projectId, transaction);
|
||||
if ((await this.bindingCount(command.projectId, transaction)) > 0) {
|
||||
throw new ProjectOwnerBootstrapProjectNotPristineError();
|
||||
}
|
||||
const latest = await this.latestChallenge(
|
||||
command.projectId,
|
||||
transaction,
|
||||
);
|
||||
if (
|
||||
latest?.consumedAtMs !== undefined ||
|
||||
(latest && latest.expiresAtMs > command.issuedAtMs)
|
||||
) {
|
||||
if (latest?.consumedAtMs !== undefined) {
|
||||
throw new ProjectOwnerBootstrapProjectNotPristineError();
|
||||
}
|
||||
throw new ProjectOwnerBootstrapChallengeActiveError();
|
||||
}
|
||||
const version = (latest?.version ?? 0) + 1;
|
||||
if (version > OWNER_BOOTSTRAP_MAX_VERSION) {
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
await this.database.query(
|
||||
`INSERT INTO "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}"
|
||||
(project_id, version, challenge_id, token_digest,
|
||||
issued_at_ms, expires_at_ms, consumed_at_ms,
|
||||
claimed_subject_type, claimed_subject_id)
|
||||
VALUES
|
||||
(:projectId, :version, :challengeId, :tokenDigest,
|
||||
:issuedAtMs, :expiresAtMs, NULL, NULL, NULL)`,
|
||||
{
|
||||
type: QueryTypes.INSERT,
|
||||
replacements: { ...command, version },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
return normalizeProjectOwnerBootstrapChallengeRecord({
|
||||
...command,
|
||||
version,
|
||||
});
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (isExpectedError(error)) throw error;
|
||||
if (
|
||||
errorCode(error) === 'SQLITE_BUSY' &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
if (error instanceof UniqueConstraintError) {
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
}
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
|
||||
async claim(
|
||||
rawCommand: ClaimProjectOwnerBootstrapChallengeCommand,
|
||||
): Promise<ClaimProjectOwnerBootstrapChallengeResult> {
|
||||
const command = normalizeClaimCommand(rawCommand);
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
await this.assertActiveProject(command.projectId, transaction);
|
||||
const latest = await this.latestChallenge(
|
||||
command.projectId,
|
||||
transaction,
|
||||
);
|
||||
if (
|
||||
!latest ||
|
||||
latest.challengeId !== command.challengeId ||
|
||||
!digestMatches(latest.tokenDigest, command.tokenDigest)
|
||||
) {
|
||||
throw new ProjectOwnerBootstrapClaimRejectedError();
|
||||
}
|
||||
if (latest.consumedAtMs !== undefined) {
|
||||
if (
|
||||
latest.claimedSubject?.type !== command.subject.type ||
|
||||
latest.claimedSubject.id !== command.subject.id
|
||||
) {
|
||||
throw new ProjectOwnerBootstrapClaimRejectedError();
|
||||
}
|
||||
const binding = await this.bootstrapBinding(
|
||||
command.projectId,
|
||||
latest.challengeId,
|
||||
transaction,
|
||||
);
|
||||
if (
|
||||
!binding ||
|
||||
binding.subject.type !== command.subject.type ||
|
||||
binding.subject.id !== command.subject.id ||
|
||||
binding.role !== 'owner' ||
|
||||
binding.state !== 'active'
|
||||
) {
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
return { status: 'existing', binding };
|
||||
}
|
||||
if (
|
||||
command.claimedAtMs < latest.issuedAtMs ||
|
||||
command.claimedAtMs >= latest.expiresAtMs
|
||||
) {
|
||||
throw new ProjectOwnerBootstrapClaimRejectedError();
|
||||
}
|
||||
if ((await this.bindingCount(command.projectId, transaction)) > 0) {
|
||||
throw new ProjectOwnerBootstrapProjectNotPristineError();
|
||||
}
|
||||
const binding = normalizeProjectRoleBindingRecord({
|
||||
projectId: command.projectId,
|
||||
subject: command.subject,
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role: 'owner',
|
||||
mutationId: mutationId(latest.challengeId),
|
||||
changedBy: OWNER_BOOTSTRAP_SYSTEM_SUBJECT,
|
||||
createdAtMs: command.claimedAtMs,
|
||||
});
|
||||
const [, consumedCount] = await this.database.query(
|
||||
`UPDATE "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}"
|
||||
SET consumed_at_ms = :claimedAtMs,
|
||||
claimed_subject_type = :subjectType,
|
||||
claimed_subject_id = :subjectId
|
||||
WHERE project_id = :projectId
|
||||
AND version = :version
|
||||
AND challenge_id = :challengeId
|
||||
AND consumed_at_ms IS NULL`,
|
||||
{
|
||||
type: QueryTypes.UPDATE,
|
||||
replacements: {
|
||||
projectId: command.projectId,
|
||||
version: latest.version,
|
||||
challengeId: latest.challengeId,
|
||||
claimedAtMs: command.claimedAtMs,
|
||||
subjectType: command.subject.type,
|
||||
subjectId: command.subject.id,
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (consumedCount !== 1) {
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
await this.database.query(
|
||||
`INSERT INTO "${PROJECT_ROLE_BINDING_TABLE}"
|
||||
(project_id, subject_type, subject_id, version, state, role,
|
||||
mutation_id, changed_by_type, changed_by_id, created_at_ms)
|
||||
VALUES
|
||||
(:projectId, :subjectType, :subjectId, 1, 'active', 'owner',
|
||||
:mutationId, :changedByType, :changedById, :createdAtMs)`,
|
||||
{
|
||||
type: QueryTypes.INSERT,
|
||||
replacements: {
|
||||
projectId: binding.projectId,
|
||||
subjectType: binding.subject.type,
|
||||
subjectId: binding.subject.id,
|
||||
mutationId: binding.mutationId,
|
||||
changedByType: binding.changedBy.type,
|
||||
changedById: binding.changedBy.id,
|
||||
createdAtMs: binding.createdAtMs,
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
return { status: 'claimed', binding };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (isExpectedError(error)) throw error;
|
||||
if (
|
||||
errorCode(error) === 'SQLITE_BUSY' &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
}
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
|
||||
private async assertActiveProject(
|
||||
projectId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<void> {
|
||||
const projects = await this.database.query<ProjectStatusRow>(
|
||||
`SELECT status FROM "${PROJECT_TABLE}" WHERE id = :projectId LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (projects.length === 0) {
|
||||
throw new ProjectOwnerBootstrapProjectNotFoundError();
|
||||
}
|
||||
if (projects.length !== 1 || projects[0].status !== 'active') {
|
||||
throw new ProjectOwnerBootstrapProjectInactiveError();
|
||||
}
|
||||
}
|
||||
|
||||
private async bindingCount(
|
||||
projectId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<number> {
|
||||
const rows = await this.database.query<{ count: number | string }>(
|
||||
`SELECT COUNT(*) AS count
|
||||
FROM "${PROJECT_ROLE_BINDING_TABLE}"
|
||||
WHERE project_id = :projectId`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
const count = Number(rows[0]?.count);
|
||||
if (!Number.isSafeInteger(count) || count < 0) {
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private async latestChallenge(
|
||||
projectId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<Readonly<ProjectOwnerBootstrapChallengeRecord> | null> {
|
||||
const rows = await this.database.query<BootstrapChallengeRow>(
|
||||
`SELECT project_id, version, challenge_id, token_digest,
|
||||
issued_at_ms, expires_at_ms, consumed_at_ms,
|
||||
claimed_subject_type, claimed_subject_id
|
||||
FROM "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}"
|
||||
WHERE project_id = :projectId
|
||||
ORDER BY version DESC
|
||||
LIMIT 1`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length !== 1) throw new ProjectOwnerBootstrapUnavailableError();
|
||||
return rowToChallenge(rows[0]);
|
||||
}
|
||||
|
||||
private async bootstrapBinding(
|
||||
projectId: string,
|
||||
challengeId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<Readonly<ProjectRoleBindingRecord> | null> {
|
||||
const rows = await this.database.query<BootstrapBindingRow>(
|
||||
`SELECT project_id, subject_type, subject_id, version, state, role,
|
||||
mutation_id, changed_by_type, changed_by_id, created_at_ms
|
||||
FROM "${PROJECT_ROLE_BINDING_TABLE}"
|
||||
WHERE project_id = :projectId
|
||||
AND mutation_id = :mutationId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId, mutationId: mutationId(challengeId) },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length !== 1) throw new ProjectOwnerBootstrapUnavailableError();
|
||||
return rowToBinding(rows[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
PROJECT_ROLE_BINDING_TABLE,
|
||||
PROJECT_TABLE,
|
||||
} from '../../../migrations/0017-project-policy';
|
||||
import {
|
||||
MAX_PROJECT_ROLE_BINDING_VERSION,
|
||||
ProjectPolicyProjectNotFoundError,
|
||||
ProjectPolicyUnavailableError,
|
||||
ProjectRoleBindingMutationConflictError,
|
||||
ProjectRoleBindingVersionConflictError,
|
||||
assertProjectPolicyProjectId,
|
||||
normalizePolicySubject,
|
||||
normalizeProjectPolicySnapshot,
|
||||
normalizeProjectRoleBindingRecord,
|
||||
type ProjectPolicySnapshot,
|
||||
type ProjectRoleBindingRecord,
|
||||
} from '../../domain/projectPolicy';
|
||||
import type {
|
||||
AppendProjectRoleBindingCommand,
|
||||
AppendProjectRoleBindingResult,
|
||||
ProjectPolicyRepository,
|
||||
} from '../../ports/projectPolicyRepository';
|
||||
|
||||
const RETRY_ATTEMPTS = 5;
|
||||
|
||||
interface ProjectRoleBindingRow {
|
||||
projectId: string;
|
||||
subjectType: string;
|
||||
subjectId: string;
|
||||
version: number;
|
||||
state: string;
|
||||
role: string | null;
|
||||
mutationId: string;
|
||||
changedByType: string;
|
||||
changedById: string;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface ProjectRoleBindingInstance
|
||||
extends Model<ProjectRoleBindingRow, ProjectRoleBindingRow>,
|
||||
ProjectRoleBindingRow {}
|
||||
|
||||
interface ProjectPolicySnapshotRow {
|
||||
project_id: string;
|
||||
project_name: string;
|
||||
project_slug: string;
|
||||
project_status: string;
|
||||
project_version: number;
|
||||
project_created_at_ms: number | string;
|
||||
project_updated_at_ms: number | string;
|
||||
binding_project_id: string | null;
|
||||
binding_subject_type: string | null;
|
||||
binding_subject_id: string | null;
|
||||
binding_version: number | null;
|
||||
binding_state: string | null;
|
||||
binding_role: string | null;
|
||||
binding_mutation_id: string | null;
|
||||
binding_changed_by_type: string | null;
|
||||
binding_changed_by_id: string | null;
|
||||
binding_created_at_ms: number | string | null;
|
||||
}
|
||||
|
||||
function defineBindingModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<ProjectRoleBindingInstance> {
|
||||
return database.define<ProjectRoleBindingInstance>(
|
||||
'Ql3ProjectRoleBinding',
|
||||
{
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
subjectType: {
|
||||
field: 'subject_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
subjectId: {
|
||||
field: 'subject_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
version: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
state: { type: DataTypes.STRING(16), allowNull: false },
|
||||
role: { type: DataTypes.STRING(16), allowNull: true },
|
||||
mutationId: {
|
||||
field: 'mutation_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
changedByType: {
|
||||
field: 'changed_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
changedById: {
|
||||
field: 'changed_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: PROJECT_ROLE_BINDING_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function rowToBinding(
|
||||
row: ProjectRoleBindingRow,
|
||||
): Readonly<ProjectRoleBindingRecord> {
|
||||
try {
|
||||
return normalizeProjectRoleBindingRecord({
|
||||
projectId: row.projectId,
|
||||
subject: {
|
||||
type: row.subjectType as ProjectRoleBindingRecord['subject']['type'],
|
||||
id: row.subjectId,
|
||||
},
|
||||
version: Number(row.version),
|
||||
state: row.state as ProjectRoleBindingRecord['state'],
|
||||
...(row.role === null
|
||||
? {}
|
||||
: { role: row.role as NonNullable<ProjectRoleBindingRecord['role']> }),
|
||||
mutationId: row.mutationId,
|
||||
changedBy: {
|
||||
type: row.changedByType as ProjectRoleBindingRecord['changedBy']['type'],
|
||||
id: row.changedById,
|
||||
},
|
||||
createdAtMs: Number(row.createdAtMs),
|
||||
});
|
||||
} catch {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function snapshotRowToValue(
|
||||
row: ProjectPolicySnapshotRow,
|
||||
): Readonly<ProjectPolicySnapshot> {
|
||||
const bindingFields = [
|
||||
row.binding_project_id,
|
||||
row.binding_subject_type,
|
||||
row.binding_subject_id,
|
||||
row.binding_version,
|
||||
row.binding_state,
|
||||
row.binding_mutation_id,
|
||||
row.binding_changed_by_type,
|
||||
row.binding_changed_by_id,
|
||||
row.binding_created_at_ms,
|
||||
];
|
||||
const noBinding = bindingFields.every((value) => value === null);
|
||||
if (!noBinding && bindingFields.some((value) => value === null)) {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
try {
|
||||
return normalizeProjectPolicySnapshot({
|
||||
project: {
|
||||
id: row.project_id,
|
||||
name: row.project_name,
|
||||
slug: row.project_slug,
|
||||
status:
|
||||
row.project_status as ProjectPolicySnapshot['project']['status'],
|
||||
version: Number(row.project_version),
|
||||
createdAtMs: Number(row.project_created_at_ms),
|
||||
updatedAtMs: Number(row.project_updated_at_ms),
|
||||
},
|
||||
...(noBinding
|
||||
? {}
|
||||
: {
|
||||
binding: {
|
||||
projectId: row.binding_project_id!,
|
||||
subject: {
|
||||
type: row.binding_subject_type as ProjectRoleBindingRecord['subject']['type'],
|
||||
id: row.binding_subject_id!,
|
||||
},
|
||||
version: Number(row.binding_version),
|
||||
state: row.binding_state as ProjectRoleBindingRecord['state'],
|
||||
...(row.binding_role === null
|
||||
? {}
|
||||
: {
|
||||
role: row.binding_role as NonNullable<
|
||||
ProjectRoleBindingRecord['role']
|
||||
>,
|
||||
}),
|
||||
mutationId: row.binding_mutation_id!,
|
||||
changedBy: {
|
||||
type: row.binding_changed_by_type as ProjectRoleBindingRecord['changedBy']['type'],
|
||||
id: row.binding_changed_by_id!,
|
||||
},
|
||||
createdAtMs: Number(row.binding_created_at_ms),
|
||||
},
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof ProjectPolicyUnavailableError) throw error;
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function sameBinding(
|
||||
left: Readonly<ProjectRoleBindingRecord>,
|
||||
right: Readonly<ProjectRoleBindingRecord>,
|
||||
): boolean {
|
||||
return (
|
||||
left.projectId === right.projectId &&
|
||||
left.subject.type === right.subject.type &&
|
||||
left.subject.id === right.subject.id &&
|
||||
left.version === right.version &&
|
||||
left.state === right.state &&
|
||||
left.role === right.role &&
|
||||
left.mutationId === right.mutationId &&
|
||||
left.changedBy.type === right.changedBy.type &&
|
||||
left.changedBy.id === right.changedBy.id &&
|
||||
left.createdAtMs === right.createdAtMs
|
||||
);
|
||||
}
|
||||
|
||||
function assertExpectedVersion(value: number): void {
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < 0 ||
|
||||
value >= MAX_PROJECT_ROLE_BINDING_VERSION
|
||||
) {
|
||||
throw new TypeError('Project role binding expected version is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
export class LegacySequelizeProjectPolicyRepository
|
||||
implements ProjectPolicyRepository
|
||||
{
|
||||
private readonly bindings: ModelStatic<ProjectRoleBindingInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Project policy repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.bindings = defineBindingModel(database);
|
||||
}
|
||||
|
||||
async resolve(
|
||||
projectId: string,
|
||||
requestedSubject: Parameters<ProjectPolicyRepository['resolve']>[1],
|
||||
): Promise<Readonly<ProjectPolicySnapshot> | null> {
|
||||
assertProjectPolicyProjectId(projectId);
|
||||
const subject = normalizePolicySubject(requestedSubject);
|
||||
const rows = await this.database.query<ProjectPolicySnapshotRow>(
|
||||
`SELECT project.id AS project_id,
|
||||
project.name AS project_name,
|
||||
project.slug AS project_slug,
|
||||
project.status AS project_status,
|
||||
project.version AS project_version,
|
||||
project.created_at_ms AS project_created_at_ms,
|
||||
project.updated_at_ms AS project_updated_at_ms,
|
||||
binding.project_id AS binding_project_id,
|
||||
binding.subject_type AS binding_subject_type,
|
||||
binding.subject_id AS binding_subject_id,
|
||||
binding.version AS binding_version,
|
||||
binding.state AS binding_state,
|
||||
binding.role AS binding_role,
|
||||
binding.mutation_id AS binding_mutation_id,
|
||||
binding.changed_by_type AS binding_changed_by_type,
|
||||
binding.changed_by_id AS binding_changed_by_id,
|
||||
binding.created_at_ms AS binding_created_at_ms
|
||||
FROM "${PROJECT_TABLE}" AS project
|
||||
LEFT JOIN "${PROJECT_ROLE_BINDING_TABLE}" AS binding
|
||||
ON binding.project_id = project.id
|
||||
AND binding.subject_type = :subjectType
|
||||
AND binding.subject_id = :subjectId
|
||||
AND binding.version = (
|
||||
SELECT MAX(current.version)
|
||||
FROM "${PROJECT_ROLE_BINDING_TABLE}" AS current
|
||||
WHERE current.project_id = project.id
|
||||
AND current.subject_type = :subjectType
|
||||
AND current.subject_id = :subjectId
|
||||
)
|
||||
WHERE project.id = :projectId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
projectId,
|
||||
subjectType: subject.type,
|
||||
subjectId: subject.id,
|
||||
},
|
||||
},
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length !== 1) throw new ProjectPolicyUnavailableError();
|
||||
return snapshotRowToValue(rows[0]);
|
||||
}
|
||||
|
||||
async append(
|
||||
command: AppendProjectRoleBindingCommand,
|
||||
): Promise<AppendProjectRoleBindingResult> {
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Project role binding command must be an object');
|
||||
}
|
||||
assertExpectedVersion(command.expectedCurrentVersion);
|
||||
const binding = normalizeProjectRoleBindingRecord(command.binding);
|
||||
if (binding.version !== command.expectedCurrentVersion + 1) {
|
||||
throw new ProjectRoleBindingVersionConflictError();
|
||||
}
|
||||
const values: ProjectRoleBindingRow = {
|
||||
projectId: binding.projectId,
|
||||
subjectType: binding.subject.type,
|
||||
subjectId: binding.subject.id,
|
||||
version: binding.version,
|
||||
state: binding.state,
|
||||
role: binding.role ?? null,
|
||||
mutationId: binding.mutationId,
|
||||
changedByType: binding.changedBy.type,
|
||||
changedById: binding.changedBy.id,
|
||||
createdAtMs: binding.createdAtMs,
|
||||
};
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const replay = await this.bindings.findOne({
|
||||
where: {
|
||||
projectId: binding.projectId,
|
||||
mutationId: binding.mutationId,
|
||||
},
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (replay) {
|
||||
const previous = rowToBinding(replay);
|
||||
if (!sameBinding(previous, binding)) {
|
||||
throw new ProjectRoleBindingMutationConflictError();
|
||||
}
|
||||
return { status: 'existing', binding: previous };
|
||||
}
|
||||
const projects = await this.database.query<{ id: string }>(
|
||||
`SELECT id FROM "${PROJECT_TABLE}" WHERE id = :projectId LIMIT 1`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId: binding.projectId },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (projects.length !== 1) {
|
||||
throw new ProjectPolicyProjectNotFoundError();
|
||||
}
|
||||
const current = await this.bindings.findOne({
|
||||
where: {
|
||||
projectId: binding.projectId,
|
||||
subjectType: binding.subject.type,
|
||||
subjectId: binding.subject.id,
|
||||
},
|
||||
order: [['version', 'DESC']],
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
const currentVersion = current ? Number(current.version) : 0;
|
||||
if (currentVersion !== command.expectedCurrentVersion) {
|
||||
throw new ProjectRoleBindingVersionConflictError();
|
||||
}
|
||||
await this.bindings.create(values, { transaction });
|
||||
return { status: 'inserted', binding };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof ProjectRoleBindingVersionConflictError ||
|
||||
error instanceof ProjectRoleBindingMutationConflictError ||
|
||||
error instanceof ProjectPolicyProjectNotFoundError ||
|
||||
error instanceof ProjectPolicyUnavailableError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { Sequelize, Transaction } from 'sequelize';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunAttemptStatus,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '../../domain/run';
|
||||
import type { RunRetryPolicyRecord } from '../../domain/runRetryPolicy';
|
||||
import type { RunRepositoryTransaction } from '../../ports/runRepository';
|
||||
import {
|
||||
LegacySequelizeRunRepository,
|
||||
LegacySequelizeRunTransaction,
|
||||
} from './runRepository';
|
||||
|
||||
export interface SequelizeRunProjectionContext {
|
||||
transaction: Transaction;
|
||||
runs: RunRepositoryTransaction;
|
||||
changedRunIds: readonly string[];
|
||||
changedAttemptIds: readonly string[];
|
||||
}
|
||||
|
||||
export interface SequelizeRunProjectionParticipant {
|
||||
apply(context: SequelizeRunProjectionContext): Promise<void>;
|
||||
}
|
||||
|
||||
class TrackingRunRepositoryTransaction implements RunRepositoryTransaction {
|
||||
readonly changedRunIds = new Set<string>();
|
||||
readonly changedAttemptIds = new Set<string>();
|
||||
|
||||
constructor(private readonly delegate: RunRepositoryTransaction) {}
|
||||
|
||||
findRunById(runId: string): Promise<RunRecord | null> {
|
||||
return this.delegate.findRunById(runId);
|
||||
}
|
||||
|
||||
findAttemptById(attemptId: string): Promise<RunAttemptRecord | null> {
|
||||
return this.delegate.findAttemptById(attemptId);
|
||||
}
|
||||
|
||||
findLatestAttemptByRunId(runId: string): Promise<RunAttemptRecord | null> {
|
||||
return this.delegate.findLatestAttemptByRunId(runId);
|
||||
}
|
||||
|
||||
findRetryPolicyByRunId(runId: string): Promise<RunRetryPolicyRecord | null> {
|
||||
return this.delegate.findRetryPolicyByRunId(runId);
|
||||
}
|
||||
|
||||
listEvents(
|
||||
runId: string,
|
||||
options?: { afterSequence?: number; limit?: number },
|
||||
): Promise<RunEventRecord[]> {
|
||||
return this.delegate.listEvents(runId, options);
|
||||
}
|
||||
|
||||
listCancellationRequested(options?: {
|
||||
beforeMs?: number;
|
||||
limit?: number;
|
||||
}): Promise<RunRecord[]> {
|
||||
return this.delegate.listCancellationRequested(options);
|
||||
}
|
||||
|
||||
async insertRun(run: RunRecord): Promise<void> {
|
||||
await this.delegate.insertRun(run);
|
||||
this.changedRunIds.add(run.id);
|
||||
}
|
||||
|
||||
async insertAttempt(attempt: RunAttemptRecord): Promise<void> {
|
||||
await this.delegate.insertAttempt(attempt);
|
||||
this.changedRunIds.add(attempt.runId);
|
||||
this.changedAttemptIds.add(attempt.id);
|
||||
}
|
||||
|
||||
insertRetryPolicy(policy: RunRetryPolicyRecord): Promise<void> {
|
||||
return this.delegate.insertRetryPolicy(policy);
|
||||
}
|
||||
|
||||
async compareAndSetRun(
|
||||
run: RunRecord,
|
||||
expectedVersion: number,
|
||||
): Promise<boolean> {
|
||||
const updated = await this.delegate.compareAndSetRun(run, expectedVersion);
|
||||
if (updated) this.changedRunIds.add(run.id);
|
||||
return updated;
|
||||
}
|
||||
|
||||
async compareAndSetAttempt(
|
||||
attempt: RunAttemptRecord,
|
||||
expected: { status: RunAttemptStatus; callbackSequence: number },
|
||||
): Promise<boolean> {
|
||||
const updated = await this.delegate.compareAndSetAttempt(attempt, expected);
|
||||
if (updated) {
|
||||
this.changedRunIds.add(attempt.runId);
|
||||
this.changedAttemptIds.add(attempt.id);
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
compareAndSetRetryPolicy(
|
||||
policy: RunRetryPolicyRecord,
|
||||
expectedVersion: number,
|
||||
): Promise<boolean> {
|
||||
return this.delegate.compareAndSetRetryPolicy(policy, expectedVersion);
|
||||
}
|
||||
|
||||
appendEvent(event: RunEventRecord): Promise<void> {
|
||||
return this.delegate.appendEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Primary-only repository. Existing Shadow repositories keep their original
|
||||
* transaction implementation and never execute these projection participants.
|
||||
*/
|
||||
export class LegacySequelizeProjectedRunRepository extends LegacySequelizeRunRepository {
|
||||
private readonly participants: readonly SequelizeRunProjectionParticipant[];
|
||||
|
||||
constructor(
|
||||
private readonly projectedDatabase: Sequelize,
|
||||
participants: readonly SequelizeRunProjectionParticipant[],
|
||||
) {
|
||||
super(projectedDatabase);
|
||||
this.participants = [...participants];
|
||||
}
|
||||
|
||||
override async transaction<T>(
|
||||
work: (transaction: RunRepositoryTransaction) => Promise<T>,
|
||||
): Promise<T> {
|
||||
return this.projectedDatabase.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const runs = new LegacySequelizeRunTransaction(
|
||||
this.models,
|
||||
transaction,
|
||||
);
|
||||
const tracked = new TrackingRunRepositoryTransaction(runs);
|
||||
const result = await work(tracked);
|
||||
if (
|
||||
tracked.changedRunIds.size > 0 ||
|
||||
tracked.changedAttemptIds.size > 0
|
||||
) {
|
||||
const context: SequelizeRunProjectionContext = {
|
||||
transaction,
|
||||
runs,
|
||||
changedRunIds: [...tracked.changedRunIds],
|
||||
changedAttemptIds: [...tracked.changedAttemptIds],
|
||||
};
|
||||
for (const participant of this.participants) {
|
||||
await participant.apply(context);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { RUN_DISPATCH_LEASE_TABLE } from '../../../migrations/0009-run-dispatch-lease';
|
||||
import { RUN_DISPATCH_CANDIDATE_RUN_INDEX } from '../../../migrations/0010-run-dispatch-candidates';
|
||||
import {
|
||||
MAX_RUN_DISPATCH_CANDIDATE_PAGE_SIZE,
|
||||
assertRunDispatchCandidate,
|
||||
assertRunDispatchCandidateCursor,
|
||||
assertRunDispatchCandidatePageSize,
|
||||
type RunDispatchCandidate,
|
||||
} from '../../domain/runDispatchCandidate';
|
||||
import { assertRunDispatchLeaseVersion } from '../../domain/runDispatchLease';
|
||||
import type {
|
||||
ListRunDispatchCandidatesOptions,
|
||||
RunDispatchCandidateSource,
|
||||
} from '../../ports/runDispatchCandidateSource';
|
||||
|
||||
interface CandidateRow {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
projectId: string;
|
||||
taskId: string;
|
||||
taskRevision: string;
|
||||
priority: number | string;
|
||||
queuedAtMs: number | string;
|
||||
attemptCreatedAtMs: number | string;
|
||||
executorType: string;
|
||||
}
|
||||
|
||||
const CANDIDATE_ORDER = `
|
||||
r.priority DESC,
|
||||
r.queued_at_ms ASC,
|
||||
a.created_at_ms ASC,
|
||||
a.id ASC
|
||||
`;
|
||||
|
||||
export class LegacySequelizeRunDispatchCandidateSource
|
||||
implements RunDispatchCandidateSource
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy Run dispatch candidate source is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async listCandidates({
|
||||
observedAtMs,
|
||||
after,
|
||||
limit = MAX_RUN_DISPATCH_CANDIDATE_PAGE_SIZE,
|
||||
}: ListRunDispatchCandidatesOptions): Promise<RunDispatchCandidate[]> {
|
||||
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
|
||||
assertRunDispatchCandidatePageSize(limit);
|
||||
if (after) assertRunDispatchCandidateCursor(after);
|
||||
|
||||
const cursorPredicate = after
|
||||
? `AND (
|
||||
r.priority < :afterPriority
|
||||
OR (r.priority = :afterPriority AND r.queued_at_ms > :afterQueuedAtMs)
|
||||
OR (
|
||||
r.priority = :afterPriority
|
||||
AND r.queued_at_ms = :afterQueuedAtMs
|
||||
AND a.created_at_ms > :afterAttemptCreatedAtMs
|
||||
)
|
||||
OR (
|
||||
r.priority = :afterPriority
|
||||
AND r.queued_at_ms = :afterQueuedAtMs
|
||||
AND a.created_at_ms = :afterAttemptCreatedAtMs
|
||||
AND a.id > :afterAttemptId
|
||||
)
|
||||
)`
|
||||
: '';
|
||||
const rows = await this.database.query<CandidateRow>(
|
||||
`SELECT
|
||||
r.id AS runId,
|
||||
a.id AS attemptId,
|
||||
r.project_id AS projectId,
|
||||
r.task_id AS taskId,
|
||||
r.task_revision AS taskRevision,
|
||||
r.priority AS priority,
|
||||
r.queued_at_ms AS queuedAtMs,
|
||||
a.created_at_ms AS attemptCreatedAtMs,
|
||||
a.executor_type AS executorType
|
||||
FROM ${RUN_TABLE} r INDEXED BY ${RUN_DISPATCH_CANDIDATE_RUN_INDEX}
|
||||
INNER JOIN ${RUN_ATTEMPT_TABLE} a ON a.run_id = r.id
|
||||
LEFT JOIN ${RUN_DISPATCH_LEASE_TABLE} l ON l.attempt_id = a.id
|
||||
WHERE r.execution_owner = 'runtime'
|
||||
AND r.status IN ('queued', 'dispatching')
|
||||
AND r.queued_at_ms IS NOT NULL
|
||||
AND r.cancel_requested_at_ms IS NULL
|
||||
AND a.status = 'claimed'
|
||||
AND (
|
||||
l.attempt_id IS NULL
|
||||
OR l.status = 'released'
|
||||
OR (l.status = 'leased' AND l.expires_at_ms <= :observedAtMs)
|
||||
)
|
||||
${cursorPredicate}
|
||||
ORDER BY ${CANDIDATE_ORDER}
|
||||
LIMIT :limit`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
observedAtMs,
|
||||
limit,
|
||||
...(after
|
||||
? {
|
||||
afterPriority: after.priority,
|
||||
afterQueuedAtMs: after.queuedAtMs,
|
||||
afterAttemptCreatedAtMs: after.attemptCreatedAtMs,
|
||||
afterAttemptId: after.attemptId,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
);
|
||||
return rows.map((row) => {
|
||||
const candidate: RunDispatchCandidate = {
|
||||
runId: row.runId,
|
||||
attemptId: row.attemptId,
|
||||
projectId: row.projectId,
|
||||
taskId: row.taskId,
|
||||
taskRevision: row.taskRevision,
|
||||
priority: Number(row.priority),
|
||||
queuedAtMs: Number(row.queuedAtMs),
|
||||
attemptCreatedAtMs: Number(row.attemptCreatedAtMs),
|
||||
executorType: row.executorType,
|
||||
};
|
||||
assertRunDispatchCandidate(candidate);
|
||||
return candidate;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import {
|
||||
RUN_DISPATCH_LEASE_EXPIRY_INDEX,
|
||||
RUN_DISPATCH_LEASE_TABLE,
|
||||
} from '../../../migrations/0009-run-dispatch-lease';
|
||||
import {
|
||||
assertRunDispatchId,
|
||||
assertRunDispatchLeaseVersion,
|
||||
} from '../../domain/runDispatchLease';
|
||||
import {
|
||||
MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE,
|
||||
type ExpiredRunDispatchLeaseCandidate,
|
||||
type ListExpiredRunDispatchLeasesOptions,
|
||||
type RunDispatchLeaseExpirySource,
|
||||
} from '../../ports/runDispatchLeaseExpirySource';
|
||||
|
||||
interface ExpiredLeaseRow {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
expiresAtMs: number | string;
|
||||
}
|
||||
|
||||
function assertLimit(limit: number): void {
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
`limit must be between 1 and ${MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeRunDispatchLeaseExpirySource
|
||||
implements RunDispatchLeaseExpirySource
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy Run dispatch lease expiry source is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async listExpired({
|
||||
observedAtMs,
|
||||
after,
|
||||
limit = 16,
|
||||
}: ListExpiredRunDispatchLeasesOptions): Promise<
|
||||
readonly ExpiredRunDispatchLeaseCandidate[]
|
||||
> {
|
||||
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
|
||||
assertLimit(limit);
|
||||
if (after) {
|
||||
assertRunDispatchLeaseVersion('after.expiresAtMs', after.expiresAtMs);
|
||||
assertRunDispatchId('after.attemptId', after.attemptId);
|
||||
}
|
||||
const cursorPredicate = after
|
||||
? `AND (
|
||||
l.expires_at_ms > :afterExpiresAtMs
|
||||
OR (
|
||||
l.expires_at_ms = :afterExpiresAtMs
|
||||
AND l.attempt_id > :afterAttemptId
|
||||
)
|
||||
)`
|
||||
: '';
|
||||
const rows = await this.database.query<ExpiredLeaseRow>(
|
||||
`SELECT
|
||||
l.run_id AS runId,
|
||||
l.attempt_id AS attemptId,
|
||||
l.expires_at_ms AS expiresAtMs
|
||||
FROM ${RUN_DISPATCH_LEASE_TABLE} l INDEXED BY ${RUN_DISPATCH_LEASE_EXPIRY_INDEX}
|
||||
INNER JOIN ${RUN_TABLE} r ON r.id = l.run_id
|
||||
INNER JOIN ${RUN_ATTEMPT_TABLE} a ON a.id = l.attempt_id
|
||||
WHERE l.status = 'leased'
|
||||
AND l.expires_at_ms <= :observedAtMs
|
||||
AND r.execution_owner = 'runtime'
|
||||
AND r.status IN ('dispatching', 'running')
|
||||
AND a.run_id = r.id
|
||||
AND a.status IN ('claimed', 'starting', 'running')
|
||||
${cursorPredicate}
|
||||
ORDER BY l.expires_at_ms ASC, l.attempt_id ASC
|
||||
LIMIT :limit`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
observedAtMs,
|
||||
limit,
|
||||
...(after
|
||||
? {
|
||||
afterExpiresAtMs: after.expiresAtMs,
|
||||
afterAttemptId: after.attemptId,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
);
|
||||
return rows.map((row) => {
|
||||
const candidate = {
|
||||
runId: row.runId,
|
||||
attemptId: row.attemptId,
|
||||
expiresAtMs: Number(row.expiresAtMs),
|
||||
};
|
||||
assertRunDispatchId('runId', candidate.runId);
|
||||
assertRunDispatchId('attemptId', candidate.attemptId);
|
||||
assertRunDispatchLeaseVersion('expiresAtMs', candidate.expiresAtMs);
|
||||
if (candidate.expiresAtMs > observedAtMs) {
|
||||
throw new TypeError('Expiry source returned a live Run lease');
|
||||
}
|
||||
return candidate;
|
||||
});
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,164 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import {
|
||||
RUN_DISPATCH_LEASE_EXPIRY_INDEX,
|
||||
RUN_DISPATCH_LEASE_TABLE,
|
||||
} from '../../../migrations/0009-run-dispatch-lease';
|
||||
import { WORKER_REGISTRY_TABLE } from '../../../migrations/0008-worker-registry';
|
||||
import type { RunDispatchCandidate } from '../../domain/runDispatchCandidate';
|
||||
import {
|
||||
MAX_RUN_DISPATCH_RECOVERY_PAGE_SIZE,
|
||||
assertRecoverableRunDispatch,
|
||||
assertRunDispatchRecoveryCursor,
|
||||
assertRunDispatchRecoveryPageSize,
|
||||
type RecoverableRunDispatch,
|
||||
} from '../../domain/runDispatchRecovery';
|
||||
import {
|
||||
assertRunDispatchLeaseVersion,
|
||||
type RunDispatchLeaseRecord,
|
||||
} from '../../domain/runDispatchLease';
|
||||
import type {
|
||||
ListRecoverableRunDispatchesOptions,
|
||||
RunDispatchRecoverySource,
|
||||
} from '../../ports/runDispatchRecoverySource';
|
||||
|
||||
interface RecoveryRow {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
projectId: string;
|
||||
taskId: string;
|
||||
taskRevision: string;
|
||||
priority: number | string;
|
||||
queuedAtMs: number | string;
|
||||
attemptCreatedAtMs: number | string;
|
||||
executorType: string;
|
||||
version: number | string;
|
||||
leaseGeneration: number | string;
|
||||
workerId: string;
|
||||
workerSessionId: string;
|
||||
workerGeneration: number | string;
|
||||
leaseToken: string;
|
||||
acquiredAtMs: number | string;
|
||||
renewedAtMs: number | string;
|
||||
expiresAtMs: number | string;
|
||||
updatedAtMs: number | string;
|
||||
}
|
||||
|
||||
export class LegacySequelizeRunDispatchRecoverySource
|
||||
implements RunDispatchRecoverySource
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy Run dispatch recovery source is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async listRecoverable({
|
||||
observedAtMs,
|
||||
after,
|
||||
limit = MAX_RUN_DISPATCH_RECOVERY_PAGE_SIZE,
|
||||
}: ListRecoverableRunDispatchesOptions): Promise<RecoverableRunDispatch[]> {
|
||||
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
|
||||
assertRunDispatchRecoveryPageSize(limit);
|
||||
if (after) assertRunDispatchRecoveryCursor(after);
|
||||
const cursorPredicate = after
|
||||
? `AND (
|
||||
l.expires_at_ms > :afterExpiresAtMs
|
||||
OR (
|
||||
l.expires_at_ms = :afterExpiresAtMs
|
||||
AND l.attempt_id > :afterAttemptId
|
||||
)
|
||||
)`
|
||||
: '';
|
||||
const rows = await this.database.query<RecoveryRow>(
|
||||
`SELECT
|
||||
r.id AS runId,
|
||||
a.id AS attemptId,
|
||||
r.project_id AS projectId,
|
||||
r.task_id AS taskId,
|
||||
r.task_revision AS taskRevision,
|
||||
r.priority AS priority,
|
||||
r.queued_at_ms AS queuedAtMs,
|
||||
a.created_at_ms AS attemptCreatedAtMs,
|
||||
a.executor_type AS executorType,
|
||||
l.version AS version,
|
||||
l.lease_generation AS leaseGeneration,
|
||||
l.worker_id AS workerId,
|
||||
l.worker_session_id AS workerSessionId,
|
||||
l.worker_generation AS workerGeneration,
|
||||
l.lease_token AS leaseToken,
|
||||
l.acquired_at_ms AS acquiredAtMs,
|
||||
l.renewed_at_ms AS renewedAtMs,
|
||||
l.expires_at_ms AS expiresAtMs,
|
||||
l.updated_at_ms AS updatedAtMs
|
||||
FROM ${RUN_DISPATCH_LEASE_TABLE} l INDEXED BY ${RUN_DISPATCH_LEASE_EXPIRY_INDEX}
|
||||
INNER JOIN ${RUN_TABLE} r ON r.id = l.run_id
|
||||
INNER JOIN ${RUN_ATTEMPT_TABLE} a ON a.id = l.attempt_id
|
||||
INNER JOIN ${WORKER_REGISTRY_TABLE} w ON w.id = l.worker_id
|
||||
WHERE l.status = 'leased'
|
||||
AND l.expires_at_ms > :observedAtMs
|
||||
AND r.execution_owner = 'runtime'
|
||||
AND r.status = 'dispatching'
|
||||
AND r.cancel_requested_at_ms IS NULL
|
||||
AND r.queued_at_ms IS NOT NULL
|
||||
AND a.run_id = r.id
|
||||
AND a.status = 'claimed'
|
||||
AND w.session_id = l.worker_session_id
|
||||
AND w.generation = l.worker_generation
|
||||
AND w.status IN ('online', 'draining')
|
||||
AND w.lease_expires_at_ms > :observedAtMs
|
||||
${cursorPredicate}
|
||||
ORDER BY l.expires_at_ms ASC, l.attempt_id ASC
|
||||
LIMIT :limit`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
observedAtMs,
|
||||
limit,
|
||||
...(after
|
||||
? {
|
||||
afterExpiresAtMs: after.expiresAtMs,
|
||||
afterAttemptId: after.attemptId,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
);
|
||||
return rows.map((row) => {
|
||||
const candidate: RunDispatchCandidate = {
|
||||
runId: row.runId,
|
||||
attemptId: row.attemptId,
|
||||
projectId: row.projectId,
|
||||
taskId: row.taskId,
|
||||
taskRevision: row.taskRevision,
|
||||
priority: Number(row.priority),
|
||||
queuedAtMs: Number(row.queuedAtMs),
|
||||
attemptCreatedAtMs: Number(row.attemptCreatedAtMs),
|
||||
executorType: row.executorType,
|
||||
};
|
||||
const lease: RunDispatchLeaseRecord = {
|
||||
attemptId: row.attemptId,
|
||||
runId: row.runId,
|
||||
status: 'leased',
|
||||
version: Number(row.version),
|
||||
leaseGeneration: Number(row.leaseGeneration),
|
||||
workerId: row.workerId,
|
||||
workerSessionId: row.workerSessionId,
|
||||
workerGeneration: Number(row.workerGeneration),
|
||||
leaseToken: row.leaseToken,
|
||||
acquiredAtMs: Number(row.acquiredAtMs),
|
||||
renewedAtMs: Number(row.renewedAtMs),
|
||||
expiresAtMs: Number(row.expiresAtMs),
|
||||
updatedAtMs: Number(row.updatedAtMs),
|
||||
};
|
||||
const recovery = { candidate, lease };
|
||||
assertRecoverableRunDispatch(recovery);
|
||||
return recovery;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import { RUN_TABLE } from '../../../migrations/0002-run-schema';
|
||||
import {
|
||||
RUN_LOST_RETRY_INDEX,
|
||||
RUN_RETRY_POLICY_DUE_INDEX,
|
||||
RUN_RETRY_POLICY_TABLE,
|
||||
} from '../../../migrations/0011-run-retry-policy';
|
||||
import {
|
||||
MAX_RUN_LOST_RETRY_PAGE_SIZE,
|
||||
type ListRunLostRetryCandidatesOptions,
|
||||
type RunLostRetryCandidate,
|
||||
type RunLostRetrySource,
|
||||
} from '../../ports/runLostRetrySource';
|
||||
|
||||
interface RunLostRetryCandidateRow {
|
||||
runId: string;
|
||||
phase: 'lost' | 'retry_wait';
|
||||
availableAtMs: number | string;
|
||||
}
|
||||
|
||||
export class LegacySequelizeRunLostRetrySource implements RunLostRetrySource {
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy lost retry source is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async listCandidates({
|
||||
observedAtMs,
|
||||
limit = 16,
|
||||
}: ListRunLostRetryCandidatesOptions): Promise<
|
||||
readonly RunLostRetryCandidate[]
|
||||
> {
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new RangeError('observedAtMs must be a non-negative safe integer');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_RUN_LOST_RETRY_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
`limit must be between 1 and ${MAX_RUN_LOST_RETRY_PAGE_SIZE}`,
|
||||
);
|
||||
}
|
||||
const rows = await this.database.query<RunLostRetryCandidateRow>(
|
||||
`SELECT runId, phase, availableAtMs
|
||||
FROM (
|
||||
SELECT
|
||||
r.id AS runId,
|
||||
'lost' AS phase,
|
||||
0 AS availableAtMs
|
||||
FROM ${RUN_TABLE} r INDEXED BY ${RUN_LOST_RETRY_INDEX}
|
||||
WHERE r.execution_owner = 'runtime'
|
||||
AND r.status = 'lost'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
r.id AS runId,
|
||||
'retry_wait' AS phase,
|
||||
p.next_attempt_at_ms AS availableAtMs
|
||||
FROM ${RUN_RETRY_POLICY_TABLE} p INDEXED BY ${RUN_RETRY_POLICY_DUE_INDEX}
|
||||
INNER JOIN ${RUN_TABLE} r ON r.id = p.run_id
|
||||
WHERE p.next_attempt_at_ms IS NOT NULL
|
||||
AND p.next_attempt_at_ms <= :observedAtMs
|
||||
AND r.execution_owner = 'runtime'
|
||||
AND r.status = 'retry_wait'
|
||||
) candidates
|
||||
ORDER BY availableAtMs ASC, runId ASC
|
||||
LIMIT :limit`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { observedAtMs, limit },
|
||||
},
|
||||
);
|
||||
return rows.map((row) => ({
|
||||
runId: row.runId,
|
||||
phase: row.phase,
|
||||
availableAtMs: Number(row.availableAtMs),
|
||||
}));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,278 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Sequelize,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { TASK_EXECUTION_REVISION_TABLE } from '../../../migrations/0012-task-execution-revisions';
|
||||
import { EXECUTOR_TYPES, type ExecutorType } from '../../domain/execution';
|
||||
import type { PinnedTaskExecutionRevision } from '../../domain/taskExecutionRevision';
|
||||
import {
|
||||
createPinnedTaskExecutionRevisionRecord,
|
||||
normalizePinnedTaskExecutionRevision,
|
||||
taskExecutionRevisionDigest,
|
||||
TaskExecutionRevisionCorruptError,
|
||||
} from '../../domain/taskExecutionRevisionRecord';
|
||||
import type {
|
||||
InsertTaskExecutionRevisionResult,
|
||||
TaskExecutionRevisionRepository,
|
||||
} from '../../ports/taskExecutionRevisionRepository';
|
||||
import type { TaskExecutionRevisionRequest } from '../../ports/taskExecutionRevisionSource';
|
||||
|
||||
interface TaskExecutionRevisionRow {
|
||||
projectId: string;
|
||||
taskId: string;
|
||||
taskRevision: string;
|
||||
executorType: string;
|
||||
executionTemplate: string;
|
||||
contextRef: string;
|
||||
contentDigest: string;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface TaskExecutionRevisionInstance
|
||||
extends Model<TaskExecutionRevisionRow, TaskExecutionRevisionRow>,
|
||||
TaskExecutionRevisionRow {}
|
||||
|
||||
export class TaskExecutionRevisionConflictError extends Error {
|
||||
constructor(
|
||||
readonly projectId: string,
|
||||
readonly taskId: string,
|
||||
readonly taskRevision: string,
|
||||
) {
|
||||
super(
|
||||
`Task execution revision ${projectId}/${taskId}@${taskRevision} is immutable`,
|
||||
);
|
||||
this.name = 'TaskExecutionRevisionConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
function defineTaskExecutionRevisionModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<TaskExecutionRevisionInstance> {
|
||||
return database.define<TaskExecutionRevisionInstance>(
|
||||
'Ql3TaskExecutionRevision',
|
||||
{
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
taskId: {
|
||||
field: 'task_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
taskRevision: {
|
||||
field: 'task_revision',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
executorType: {
|
||||
field: 'executor_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
executionTemplate: {
|
||||
field: 'execution_template',
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: false,
|
||||
},
|
||||
contextRef: {
|
||||
field: 'context_ref',
|
||||
type: DataTypes.STRING(512),
|
||||
allowNull: false,
|
||||
},
|
||||
contentDigest: {
|
||||
field: 'content_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: TASK_EXECUTION_REVISION_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function assertIdentity(name: string, value: string, maximum: number): void {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > maximum ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) {
|
||||
throw new TypeError(`${name} is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertRequest(request: Readonly<TaskExecutionRevisionRequest>): void {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError('Task execution revision request must be an object');
|
||||
}
|
||||
assertIdentity('projectId', request.projectId, 128);
|
||||
assertIdentity('taskId', request.taskId, 255);
|
||||
assertIdentity('taskRevision', request.taskRevision, 128);
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
function rowToRevision(
|
||||
row: TaskExecutionRevisionRow,
|
||||
): PinnedTaskExecutionRevision {
|
||||
if (!EXECUTOR_TYPES.includes(row.executorType as ExecutorType)) {
|
||||
throw new TaskExecutionRevisionCorruptError(
|
||||
'Stored Task execution revision has an invalid executor type',
|
||||
);
|
||||
}
|
||||
let execution: unknown;
|
||||
try {
|
||||
execution = JSON.parse(row.executionTemplate);
|
||||
} catch {
|
||||
throw new TaskExecutionRevisionCorruptError(
|
||||
'Stored Task execution revision template is not valid JSON',
|
||||
);
|
||||
}
|
||||
try {
|
||||
const normalized = normalizePinnedTaskExecutionRevision({
|
||||
projectId: row.projectId,
|
||||
taskId: row.taskId,
|
||||
taskRevision: row.taskRevision,
|
||||
executorType: row.executorType as ExecutorType,
|
||||
execution: execution as PinnedTaskExecutionRevision['execution'],
|
||||
contextRef: row.contextRef,
|
||||
});
|
||||
if (JSON.stringify(normalized.execution) !== row.executionTemplate) {
|
||||
throw new TaskExecutionRevisionCorruptError(
|
||||
'Stored Task execution revision template is not canonical',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!/^[0-9a-f]{64}$/.test(row.contentDigest) ||
|
||||
taskExecutionRevisionDigest(normalized) !== row.contentDigest
|
||||
) {
|
||||
throw new TaskExecutionRevisionCorruptError(
|
||||
'Stored Task execution revision digest does not match its content',
|
||||
);
|
||||
}
|
||||
const createdAtMs = Number(row.createdAtMs);
|
||||
return createPinnedTaskExecutionRevisionRecord(normalized, createdAtMs);
|
||||
} catch (error) {
|
||||
if (error instanceof TaskExecutionRevisionCorruptError) throw error;
|
||||
throw new TaskExecutionRevisionCorruptError(
|
||||
`Stored Task execution revision is invalid: ${
|
||||
error instanceof Error ? error.message : 'unknown validation error'
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeTaskExecutionRevisionRepository
|
||||
implements TaskExecutionRevisionRepository
|
||||
{
|
||||
private readonly revision: ModelStatic<TaskExecutionRevisionInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy Task execution revision repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.revision = defineTaskExecutionRevisionModel(database);
|
||||
}
|
||||
|
||||
async resolve(
|
||||
request: Readonly<TaskExecutionRevisionRequest>,
|
||||
): Promise<PinnedTaskExecutionRevision | null> {
|
||||
assertRequest(request);
|
||||
const row = (await this.revision.findOne({
|
||||
where: {
|
||||
projectId: request.projectId,
|
||||
taskId: request.taskId,
|
||||
taskRevision: request.taskRevision,
|
||||
},
|
||||
raw: true,
|
||||
})) as unknown as TaskExecutionRevisionRow | null;
|
||||
return row ? rowToRevision(row) : null;
|
||||
}
|
||||
|
||||
async insert(
|
||||
revision: PinnedTaskExecutionRevision,
|
||||
createdAtMs: number,
|
||||
): Promise<InsertTaskExecutionRevisionResult> {
|
||||
const record = createPinnedTaskExecutionRevisionRecord(
|
||||
revision,
|
||||
createdAtMs,
|
||||
);
|
||||
const values: TaskExecutionRevisionRow = {
|
||||
projectId: record.projectId,
|
||||
taskId: record.taskId,
|
||||
taskRevision: record.taskRevision,
|
||||
executorType: record.executorType,
|
||||
executionTemplate: JSON.stringify(record.execution),
|
||||
contextRef: record.contextRef,
|
||||
contentDigest: record.contentDigest,
|
||||
createdAtMs: record.createdAtMs,
|
||||
};
|
||||
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
try {
|
||||
await this.revision.create(values);
|
||||
return 'inserted';
|
||||
} catch (error) {
|
||||
if (error instanceof UniqueConstraintError) {
|
||||
const existing = await this.resolve(record);
|
||||
if (
|
||||
existing &&
|
||||
taskExecutionRevisionDigest(existing) === record.contentDigest
|
||||
) {
|
||||
return 'idempotent';
|
||||
}
|
||||
throw new TaskExecutionRevisionConflictError(
|
||||
record.projectId,
|
||||
record.taskId,
|
||||
record.taskRevision,
|
||||
);
|
||||
}
|
||||
if (errorCode(error) === 'SQLITE_BUSY' && attempt < 4) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new Error('Task execution revision insert retry budget exhausted');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,504 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Op,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { WORKER_REGISTRY_TABLE } from '../../../migrations/0008-worker-registry';
|
||||
import {
|
||||
WORKER_STATUSES,
|
||||
WorkerFenceRejectedError,
|
||||
WorkerSessionConflictError,
|
||||
assertWorkerConcurrency,
|
||||
assertWorkerId,
|
||||
assertWorkerSessionId,
|
||||
hashWorkerCapabilities,
|
||||
parseWorkerCapabilities,
|
||||
type WorkerRecord,
|
||||
type WorkerStatus,
|
||||
} from '../../domain/worker';
|
||||
import {
|
||||
MAX_AVAILABLE_WORKER_PAGE_SIZE,
|
||||
type AvailableWorkerPage,
|
||||
type HeartbeatWorkerSessionCommand,
|
||||
type RegisterWorkerSessionCommand,
|
||||
type RegisterWorkerSessionResult,
|
||||
type TransitionWorkerSessionCommand,
|
||||
type WorkerRegistryRepository,
|
||||
} from '../../ports/workerRegistryRepository';
|
||||
|
||||
interface WorkerRow {
|
||||
id: string;
|
||||
sessionId: string;
|
||||
generation: number;
|
||||
status: string;
|
||||
version: number;
|
||||
capabilitiesJson: string;
|
||||
capabilitiesHash: string;
|
||||
maxConcurrentRuns: number;
|
||||
availableSlots: number;
|
||||
registeredAtMs: number;
|
||||
lastHeartbeatAtMs: number;
|
||||
leaseExpiresAtMs: number;
|
||||
updatedAtMs: number;
|
||||
}
|
||||
|
||||
interface WorkerInstance extends Model<WorkerRow, WorkerRow>, WorkerRow {}
|
||||
|
||||
function defineWorkerModel(database: Sequelize): ModelStatic<WorkerInstance> {
|
||||
return database.define<WorkerInstance>(
|
||||
'Ql3WorkerRegistry',
|
||||
{
|
||||
id: { type: DataTypes.STRING(128), primaryKey: true },
|
||||
sessionId: {
|
||||
field: 'session_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
generation: { type: DataTypes.INTEGER, allowNull: false },
|
||||
status: { type: DataTypes.STRING(16), allowNull: false },
|
||||
version: { type: DataTypes.INTEGER, allowNull: false },
|
||||
capabilitiesJson: {
|
||||
field: 'capabilities_json',
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: false,
|
||||
},
|
||||
capabilitiesHash: {
|
||||
field: 'capabilities_hash',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
maxConcurrentRuns: {
|
||||
field: 'max_concurrent_runs',
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
},
|
||||
availableSlots: {
|
||||
field: 'available_slots',
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
},
|
||||
registeredAtMs: {
|
||||
field: 'registered_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
lastHeartbeatAtMs: {
|
||||
field: 'last_heartbeat_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
leaseExpiresAtMs: {
|
||||
field: 'lease_expires_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
updatedAtMs: {
|
||||
field: 'updated_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: WORKER_REGISTRY_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function nonNegativeTimestamp(value: number, name: string): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new RangeError(`${name} must be a non-negative safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function positiveInteger(value: number, name: string): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertCapabilities(
|
||||
capabilitiesJson: string,
|
||||
capabilitiesHash: string,
|
||||
): void {
|
||||
parseWorkerCapabilities(capabilitiesJson);
|
||||
if (
|
||||
!/^[0-9a-f]{64}$/.test(capabilitiesHash) ||
|
||||
hashWorkerCapabilities(capabilitiesJson) !== capabilitiesHash
|
||||
) {
|
||||
throw new TypeError('capabilitiesHash does not match capabilitiesJson');
|
||||
}
|
||||
}
|
||||
|
||||
function toRecord(row: WorkerRow): WorkerRecord {
|
||||
if (!WORKER_STATUSES.includes(row.status as WorkerStatus)) {
|
||||
throw new Error(`Worker ${row.id} has an invalid status`);
|
||||
}
|
||||
assertWorkerId(row.id);
|
||||
assertWorkerSessionId(row.sessionId);
|
||||
positiveInteger(Number(row.generation), 'generation');
|
||||
nonNegativeTimestamp(Number(row.version), 'version');
|
||||
assertCapabilities(row.capabilitiesJson, row.capabilitiesHash);
|
||||
assertWorkerConcurrency(
|
||||
Number(row.maxConcurrentRuns),
|
||||
Number(row.availableSlots),
|
||||
);
|
||||
for (const [name, value] of [
|
||||
['registeredAtMs', row.registeredAtMs],
|
||||
['lastHeartbeatAtMs', row.lastHeartbeatAtMs],
|
||||
['leaseExpiresAtMs', row.leaseExpiresAtMs],
|
||||
['updatedAtMs', row.updatedAtMs],
|
||||
] as const) {
|
||||
nonNegativeTimestamp(Number(value), name);
|
||||
}
|
||||
if (
|
||||
Number(row.lastHeartbeatAtMs) < Number(row.registeredAtMs) ||
|
||||
Number(row.leaseExpiresAtMs) <= Number(row.lastHeartbeatAtMs) ||
|
||||
Number(row.updatedAtMs) < Number(row.lastHeartbeatAtMs)
|
||||
) {
|
||||
throw new Error(`Worker ${row.id} timestamps are corrupt`);
|
||||
}
|
||||
return {
|
||||
id: row.id,
|
||||
sessionId: row.sessionId,
|
||||
generation: Number(row.generation),
|
||||
status: row.status as WorkerStatus,
|
||||
version: Number(row.version),
|
||||
capabilities: parseWorkerCapabilities(row.capabilitiesJson),
|
||||
capabilitiesHash: row.capabilitiesHash,
|
||||
maxConcurrentRuns: Number(row.maxConcurrentRuns),
|
||||
availableSlots: Number(row.availableSlots),
|
||||
registeredAtMs: Number(row.registeredAtMs),
|
||||
lastHeartbeatAtMs: Number(row.lastHeartbeatAtMs),
|
||||
leaseExpiresAtMs: Number(row.leaseExpiresAtMs),
|
||||
updatedAtMs: Number(row.updatedAtMs),
|
||||
};
|
||||
}
|
||||
|
||||
function assertRegister(command: RegisterWorkerSessionCommand): void {
|
||||
assertWorkerId(command.workerId);
|
||||
assertWorkerSessionId(command.sessionId);
|
||||
assertCapabilities(command.capabilitiesJson, command.capabilitiesHash);
|
||||
assertWorkerConcurrency(command.maxConcurrentRuns, command.availableSlots);
|
||||
nonNegativeTimestamp(command.registeredAtMs, 'registeredAtMs');
|
||||
nonNegativeTimestamp(command.leaseExpiresAtMs, 'leaseExpiresAtMs');
|
||||
if (command.leaseExpiresAtMs <= command.registeredAtMs) {
|
||||
throw new RangeError('leaseExpiresAtMs must be after registeredAtMs');
|
||||
}
|
||||
}
|
||||
|
||||
function assertHeartbeat(command: HeartbeatWorkerSessionCommand): void {
|
||||
assertWorkerId(command.workerId);
|
||||
assertWorkerSessionId(command.sessionId);
|
||||
positiveInteger(command.generation, 'generation');
|
||||
nonNegativeTimestamp(command.expectedVersion, 'expectedVersion');
|
||||
nonNegativeTimestamp(command.availableSlots, 'availableSlots');
|
||||
nonNegativeTimestamp(command.heartbeatAtMs, 'heartbeatAtMs');
|
||||
nonNegativeTimestamp(command.leaseExpiresAtMs, 'leaseExpiresAtMs');
|
||||
if (command.leaseExpiresAtMs <= command.heartbeatAtMs) {
|
||||
throw new RangeError('leaseExpiresAtMs must be after heartbeatAtMs');
|
||||
}
|
||||
}
|
||||
|
||||
function assertTransition(command: TransitionWorkerSessionCommand): void {
|
||||
assertWorkerId(command.workerId);
|
||||
assertWorkerSessionId(command.sessionId);
|
||||
positiveInteger(command.generation, 'generation');
|
||||
nonNegativeTimestamp(command.expectedVersion, 'expectedVersion');
|
||||
nonNegativeTimestamp(command.transitionedAtMs, 'transitionedAtMs');
|
||||
if (command.status !== 'draining' && command.status !== 'offline') {
|
||||
throw new TypeError('Worker transition status is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function fenceReason(
|
||||
row: WorkerRow | null,
|
||||
command: {
|
||||
workerId: string;
|
||||
sessionId: string;
|
||||
generation: number;
|
||||
expectedVersion: number;
|
||||
},
|
||||
): WorkerFenceRejectedError['reason'] | undefined {
|
||||
if (!row) return 'missing';
|
||||
if (row.sessionId !== command.sessionId) return 'session_mismatch';
|
||||
if (Number(row.generation) !== command.generation) {
|
||||
return 'generation_mismatch';
|
||||
}
|
||||
if (Number(row.version) !== command.expectedVersion) {
|
||||
return 'version_mismatch';
|
||||
}
|
||||
if (row.status === 'offline') return 'offline';
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
export class LegacySequelizeWorkerRegistryRepository
|
||||
implements WorkerRegistryRepository
|
||||
{
|
||||
private readonly worker: ModelStatic<WorkerInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
this.worker = defineWorkerModel(database);
|
||||
}
|
||||
|
||||
async findById(workerId: string): Promise<WorkerRecord | null> {
|
||||
assertWorkerId(workerId);
|
||||
const row = (await this.worker.findByPk(workerId, {
|
||||
raw: true,
|
||||
})) as unknown as WorkerRow | null;
|
||||
return row ? toRecord(row) : null;
|
||||
}
|
||||
|
||||
async register(
|
||||
command: RegisterWorkerSessionCommand,
|
||||
): Promise<RegisterWorkerSessionResult> {
|
||||
assertRegister(command);
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
this.database.getDialect() === 'sqlite'
|
||||
? { type: Transaction.TYPES.IMMEDIATE }
|
||||
: {},
|
||||
async (transaction) => {
|
||||
const current = await this.worker.findByPk(command.workerId, {
|
||||
transaction,
|
||||
lock: transaction.LOCK.UPDATE,
|
||||
});
|
||||
if (!current) {
|
||||
const created = await this.worker.create(
|
||||
{
|
||||
id: command.workerId,
|
||||
sessionId: command.sessionId,
|
||||
generation: 1,
|
||||
status: 'online',
|
||||
version: 0,
|
||||
capabilitiesJson: command.capabilitiesJson,
|
||||
capabilitiesHash: command.capabilitiesHash,
|
||||
maxConcurrentRuns: command.maxConcurrentRuns,
|
||||
availableSlots: command.availableSlots,
|
||||
registeredAtMs: command.registeredAtMs,
|
||||
lastHeartbeatAtMs: command.registeredAtMs,
|
||||
leaseExpiresAtMs: command.leaseExpiresAtMs,
|
||||
updatedAtMs: command.registeredAtMs,
|
||||
},
|
||||
{ transaction },
|
||||
);
|
||||
return {
|
||||
worker: toRecord(created.get()),
|
||||
replacedSession: false,
|
||||
};
|
||||
}
|
||||
|
||||
const row = current.get();
|
||||
if (row.sessionId === command.sessionId) {
|
||||
if (
|
||||
row.capabilitiesHash !== command.capabilitiesHash ||
|
||||
Number(row.maxConcurrentRuns) !== command.maxConcurrentRuns ||
|
||||
Number(row.availableSlots) !== command.availableSlots
|
||||
) {
|
||||
throw new WorkerSessionConflictError(command.workerId);
|
||||
}
|
||||
if (Number(row.leaseExpiresAtMs) <= command.registeredAtMs) {
|
||||
throw new WorkerFenceRejectedError(
|
||||
command.workerId,
|
||||
'lease_expired',
|
||||
);
|
||||
}
|
||||
return { worker: toRecord(row), replacedSession: false };
|
||||
}
|
||||
|
||||
const next: Partial<WorkerRow> = {
|
||||
sessionId: command.sessionId,
|
||||
generation: Number(row.generation) + 1,
|
||||
status: 'online',
|
||||
version: Number(row.version) + 1,
|
||||
capabilitiesJson: command.capabilitiesJson,
|
||||
capabilitiesHash: command.capabilitiesHash,
|
||||
maxConcurrentRuns: command.maxConcurrentRuns,
|
||||
availableSlots: command.availableSlots,
|
||||
registeredAtMs: command.registeredAtMs,
|
||||
lastHeartbeatAtMs: command.registeredAtMs,
|
||||
leaseExpiresAtMs: command.leaseExpiresAtMs,
|
||||
updatedAtMs: command.registeredAtMs,
|
||||
};
|
||||
await current.update(next, { transaction });
|
||||
return {
|
||||
worker: toRecord(current.get()),
|
||||
replacedSession: true,
|
||||
};
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < 4
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new Error('Worker registration retry budget exhausted');
|
||||
}
|
||||
|
||||
async heartbeat(
|
||||
command: HeartbeatWorkerSessionCommand,
|
||||
): Promise<WorkerRecord> {
|
||||
assertHeartbeat(command);
|
||||
return this.database.transaction(async (transaction) => {
|
||||
const current = await this.worker.findByPk(command.workerId, {
|
||||
transaction,
|
||||
lock: transaction.LOCK.UPDATE,
|
||||
});
|
||||
const row = current?.get() ?? null;
|
||||
const reason = fenceReason(row, command);
|
||||
if (reason) throw new WorkerFenceRejectedError(command.workerId, reason);
|
||||
if (!current || !row) {
|
||||
throw new WorkerFenceRejectedError(command.workerId, 'missing');
|
||||
}
|
||||
if (Number(row.leaseExpiresAtMs) <= command.heartbeatAtMs) {
|
||||
throw new WorkerFenceRejectedError(command.workerId, 'lease_expired');
|
||||
}
|
||||
if (command.heartbeatAtMs < Number(row.lastHeartbeatAtMs)) {
|
||||
throw new RangeError('heartbeatAtMs must not move backwards');
|
||||
}
|
||||
assertWorkerConcurrency(
|
||||
Number(row.maxConcurrentRuns),
|
||||
command.availableSlots,
|
||||
);
|
||||
await current.update(
|
||||
{
|
||||
version: Number(row.version) + 1,
|
||||
availableSlots:
|
||||
row.status === 'draining' ? 0 : command.availableSlots,
|
||||
lastHeartbeatAtMs: command.heartbeatAtMs,
|
||||
leaseExpiresAtMs: command.leaseExpiresAtMs,
|
||||
updatedAtMs: command.heartbeatAtMs,
|
||||
},
|
||||
{ transaction },
|
||||
);
|
||||
return toRecord(current.get());
|
||||
});
|
||||
}
|
||||
|
||||
async transition(
|
||||
command: TransitionWorkerSessionCommand,
|
||||
): Promise<WorkerRecord> {
|
||||
assertTransition(command);
|
||||
return this.database.transaction(async (transaction) => {
|
||||
const current = await this.worker.findByPk(command.workerId, {
|
||||
transaction,
|
||||
lock: transaction.LOCK.UPDATE,
|
||||
});
|
||||
const row = current?.get() ?? null;
|
||||
if (
|
||||
row &&
|
||||
row.sessionId === command.sessionId &&
|
||||
Number(row.generation) === command.generation &&
|
||||
row.status === command.status &&
|
||||
Number(row.version) === command.expectedVersion + 1 &&
|
||||
Number(row.updatedAtMs) === command.transitionedAtMs
|
||||
) {
|
||||
return toRecord(row);
|
||||
}
|
||||
const reason = fenceReason(row, command);
|
||||
if (reason) throw new WorkerFenceRejectedError(command.workerId, reason);
|
||||
if (!current || !row) {
|
||||
throw new WorkerFenceRejectedError(command.workerId, 'missing');
|
||||
}
|
||||
if (
|
||||
command.status === 'draining' &&
|
||||
Number(row.leaseExpiresAtMs) <= command.transitionedAtMs
|
||||
) {
|
||||
throw new WorkerFenceRejectedError(command.workerId, 'lease_expired');
|
||||
}
|
||||
if (command.transitionedAtMs < Number(row.lastHeartbeatAtMs)) {
|
||||
throw new RangeError('transitionedAtMs must not move backwards');
|
||||
}
|
||||
await current.update(
|
||||
{
|
||||
status: command.status,
|
||||
version: Number(row.version) + 1,
|
||||
availableSlots: 0,
|
||||
updatedAtMs: command.transitionedAtMs,
|
||||
},
|
||||
{ transaction },
|
||||
);
|
||||
return toRecord(current.get());
|
||||
});
|
||||
}
|
||||
|
||||
async listAvailable({
|
||||
observedAtMs,
|
||||
afterWorkerId,
|
||||
limit = 32,
|
||||
}: {
|
||||
observedAtMs: number;
|
||||
afterWorkerId?: string;
|
||||
limit?: number;
|
||||
}): Promise<AvailableWorkerPage> {
|
||||
nonNegativeTimestamp(observedAtMs, 'observedAtMs');
|
||||
if (afterWorkerId !== undefined) assertWorkerId(afterWorkerId);
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_AVAILABLE_WORKER_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_AVAILABLE_WORKER_PAGE_SIZE',
|
||||
);
|
||||
}
|
||||
const rows = (await this.worker.findAll({
|
||||
where: {
|
||||
status: 'online',
|
||||
availableSlots: { [Op.gt]: 0 },
|
||||
leaseExpiresAtMs: { [Op.gt]: observedAtMs },
|
||||
...(afterWorkerId === undefined
|
||||
? {}
|
||||
: { id: { [Op.gt]: afterWorkerId } }),
|
||||
},
|
||||
order: [['id', 'ASC']],
|
||||
limit: limit + 1,
|
||||
raw: true,
|
||||
})) as unknown as WorkerRow[];
|
||||
const truncated = rows.length > limit;
|
||||
const bounded = rows.slice(0, limit).map(toRecord);
|
||||
return {
|
||||
workers: bounded,
|
||||
truncated,
|
||||
...(bounded.length === 0
|
||||
? {}
|
||||
: { nextCursor: bounded[bounded.length - 1].id }),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import path from 'path';
|
||||
import config from '../../../config';
|
||||
import Logger from '../../../loaders/logger';
|
||||
import {
|
||||
activateManualPrimaryRuntime,
|
||||
type ManualPrimaryActivationAudit,
|
||||
type ManualPrimaryActivationStack,
|
||||
} from '../../application/manualPrimaryRuntimeActivation';
|
||||
import { installManualPrimaryExecutionRouter } from '../../compatibility/manualPrimaryExecutionBridge';
|
||||
import type { RuntimeRolloutPolicy } from '../../domain/runtimeRollout';
|
||||
import { parseDeploymentProfile } from '../../domain/deploymentProfile';
|
||||
import type { RuntimeRolloutLoadResult } from '../../ports/runtimeRolloutLoader';
|
||||
import { loadRuntimeRolloutManifest } from '../fs/runtimeRolloutManifestLoader';
|
||||
import type { DefaultManualPrimaryActivationOptions } from './defaultManualPrimaryActivation';
|
||||
|
||||
export const DEFAULT_RUNTIME_ROLLOUT_MANIFEST_FILE = 'qinglong3-rollout.json';
|
||||
|
||||
interface DefaultManualPrimaryStackModule {
|
||||
createDefaultManualPrimaryActivationStack(
|
||||
rollout: RuntimeRolloutPolicy,
|
||||
options?: DefaultManualPrimaryActivationOptions,
|
||||
): ManualPrimaryActivationStack;
|
||||
}
|
||||
|
||||
export interface BootstrapDefaultManualPrimaryRuntimeOptions
|
||||
extends DefaultManualPrimaryActivationOptions {
|
||||
load?: () => Promise<RuntimeRolloutLoadResult>;
|
||||
loadStack?: () => Promise<DefaultManualPrimaryStackModule>;
|
||||
install?: typeof installManualPrimaryExecutionRouter;
|
||||
audit?: (record: ManualPrimaryActivationAudit) => void | Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lightweight HTTP-worker bootstrap. Heavy Runtime adapters are imported only
|
||||
* after an accepted manifest explicitly selects manual Primary ownership.
|
||||
*/
|
||||
export async function bootstrapDefaultManualPrimaryRuntime(
|
||||
options: BootstrapDefaultManualPrimaryRuntimeOptions = {},
|
||||
) {
|
||||
const sourcePath = path.join(
|
||||
config.configPath,
|
||||
DEFAULT_RUNTIME_ROLLOUT_MANIFEST_FILE,
|
||||
);
|
||||
const load = await (
|
||||
options.load ?? (() => loadRuntimeRolloutManifest(sourcePath))
|
||||
)();
|
||||
const selected =
|
||||
load.status === 'accepted' && load.policy.modeFor('manual') === 'primary';
|
||||
const audit =
|
||||
options.audit ??
|
||||
((record: ManualPrimaryActivationAudit) => {
|
||||
Logger.info(`[runtime-activation] ${JSON.stringify(record)}`);
|
||||
});
|
||||
let stackModule: DefaultManualPrimaryStackModule | undefined;
|
||||
if (selected) {
|
||||
try {
|
||||
stackModule = await (
|
||||
options.loadStack ?? (() => import('./defaultManualPrimaryActivation'))
|
||||
)();
|
||||
} catch (error) {
|
||||
try {
|
||||
await audit({ ...load.audit, activation: 'failed' });
|
||||
} catch {
|
||||
// Preserve the module load error without exposing manifest contents.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const activationOptions: DefaultManualPrimaryActivationOptions = {
|
||||
...(options.database === undefined ? {} : { database: options.database }),
|
||||
...(options.owner === undefined ? {} : { owner: options.owner }),
|
||||
...(options.recovery === undefined ? {} : { recovery: options.recovery }),
|
||||
...(options.completion === undefined
|
||||
? {}
|
||||
: { completion: options.completion }),
|
||||
...(options.cancellation === undefined
|
||||
? {}
|
||||
: { cancellation: options.cancellation }),
|
||||
...(options.timeout === undefined ? {} : { timeout: options.timeout }),
|
||||
};
|
||||
|
||||
return activateManualPrimaryRuntime({
|
||||
load: async () => load,
|
||||
create(rollout) {
|
||||
if (!stackModule) {
|
||||
throw new Error('Primary stack was not loaded for the selected policy');
|
||||
}
|
||||
return stackModule.createDefaultManualPrimaryActivationStack(rollout, {
|
||||
...activationOptions,
|
||||
deploymentProfile:
|
||||
options.deploymentProfile ??
|
||||
parseDeploymentProfile(process.env.QL_DEPLOYMENT_PROFILE),
|
||||
});
|
||||
},
|
||||
install: options.install ?? installManualPrimaryExecutionRouter,
|
||||
audit,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
import type { Sequelize } from 'sequelize';
|
||||
import { sequelize } from '../../../data';
|
||||
import Logger from '../../../loaders/logger';
|
||||
import {
|
||||
PrimaryCompletionReceiptLifecycle,
|
||||
type PrimaryCompletionReceiptLifecycleOptions,
|
||||
} from '../../application/primaryCompletionReceiptLifecycle';
|
||||
import { PrimaryCompletionReceiptJournalScanner } from '../../application/primaryCompletionReceiptJournalScanner';
|
||||
import { PrimaryCompletionReceiptSupervisor } from '../../application/primaryCompletionReceiptSupervisor';
|
||||
import { PrimaryCompletionReceiptConsumer } from '../../application/primaryCompletionReceiptConsumer';
|
||||
import { PrimaryRunCompletionService } from '../../application/primaryRunCompletionService';
|
||||
import {
|
||||
PrimaryCancellationLifecycle,
|
||||
type PrimaryCancellationLifecycleOptions,
|
||||
} from '../../application/primaryCancellationLifecycle';
|
||||
import { PrimaryCancellationDispatcher } from '../../application/primaryCancellationDispatcher';
|
||||
import { PrimaryCancellationSupervisor } from '../../application/primaryCancellationSupervisor';
|
||||
import { PrimaryTimeoutRequester } from '../../application/primaryTimeoutRequester';
|
||||
import { PrimaryTimeoutSupervisor } from '../../application/primaryTimeoutSupervisor';
|
||||
import {
|
||||
PrimaryTimeoutLifecycle,
|
||||
type PrimaryTimeoutLifecycleOptions,
|
||||
} from '../../application/primaryTimeoutLifecycle';
|
||||
import { RunCommandService } from '../../application/runCommandService';
|
||||
import { PrimaryRunStartupReconciler } from '../../application/primaryRunStartupReconciler';
|
||||
import {
|
||||
PrimaryRunStartupSupervisor,
|
||||
type PrimaryRunStartupOptions,
|
||||
} from '../../application/primaryRunStartupSupervisor';
|
||||
import type { RuntimeRolloutPolicy } from '../../domain/runtimeRollout';
|
||||
import {
|
||||
localPrimaryResourcePolicy,
|
||||
type DeploymentProfile,
|
||||
} from '../../domain/deploymentProfile';
|
||||
import { LegacySequelizeCancellationDispatchRepository } from '../legacy-sequelize/cancellationDispatchRepository';
|
||||
import { LegacySequelizePrimaryCancellationSource } from '../legacy-sequelize/primaryCancellationSource';
|
||||
import { LegacySequelizePrimaryTimeoutSource } from '../legacy-sequelize/primaryTimeoutSource';
|
||||
import { PrimaryCronProjection } from '../legacy-sequelize/primaryCronProjection';
|
||||
import { LegacySequelizePrimaryRunRecoverySource } from '../legacy-sequelize/primaryRunRecoverySource';
|
||||
import { LegacySequelizeCompletionReceiptJournal } from '../legacy-sequelize/completionReceiptJournal';
|
||||
import { LegacySequelizeProjectedRunRepository } from '../legacy-sequelize/projectedRunRepository';
|
||||
import { LocalProcessPersistedExecutionInspector } from '../local-process/localProcessIdentity';
|
||||
import { LocalProcessPersistedExecutionController } from '../local-process/persistedLocalProcessController';
|
||||
import { LocalProcessExecutor } from '../local-process/localProcessExecutor';
|
||||
import { CompletionReceiptFileStore } from '../fs/completionReceiptFileStore';
|
||||
import {
|
||||
DEFAULT_COMPLETION_RECEIPT_ROOT,
|
||||
DEFAULT_LOCAL_PROCESS_LAUNCHER_PATH,
|
||||
LegacyManualPrimaryLogFiles,
|
||||
} from './defaultManualPrimaryRuntime';
|
||||
import { ManualPrimaryRuntime } from '../../application/manualPrimaryRuntime';
|
||||
|
||||
export interface DefaultManualPrimaryActivationOptions {
|
||||
database?: Sequelize;
|
||||
owner?: string;
|
||||
deploymentProfile?: DeploymentProfile;
|
||||
recovery?: PrimaryRunStartupOptions;
|
||||
completion?: Pick<
|
||||
PrimaryCompletionReceiptLifecycleOptions,
|
||||
'intervalMs' | 'initialDelayMs' | 'stopTimeoutMs' | 'cycle'
|
||||
>;
|
||||
cancellation?: Pick<
|
||||
PrimaryCancellationLifecycleOptions,
|
||||
'intervalMs' | 'initialDelayMs' | 'stopTimeoutMs' | 'cycle'
|
||||
>;
|
||||
timeout?: Pick<
|
||||
PrimaryTimeoutLifecycleOptions,
|
||||
'intervalMs' | 'initialDelayMs' | 'stopTimeoutMs' | 'cycle'
|
||||
>;
|
||||
}
|
||||
|
||||
function boundedOwner(value: string): string {
|
||||
if (!value || value.length > 128) {
|
||||
throw new RangeError(
|
||||
'Primary activation owner must be 1 to 128 characters',
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function createDefaultManualPrimaryActivationStack(
|
||||
rollout: RuntimeRolloutPolicy,
|
||||
options: DefaultManualPrimaryActivationOptions = {},
|
||||
) {
|
||||
const database = options.database ?? sequelize;
|
||||
const resources = localPrimaryResourcePolicy(
|
||||
options.deploymentProfile ?? 'standalone',
|
||||
);
|
||||
const repository = new LegacySequelizeProjectedRunRepository(database, [
|
||||
new PrimaryCronProjection(database),
|
||||
]);
|
||||
const recoverySource = new LegacySequelizePrimaryRunRecoverySource(database);
|
||||
const completionReceiptJournal = new LegacySequelizeCompletionReceiptJournal(
|
||||
database,
|
||||
);
|
||||
const completionReceiptStore = new CompletionReceiptFileStore(
|
||||
DEFAULT_COMPLETION_RECEIPT_ROOT,
|
||||
);
|
||||
const completionReceipts = new PrimaryCompletionReceiptConsumer(
|
||||
completionReceiptStore,
|
||||
new PrimaryRunCompletionService(repository),
|
||||
{
|
||||
journal: completionReceiptJournal,
|
||||
quarantineRetentionMs: resources.receiptQuarantineRetentionMs,
|
||||
},
|
||||
);
|
||||
const startup = new PrimaryRunStartupSupervisor(
|
||||
new PrimaryRunStartupReconciler(
|
||||
repository,
|
||||
recoverySource,
|
||||
[new LocalProcessPersistedExecutionInspector()],
|
||||
{
|
||||
completionReceipts,
|
||||
completionReceiptJournal,
|
||||
receiptPublishGraceMs: resources.receiptPublishGraceMs,
|
||||
},
|
||||
),
|
||||
);
|
||||
const completion = new PrimaryCompletionReceiptLifecycle(
|
||||
new PrimaryCompletionReceiptSupervisor(
|
||||
new PrimaryCompletionReceiptJournalScanner(
|
||||
completionReceiptJournal,
|
||||
completionReceiptStore,
|
||||
completionReceipts,
|
||||
{
|
||||
terminalMissingRetentionMs:
|
||||
resources.receiptTerminalMissingRetentionMs,
|
||||
},
|
||||
),
|
||||
),
|
||||
{
|
||||
intervalMs:
|
||||
options.completion?.intervalMs ?? resources.completion.intervalMs,
|
||||
initialDelayMs:
|
||||
options.completion?.initialDelayMs ??
|
||||
resources.completion.initialDelayMs,
|
||||
stopTimeoutMs:
|
||||
options.completion?.stopTimeoutMs ?? resources.completion.stopTimeoutMs,
|
||||
cycle: options.completion?.cycle ?? {
|
||||
pageSize: resources.completion.pageSize,
|
||||
maxPages: resources.completion.maxPages,
|
||||
},
|
||||
onCycle(summary) {
|
||||
Logger.info(
|
||||
`[runtime-completion] ${JSON.stringify({
|
||||
profile: resources.profile,
|
||||
pages: summary.pages,
|
||||
scanned: summary.scanned,
|
||||
applied: summary.applied,
|
||||
alreadyTerminal: summary.alreadyTerminal,
|
||||
quarantined: summary.quarantined,
|
||||
purgedQuarantines: summary.purgedQuarantines,
|
||||
expiredMissing: summary.expiredMissing,
|
||||
missing: summary.missing,
|
||||
cleanupPending: summary.cleanupPending,
|
||||
skipped: summary.skipped,
|
||||
ambiguous: summary.ambiguous,
|
||||
failed: summary.failed,
|
||||
stopReason: summary.stopReason,
|
||||
remaining: summary.remaining,
|
||||
})}`,
|
||||
);
|
||||
},
|
||||
onError() {
|
||||
Logger.error('[runtime-completion] cycle failed');
|
||||
},
|
||||
},
|
||||
);
|
||||
const cancellation = new PrimaryCancellationLifecycle(
|
||||
new PrimaryCancellationSupervisor(
|
||||
new PrimaryCancellationDispatcher(
|
||||
new LegacySequelizePrimaryCancellationSource(database),
|
||||
new LegacySequelizeCancellationDispatchRepository(database),
|
||||
[new LocalProcessPersistedExecutionController()],
|
||||
{ owner: boundedOwner(options.owner ?? `http:${process.pid}`) },
|
||||
),
|
||||
),
|
||||
{
|
||||
intervalMs:
|
||||
options.cancellation?.intervalMs ?? resources.cancellation.intervalMs,
|
||||
initialDelayMs:
|
||||
options.cancellation?.initialDelayMs ??
|
||||
resources.cancellation.initialDelayMs,
|
||||
stopTimeoutMs:
|
||||
options.cancellation?.stopTimeoutMs ??
|
||||
resources.cancellation.stopTimeoutMs,
|
||||
cycle: options.cancellation?.cycle ?? {
|
||||
pageSize: resources.cancellation.pageSize,
|
||||
maxPages: resources.cancellation.maxPages,
|
||||
},
|
||||
onCycle(summary) {
|
||||
Logger.info(
|
||||
`[runtime-cancellation] ${JSON.stringify({
|
||||
pages: summary.pages,
|
||||
scanned: summary.scanned,
|
||||
claimed: summary.claimed,
|
||||
pending: summary.pending,
|
||||
failed: summary.failed,
|
||||
stopReason: summary.stopReason,
|
||||
remaining: summary.remaining,
|
||||
})}`,
|
||||
);
|
||||
},
|
||||
onError() {
|
||||
Logger.error('[runtime-cancellation] cycle failed');
|
||||
},
|
||||
},
|
||||
);
|
||||
const timeout = new PrimaryTimeoutLifecycle(
|
||||
new PrimaryTimeoutSupervisor(
|
||||
new PrimaryTimeoutRequester(
|
||||
new LegacySequelizePrimaryTimeoutSource(database),
|
||||
new RunCommandService(repository),
|
||||
),
|
||||
),
|
||||
{
|
||||
intervalMs: options.timeout?.intervalMs ?? resources.timeout.intervalMs,
|
||||
initialDelayMs:
|
||||
options.timeout?.initialDelayMs ?? resources.timeout.initialDelayMs,
|
||||
stopTimeoutMs:
|
||||
options.timeout?.stopTimeoutMs ?? resources.timeout.stopTimeoutMs,
|
||||
cycle: options.timeout?.cycle ?? {
|
||||
pageSize: resources.timeout.pageSize,
|
||||
maxPages: resources.timeout.maxPages,
|
||||
},
|
||||
onCycle(summary) {
|
||||
Logger.info(
|
||||
`[runtime-timeout] ${JSON.stringify({
|
||||
profile: resources.profile,
|
||||
pages: summary.pages,
|
||||
scanned: summary.scanned,
|
||||
accepted: summary.accepted,
|
||||
alreadyRequested: summary.alreadyRequested,
|
||||
alreadyTerminal: summary.alreadyTerminal,
|
||||
failed: summary.failed,
|
||||
stopReason: summary.stopReason,
|
||||
remaining: summary.remaining,
|
||||
})}`,
|
||||
);
|
||||
},
|
||||
onError() {
|
||||
Logger.error('[runtime-timeout] cycle failed');
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
router: new ManualPrimaryRuntime(
|
||||
repository,
|
||||
new LocalProcessExecutor({
|
||||
durableLauncherPath: DEFAULT_LOCAL_PROCESS_LAUNCHER_PATH,
|
||||
}),
|
||||
rollout,
|
||||
new LegacyManualPrimaryLogFiles(
|
||||
undefined,
|
||||
undefined,
|
||||
completionReceiptJournal,
|
||||
),
|
||||
{
|
||||
orchestrator: { completionReceiptJournal },
|
||||
},
|
||||
),
|
||||
reconcile: () => startup.run(options.recovery),
|
||||
startCompletion: () => completion.start(),
|
||||
stopCompletion: () => completion.stop(),
|
||||
startTimeout: () => timeout.start(),
|
||||
stopTimeout: () => timeout.stop(),
|
||||
startCancellation: () => cancellation.start(),
|
||||
stopCancellation: () => cancellation.stop(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import dayjs from 'dayjs';
|
||||
import config from '../../../config';
|
||||
import { getUniqPath } from '../../../config/util';
|
||||
import { sequelize } from '../../../data';
|
||||
import { logStreamManager } from '../../../shared/logStreamManager';
|
||||
import {
|
||||
ManualPrimaryRuntime,
|
||||
type ManualPrimaryLogFiles,
|
||||
type PreparedManualPrimaryLog,
|
||||
} from '../../application/manualPrimaryRuntime';
|
||||
import type { ManualPrimaryStartInput } from '../../compatibility/manualPrimaryExecutionBridge';
|
||||
import { createLegacyLogOutputRef } from '../../compatibility/legacyLogOutputRef';
|
||||
import type { RuntimeRolloutPolicy } from '../../domain/runtimeRollout';
|
||||
import { PrimaryCronProjection } from '../legacy-sequelize/primaryCronProjection';
|
||||
import { LegacySequelizeProjectedRunRepository } from '../legacy-sequelize/projectedRunRepository';
|
||||
import { LocalProcessExecutor } from '../local-process/localProcessExecutor';
|
||||
import { enableDurableLocalProcessOutput } from '../local-process/durableLocalProcessOutput';
|
||||
import { CompletionReceiptFileStore } from '../fs/completionReceiptFileStore';
|
||||
import type { CompletionReceiptJournal } from '../../ports/completionReceiptJournal';
|
||||
|
||||
export const DEFAULT_COMPLETION_RECEIPT_ROOT = path.join(
|
||||
config.dataPath,
|
||||
'runtime',
|
||||
'completion-receipts',
|
||||
);
|
||||
export const DEFAULT_LOCAL_PROCESS_LAUNCHER_PATH = path.join(
|
||||
config.rootPath,
|
||||
'shell',
|
||||
'ql3-launcher.sh',
|
||||
);
|
||||
|
||||
function isWithin(root: string, candidate: string): boolean {
|
||||
return candidate === root || candidate.startsWith(root + path.sep);
|
||||
}
|
||||
|
||||
function relativeLogDirectory(root: string, value: string): string {
|
||||
const candidate = path.isAbsolute(value) ? value : path.resolve(root, value);
|
||||
if (!isWithin(root, candidate)) {
|
||||
throw new Error('Manual Primary log directory escapes the configured root');
|
||||
}
|
||||
const relative = path.relative(root, candidate).split(path.sep).join('/');
|
||||
if (!relative || relative === '.') {
|
||||
throw new Error('Manual Primary log directory must be below the log root');
|
||||
}
|
||||
return relative;
|
||||
}
|
||||
|
||||
export class LegacyManualPrimaryLogFiles implements ManualPrimaryLogFiles {
|
||||
private readonly completionReceipts: CompletionReceiptFileStore;
|
||||
private readonly completionReceiptRoot: string;
|
||||
|
||||
constructor(
|
||||
private readonly logRoot = path.resolve(config.logPath),
|
||||
completionReceiptRoot = DEFAULT_COMPLETION_RECEIPT_ROOT,
|
||||
private readonly completionReceiptJournal?: Pick<
|
||||
CompletionReceiptJournal,
|
||||
'resolve'
|
||||
>,
|
||||
) {
|
||||
this.completionReceiptRoot = path.resolve(completionReceiptRoot);
|
||||
this.completionReceipts = new CompletionReceiptFileStore(
|
||||
this.completionReceiptRoot,
|
||||
);
|
||||
}
|
||||
|
||||
async prepare(
|
||||
input: ManualPrimaryStartInput,
|
||||
): Promise<PreparedManualPrimaryLog> {
|
||||
const configured =
|
||||
!input.cron.logName || input.cron.logName === '/dev/null'
|
||||
? await getUniqPath(input.cron.command, String(input.cron.id))
|
||||
: input.cron.logName;
|
||||
const directory = relativeLogDirectory(this.logRoot, configured);
|
||||
const logPath = path.posix.join(
|
||||
directory,
|
||||
dayjs(input.acceptedAtMs).format('YYYY-MM-DD-HH-mm-ss-SSS') + '.log',
|
||||
);
|
||||
createLegacyLogOutputRef(logPath);
|
||||
const absolutePath = path.resolve(this.logRoot, ...logPath.split('/'));
|
||||
if (!isWithin(this.logRoot, absolutePath)) {
|
||||
throw new Error('Manual Primary log file escapes the configured root');
|
||||
}
|
||||
await fs.mkdir(path.dirname(absolutePath), { recursive: true });
|
||||
const output = enableDurableLocalProcessOutput(
|
||||
{
|
||||
async write(output) {
|
||||
await logStreamManager.write(
|
||||
absolutePath,
|
||||
Buffer.from(output.chunk).toString('utf8'),
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
outputFilePath: absolutePath,
|
||||
completionReceiptRoot: this.completionReceiptRoot,
|
||||
},
|
||||
);
|
||||
const completionReceipts = this.completionReceipts;
|
||||
const completionReceiptJournal = this.completionReceiptJournal;
|
||||
return {
|
||||
logPath,
|
||||
output,
|
||||
async completionCommitted(attemptId) {
|
||||
await completionReceipts.remove(attemptId);
|
||||
await completionReceiptJournal?.resolve(attemptId);
|
||||
},
|
||||
async close() {
|
||||
await logStreamManager.closeStream(absolutePath);
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Legacy factory retained for focused tests; production activation uses the
|
||||
* shared lifecycle stack in defaultManualPrimaryActivation.ts. */
|
||||
export function createDefaultManualPrimaryRuntime(
|
||||
rollout: RuntimeRolloutPolicy,
|
||||
): ManualPrimaryRuntime {
|
||||
const repository = new LegacySequelizeProjectedRunRepository(sequelize, [
|
||||
new PrimaryCronProjection(sequelize),
|
||||
]);
|
||||
return new ManualPrimaryRuntime(
|
||||
repository,
|
||||
new LocalProcessExecutor({
|
||||
durableLauncherPath: DEFAULT_LOCAL_PROCESS_LAUNCHER_PATH,
|
||||
}),
|
||||
rollout,
|
||||
new LegacyManualPrimaryLogFiles(),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import type {
|
||||
ExecutionResourcePolicy,
|
||||
ExecutionSpec,
|
||||
} from '../../domain/execution';
|
||||
import { InvalidExecutionSpecError } from '../../domain/executorErrors';
|
||||
|
||||
export interface LegacyCronSnapshot {
|
||||
id: number;
|
||||
command: string;
|
||||
taskBefore?: string;
|
||||
taskAfter?: string;
|
||||
workDirectory?: string;
|
||||
logName?: string;
|
||||
}
|
||||
|
||||
export interface LegacyCronExecutionInput {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
projectId: string;
|
||||
taskRevision: string;
|
||||
cron: LegacyCronSnapshot;
|
||||
realTime: boolean;
|
||||
realLogPath?: string;
|
||||
noDelay?: boolean;
|
||||
timeoutMs?: number;
|
||||
terminationGraceMs?: number;
|
||||
resourcePolicy?: ExecutionResourcePolicy;
|
||||
}
|
||||
|
||||
export const DEFAULT_LEGACY_TERMINATION_GRACE_MS = 10_000;
|
||||
|
||||
function quoteShellValue(value: string): string {
|
||||
return `'${value.replace(/'/g, `'\\''`)}'`;
|
||||
}
|
||||
|
||||
function normalizeHook(value: string): string {
|
||||
return value.replace(/;? *\r?\n/g, ';').trim();
|
||||
}
|
||||
|
||||
function assignment(name: string, value: string | number | boolean): string {
|
||||
return `${name}=${quoteShellValue(String(value))}`;
|
||||
}
|
||||
|
||||
function legacyTaskCommand(command: string): string {
|
||||
const trimmed = command.trim();
|
||||
if (!trimmed) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'Legacy Cron command must not be empty',
|
||||
);
|
||||
}
|
||||
if (trimmed.startsWith('task ') || trimmed.startsWith('ql ')) return trimmed;
|
||||
return `task ${trimmed}`;
|
||||
}
|
||||
|
||||
export function buildLegacyCronExecutionSpec(
|
||||
input: LegacyCronExecutionInput,
|
||||
): ExecutionSpec {
|
||||
if (!Number.isSafeInteger(input.cron.id) || input.cron.id < 1) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'Legacy Cron id must be a positive safe integer',
|
||||
);
|
||||
}
|
||||
|
||||
const variables: string[] = [];
|
||||
if (input.realLogPath) {
|
||||
variables.push(assignment('real_log_path', input.realLogPath));
|
||||
}
|
||||
if (input.noDelay) variables.push(assignment('no_delay', true));
|
||||
variables.push(assignment('real_time', input.realTime));
|
||||
variables.push(assignment('no_tee', true));
|
||||
variables.push(assignment('ID', input.cron.id));
|
||||
if (input.cron.logName) {
|
||||
variables.push(assignment('log_name', input.cron.logName));
|
||||
}
|
||||
if (input.cron.taskBefore) {
|
||||
variables.push(
|
||||
assignment('task_before', normalizeHook(input.cron.taskBefore)),
|
||||
);
|
||||
}
|
||||
if (input.cron.taskAfter) {
|
||||
variables.push(
|
||||
assignment('task_after', normalizeHook(input.cron.taskAfter)),
|
||||
);
|
||||
}
|
||||
if (input.cron.workDirectory) {
|
||||
variables.push(assignment('work_dir', input.cron.workDirectory));
|
||||
}
|
||||
|
||||
return {
|
||||
runId: input.runId,
|
||||
attemptId: input.attemptId,
|
||||
projectId: input.projectId,
|
||||
taskId: `legacy-cron:${input.cron.id}`,
|
||||
taskRevision: input.taskRevision,
|
||||
command: {
|
||||
kind: 'shell',
|
||||
command: `${variables.join(' ')} ${legacyTaskCommand(
|
||||
input.cron.command,
|
||||
)}`,
|
||||
shell: '/bin/bash',
|
||||
},
|
||||
environmentPolicy: 'inherit',
|
||||
...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }),
|
||||
terminationGraceMs:
|
||||
input.terminationGraceMs ?? DEFAULT_LEGACY_TERMINATION_GRACE_MS,
|
||||
...(input.resourcePolicy === undefined
|
||||
? {}
|
||||
: { resourcePolicy: input.resourcePolicy }),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import type { ExecutionContext, ExecutionSpec } from '../../domain/execution';
|
||||
import { assertCompletionReceiptId } from '../../domain/completionReceipt';
|
||||
import {
|
||||
ExecutorCapabilityUnavailableError,
|
||||
InvalidExecutionSpecError,
|
||||
} from '../../domain/executorErrors';
|
||||
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
|
||||
import type { DurableLocalProcessOutput } from './durableLocalProcessOutput';
|
||||
|
||||
const CALLBACK_TOKEN_PATTERN = /^[A-Za-z0-9_-]{32,128}$/;
|
||||
|
||||
export interface DurableLocalProcessLaunch {
|
||||
file: string;
|
||||
args: readonly string[];
|
||||
environment: NodeJS.ProcessEnv;
|
||||
outputDescriptor: number;
|
||||
closeParentOutput(): Promise<void>;
|
||||
}
|
||||
|
||||
function assertCallback(
|
||||
callback: ExecutionContext['completionCallback'],
|
||||
): asserts callback is NonNullable<ExecutionContext['completionCallback']> {
|
||||
if (!callback || !CALLBACK_TOKEN_PATTERN.test(callback.token)) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable completion requires a bounded base64url callback token',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(callback.callbackSequence) ||
|
||||
callback.callbackSequence < 1
|
||||
) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable completion requires a positive callback sequence',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function launcherEnvironment(
|
||||
environment: NodeJS.ProcessEnv,
|
||||
spec: ExecutionSpec,
|
||||
context: ExecutionContext,
|
||||
capability: DurableLocalProcessOutput,
|
||||
receiptTarget: string,
|
||||
receiptTemporary: string,
|
||||
startedAtMs: number,
|
||||
quotaFifo: string | undefined,
|
||||
quotaRemainingBytes: number | undefined,
|
||||
truncationTarget: string | undefined,
|
||||
truncationTemporary: string | undefined,
|
||||
): NodeJS.ProcessEnv {
|
||||
const callback = context.completionCallback!;
|
||||
return {
|
||||
...environment,
|
||||
QL3_RECEIPT_RUN_ID: spec.runId,
|
||||
QL3_RECEIPT_ATTEMPT_ID: spec.attemptId,
|
||||
QL3_RECEIPT_CALLBACK_SEQUENCE: String(callback.callbackSequence),
|
||||
QL3_RECEIPT_CALLBACK_TOKEN: callback.token,
|
||||
QL3_RECEIPT_STARTED_AT_MS: String(startedAtMs),
|
||||
QL3_RECEIPT_TARGET: receiptTarget,
|
||||
QL3_RECEIPT_TEMPORARY: receiptTemporary,
|
||||
...(quotaFifo === undefined
|
||||
? {}
|
||||
: {
|
||||
QL3_OUTPUT_QUOTA_FIFO: quotaFifo,
|
||||
QL3_OUTPUT_QUOTA_REMAINING_BYTES: String(quotaRemainingBytes),
|
||||
QL3_OUTPUT_ARTIFACT_ID: capability.logArtifactId!,
|
||||
QL3_OUTPUT_MAXIMUM_BYTES: String(capability.maximumBytes),
|
||||
QL3_OUTPUT_TRUNCATION_TARGET: truncationTarget!,
|
||||
QL3_OUTPUT_TRUNCATION_TEMPORARY: truncationTemporary!,
|
||||
}),
|
||||
...(spec.command.kind === 'shell'
|
||||
? {
|
||||
QL3_LAUNCH_SHELL: spec.command.shell ?? '/bin/bash',
|
||||
QL3_LAUNCH_SHELL_COMMAND: spec.command.command,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function prepareDurableLocalProcessLaunch(
|
||||
spec: ExecutionSpec,
|
||||
context: ExecutionContext,
|
||||
environment: NodeJS.ProcessEnv,
|
||||
capability: DurableLocalProcessOutput,
|
||||
launcherPath: string | undefined,
|
||||
startedAtMs: number,
|
||||
): Promise<DurableLocalProcessLaunch> {
|
||||
if (!launcherPath) {
|
||||
throw new ExecutorCapabilityUnavailableError('durableLocalCompletion');
|
||||
}
|
||||
if (!path.isAbsolute(launcherPath) || launcherPath.includes('\0')) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable launcher path must be absolute and contain no NUL',
|
||||
);
|
||||
}
|
||||
assertCallback(context.completionCallback);
|
||||
assertCompletionReceiptId(spec.runId, 'runId');
|
||||
assertCompletionReceiptId(spec.attemptId, 'attemptId');
|
||||
if (!Number.isSafeInteger(startedAtMs) || startedAtMs < 0) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable completion start time must be a non-negative safe integer',
|
||||
);
|
||||
}
|
||||
|
||||
const launcher = await fs.lstat(launcherPath);
|
||||
if (!launcher.isFile()) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable launcher must be a regular file',
|
||||
);
|
||||
}
|
||||
|
||||
const receiptDirectory = path.join(
|
||||
capability.completionReceiptRoot,
|
||||
spec.attemptId.slice(0, 2),
|
||||
);
|
||||
const receiptTarget = path.join(receiptDirectory, `${spec.attemptId}.json`);
|
||||
const receiptTemporary = path.join(
|
||||
receiptDirectory,
|
||||
`.${spec.attemptId}.${randomBytes(16).toString('hex')}.tmp`,
|
||||
);
|
||||
await fs.mkdir(path.dirname(capability.outputFilePath), {
|
||||
recursive: true,
|
||||
mode: 0o700,
|
||||
});
|
||||
await fs.mkdir(receiptDirectory, { recursive: true, mode: 0o700 });
|
||||
|
||||
const output = await fs.open(
|
||||
capability.outputFilePath,
|
||||
constants.O_WRONLY |
|
||||
constants.O_CREAT |
|
||||
constants.O_APPEND |
|
||||
(constants.O_NOFOLLOW ?? 0),
|
||||
0o600,
|
||||
);
|
||||
let quotaFifo: string | undefined;
|
||||
let quotaRemainingBytes: number | undefined;
|
||||
let truncationTarget: string | undefined;
|
||||
let truncationTemporary: string | undefined;
|
||||
try {
|
||||
const stat = await output.stat();
|
||||
if (!stat.isFile()) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable output target must be a regular file',
|
||||
);
|
||||
}
|
||||
await output.chmod(0o600);
|
||||
if (capability.maximumBytes !== undefined) {
|
||||
if (
|
||||
!Number.isSafeInteger(capability.maximumBytes) ||
|
||||
capability.maximumBytes < 1 ||
|
||||
!Number.isSafeInteger(stat.size) ||
|
||||
stat.size < 0 ||
|
||||
stat.size > capability.maximumBytes
|
||||
) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable output quota or existing size is invalid',
|
||||
);
|
||||
}
|
||||
quotaRemainingBytes = capability.maximumBytes - stat.size;
|
||||
if (!capability.logArtifactId) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable output quota requires a Local Artifact identity',
|
||||
);
|
||||
}
|
||||
try {
|
||||
assertLocalExecutionArtifactId(capability.logArtifactId);
|
||||
} catch {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable output Local Artifact identity is invalid',
|
||||
);
|
||||
}
|
||||
if (
|
||||
path.basename(capability.outputFilePath) !==
|
||||
`${capability.logArtifactId}.log`
|
||||
) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'durable output path does not match its Local Artifact identity',
|
||||
);
|
||||
}
|
||||
quotaFifo = path.join(
|
||||
path.dirname(capability.outputFilePath),
|
||||
`.${path.basename(capability.outputFilePath)}.fifo`,
|
||||
);
|
||||
truncationTarget = path.join(
|
||||
path.dirname(capability.outputFilePath),
|
||||
`.${path.basename(capability.outputFilePath)}.truncated.json`,
|
||||
);
|
||||
truncationTemporary = path.join(
|
||||
path.dirname(capability.outputFilePath),
|
||||
`.${path.basename(capability.outputFilePath)}.truncated.tmp`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
await output.close().catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const launchMode = spec.command.kind;
|
||||
const args =
|
||||
spec.command.kind === 'argv'
|
||||
? [launcherPath, launchMode, spec.command.file, ...spec.command.args]
|
||||
: [launcherPath, launchMode];
|
||||
return {
|
||||
file: '/bin/sh',
|
||||
args,
|
||||
environment: launcherEnvironment(
|
||||
environment,
|
||||
spec,
|
||||
context,
|
||||
capability,
|
||||
receiptTarget,
|
||||
receiptTemporary,
|
||||
startedAtMs,
|
||||
quotaFifo,
|
||||
quotaRemainingBytes,
|
||||
truncationTarget,
|
||||
truncationTemporary,
|
||||
),
|
||||
outputDescriptor: output.fd,
|
||||
closeParentOutput: () => output.close(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import path from 'path';
|
||||
import type { ExecutionOutputSink } from '../../domain/execution';
|
||||
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
|
||||
|
||||
const DURABLE_LOCAL_PROCESS_OUTPUT = Symbol('durable-local-process-output');
|
||||
|
||||
export interface DurableLocalProcessOutput {
|
||||
outputFilePath: string;
|
||||
completionReceiptRoot: string;
|
||||
maximumBytes?: number;
|
||||
logArtifactId?: string;
|
||||
}
|
||||
|
||||
type CapableExecutionOutputSink = ExecutionOutputSink & {
|
||||
[DURABLE_LOCAL_PROCESS_OUTPUT]?: DurableLocalProcessOutput;
|
||||
};
|
||||
|
||||
function assertAbsolutePath(value: string, name: string): void {
|
||||
if (!path.isAbsolute(value) || value.includes('\0')) {
|
||||
throw new RangeError(`${name} must be an absolute path containing no NUL`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds an adapter-local launch capability without widening ExecutionContext.
|
||||
* The symbol is deliberately non-enumerable so paths cannot leak through
|
||||
* routine context serialization or diagnostic logging.
|
||||
*/
|
||||
export function enableDurableLocalProcessOutput<T extends ExecutionOutputSink>(
|
||||
output: T,
|
||||
capability: DurableLocalProcessOutput,
|
||||
): T {
|
||||
assertAbsolutePath(capability.outputFilePath, 'outputFilePath');
|
||||
assertAbsolutePath(capability.completionReceiptRoot, 'completionReceiptRoot');
|
||||
if (
|
||||
capability.maximumBytes !== undefined &&
|
||||
(!Number.isSafeInteger(capability.maximumBytes) ||
|
||||
capability.maximumBytes < 1)
|
||||
) {
|
||||
throw new RangeError('maximumBytes must be a positive safe integer');
|
||||
}
|
||||
if (
|
||||
(capability.maximumBytes === undefined) !==
|
||||
(capability.logArtifactId === undefined)
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maximumBytes and logArtifactId must be provided together',
|
||||
);
|
||||
}
|
||||
if (capability.logArtifactId !== undefined) {
|
||||
assertLocalExecutionArtifactId(capability.logArtifactId);
|
||||
}
|
||||
Object.defineProperty(output, DURABLE_LOCAL_PROCESS_OUTPUT, {
|
||||
configurable: false,
|
||||
enumerable: false,
|
||||
writable: false,
|
||||
value: Object.freeze({ ...capability }),
|
||||
});
|
||||
return output;
|
||||
}
|
||||
|
||||
export function durableLocalProcessOutput(
|
||||
output: ExecutionOutputSink,
|
||||
): DurableLocalProcessOutput | undefined {
|
||||
return (output as CapableExecutionOutputSink)[DURABLE_LOCAL_PROCESS_OUTPUT];
|
||||
}
|
||||
@@ -0,0 +1,580 @@
|
||||
import { ChildProcess, SpawnOptions } from 'child_process';
|
||||
import { Readable } from 'stream';
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import { spawn } from 'cross-spawn';
|
||||
import type {
|
||||
ExecutionContext,
|
||||
ExecutionDiagnostic,
|
||||
ExecutionHandle,
|
||||
ExecutionInspection,
|
||||
ExecutionOutputStream,
|
||||
ExecutionResult,
|
||||
ExecutionSpec,
|
||||
ExecutionStopReason,
|
||||
ExecutionStopResult,
|
||||
ExecutorCapabilities,
|
||||
} from '../../domain/execution';
|
||||
import {
|
||||
ExecutorCapabilityUnavailableError,
|
||||
ExecutorHandleNotFoundError,
|
||||
ExecutorStartError,
|
||||
InvalidExecutionSpecError,
|
||||
} from '../../domain/executorErrors';
|
||||
import { assertExecutionSpec as assertDomainExecutionSpec } from '../../domain/executionSpec';
|
||||
export {
|
||||
MAX_EXECUTION_ARGUMENTS,
|
||||
MAX_EXECUTION_COMMAND_BYTES,
|
||||
MAX_EXECUTION_TIMEOUT_MS,
|
||||
MAX_TERMINATION_GRACE_MS,
|
||||
} from '../../domain/executionSpec';
|
||||
import type { Executor } from '../../ports/executor';
|
||||
import {
|
||||
createLocalProcessDurableHandle,
|
||||
LinuxProcProcessIdentityProvider,
|
||||
type LocalProcessIdentityProvider,
|
||||
} from './localProcessIdentity';
|
||||
import {
|
||||
PosixProcessTerminator,
|
||||
type ProcessTerminator,
|
||||
} from './processTerminator';
|
||||
import { durableLocalProcessOutput } from './durableLocalProcessOutput';
|
||||
import {
|
||||
prepareDurableLocalProcessLaunch,
|
||||
type DurableLocalProcessLaunch,
|
||||
} from './durableLocalProcessLaunch';
|
||||
|
||||
export const MAX_EXECUTION_ENVIRONMENT_ENTRIES = 1024;
|
||||
export const MAX_EXECUTION_ENVIRONMENT_BYTES = 512 * 1024;
|
||||
|
||||
const DEFAULT_POSIX_SHELL = '/bin/bash';
|
||||
const ISOLATED_ENVIRONMENT_KEYS = [
|
||||
'PATH',
|
||||
'LANG',
|
||||
'LC_ALL',
|
||||
'LC_CTYPE',
|
||||
'TZ',
|
||||
'TMPDIR',
|
||||
] as const;
|
||||
|
||||
const LOCAL_PROCESS_CAPABILITIES: ExecutorCapabilities = Object.freeze({
|
||||
timeout: true,
|
||||
processGroupTermination: process.platform !== 'win32',
|
||||
workingDirectory: true,
|
||||
isolatedEnvironment: true,
|
||||
memoryLimit: 'none',
|
||||
cpuLimit: 'none',
|
||||
filesystemIsolation: 'none',
|
||||
networkIsolation: 'none',
|
||||
});
|
||||
|
||||
export interface ExecutorClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface LocalProcessExecutorOptions {
|
||||
clock?: ExecutorClock;
|
||||
createHandleId?: () => string;
|
||||
terminator?: ProcessTerminator;
|
||||
identityProvider?: LocalProcessIdentityProvider;
|
||||
durableLauncherPath?: string;
|
||||
}
|
||||
|
||||
interface LocalExecutionLifecycle {
|
||||
startedAtMs: number;
|
||||
closedObserved: boolean;
|
||||
finished: boolean;
|
||||
result?: ExecutionResult;
|
||||
terminationReason?: ExecutionStopReason;
|
||||
runtimeError: boolean;
|
||||
diagnostics: ExecutionDiagnostic[];
|
||||
timeout?: NodeJS.Timeout;
|
||||
removeAbortListener?: () => void;
|
||||
}
|
||||
|
||||
interface LocalExecutionState {
|
||||
child: ChildProcess;
|
||||
processGroup: boolean;
|
||||
graceMs: number;
|
||||
closed: Promise<void>;
|
||||
lifecycle: LocalExecutionLifecycle;
|
||||
stopPromise?: Promise<ExecutionStopResult>;
|
||||
}
|
||||
|
||||
function assertHandleIdentifier(value: string): void {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > 255 ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'handleId must be between 1 and 255 characters and contain no control characters',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertResourcePolicy(spec: ExecutionSpec): void {
|
||||
const policy = spec.resourcePolicy;
|
||||
if (!policy) return;
|
||||
|
||||
if (policy.memoryBytes?.enforcement === 'required') {
|
||||
throw new ExecutorCapabilityUnavailableError('memoryLimit');
|
||||
}
|
||||
if (policy.cpuMillisPerSecond?.enforcement === 'required') {
|
||||
throw new ExecutorCapabilityUnavailableError('cpuLimit');
|
||||
}
|
||||
if (policy.filesystemIsolation === 'required') {
|
||||
throw new ExecutorCapabilityUnavailableError('filesystemIsolation');
|
||||
}
|
||||
if (policy.networkIsolation === 'required') {
|
||||
throw new ExecutorCapabilityUnavailableError('networkIsolation');
|
||||
}
|
||||
}
|
||||
|
||||
function resourcePolicyDiagnostics(spec: ExecutionSpec): ExecutionDiagnostic[] {
|
||||
const policy = spec.resourcePolicy;
|
||||
if (!policy) return [];
|
||||
|
||||
const unavailable = [
|
||||
policy.memoryBytes?.enforcement === 'best_effort' ? 'memoryLimit' : null,
|
||||
policy.cpuMillisPerSecond?.enforcement === 'best_effort'
|
||||
? 'cpuLimit'
|
||||
: null,
|
||||
policy.filesystemIsolation === 'best_effort' ? 'filesystemIsolation' : null,
|
||||
policy.networkIsolation === 'best_effort' ? 'networkIsolation' : null,
|
||||
].filter((value): value is string => value !== null);
|
||||
return unavailable.length === 0
|
||||
? []
|
||||
: [
|
||||
{
|
||||
code: 'RESOURCE_POLICY_BEST_EFFORT_UNAVAILABLE',
|
||||
summary: `Best-effort capabilities were unavailable: ${unavailable.join(
|
||||
', ',
|
||||
)}`,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function assertExecutionSpec(spec: ExecutionSpec): void {
|
||||
assertDomainExecutionSpec(spec);
|
||||
assertResourcePolicy(spec);
|
||||
}
|
||||
|
||||
function environmentBytes(environment: NodeJS.ProcessEnv): number {
|
||||
return Object.entries(environment).reduce(
|
||||
(total, [key, value]) =>
|
||||
total +
|
||||
Buffer.byteLength(key, 'utf8') +
|
||||
Buffer.byteLength(value ?? '', 'utf8'),
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
function buildEnvironment(
|
||||
policy: ExecutionSpec['environmentPolicy'],
|
||||
supplied: Readonly<Record<string, string>>,
|
||||
): NodeJS.ProcessEnv {
|
||||
if (Object.keys(supplied).length > MAX_EXECUTION_ENVIRONMENT_ENTRIES) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'execution environment has too many entries',
|
||||
);
|
||||
}
|
||||
|
||||
const environment: NodeJS.ProcessEnv = {};
|
||||
if (policy === 'inherit') {
|
||||
Object.assign(environment, process.env);
|
||||
} else {
|
||||
for (const key of ISOLATED_ENVIRONMENT_KEYS) {
|
||||
if (process.env[key] !== undefined) environment[key] = process.env[key];
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(supplied)) {
|
||||
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || value.includes('\0')) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'execution environment contains an invalid key or NUL value',
|
||||
);
|
||||
}
|
||||
environment[key] = value;
|
||||
}
|
||||
|
||||
if (environmentBytes(environment) > MAX_EXECUTION_ENVIRONMENT_BYTES) {
|
||||
throw new InvalidExecutionSpecError('execution environment is too large');
|
||||
}
|
||||
return environment;
|
||||
}
|
||||
|
||||
function diagnosticOnce(
|
||||
lifecycle: LocalExecutionLifecycle,
|
||||
diagnostic: ExecutionDiagnostic,
|
||||
): void {
|
||||
if (!lifecycle.diagnostics.some((item) => item.code === diagnostic.code)) {
|
||||
lifecycle.diagnostics.push(diagnostic);
|
||||
}
|
||||
}
|
||||
|
||||
function createResult(
|
||||
lifecycle: LocalExecutionLifecycle,
|
||||
code: number | null,
|
||||
signal: NodeJS.Signals | null,
|
||||
finishedAtMs: number,
|
||||
): ExecutionResult {
|
||||
const base = {
|
||||
startedAtMs: lifecycle.startedAtMs,
|
||||
finishedAtMs: Math.max(finishedAtMs, lifecycle.startedAtMs),
|
||||
...(code === null ? {} : { exitCode: code }),
|
||||
...(signal === null ? {} : { signal }),
|
||||
...(lifecycle.diagnostics.length === 0
|
||||
? {}
|
||||
: { diagnostics: [...lifecycle.diagnostics] }),
|
||||
};
|
||||
|
||||
if (lifecycle.terminationReason?.kind === 'timeout') {
|
||||
return {
|
||||
...base,
|
||||
outcome: 'timed_out',
|
||||
errorCode: 'EXECUTION_TIMED_OUT',
|
||||
errorSummary: 'Execution exceeded its configured timeout',
|
||||
};
|
||||
}
|
||||
if (lifecycle.terminationReason) {
|
||||
return {
|
||||
...base,
|
||||
outcome: 'cancelled',
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled',
|
||||
};
|
||||
}
|
||||
if (lifecycle.runtimeError) {
|
||||
return {
|
||||
...base,
|
||||
outcome: 'failed',
|
||||
errorCode: 'PROCESS_RUNTIME_ERROR',
|
||||
errorSummary: 'The child process reported a runtime error',
|
||||
};
|
||||
}
|
||||
if (code === 0) return { ...base, outcome: 'succeeded' };
|
||||
if (code !== null) {
|
||||
return {
|
||||
...base,
|
||||
outcome: 'failed',
|
||||
errorCode: 'PROCESS_EXIT_NON_ZERO',
|
||||
errorSummary: `Process exited with code ${code}`,
|
||||
};
|
||||
}
|
||||
if (signal !== null) {
|
||||
return {
|
||||
...base,
|
||||
outcome: 'failed',
|
||||
errorCode: 'PROCESS_SIGNALLED',
|
||||
errorSummary: `Process exited after signal ${signal}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
outcome: 'failed',
|
||||
errorCode: 'PROCESS_EXIT_UNKNOWN',
|
||||
errorSummary: 'Process exited without an exit code or signal',
|
||||
};
|
||||
}
|
||||
|
||||
export class LocalProcessExecutor implements Executor {
|
||||
readonly type = 'local_process' as const;
|
||||
|
||||
private readonly clock: ExecutorClock;
|
||||
private readonly createHandleId: () => string;
|
||||
private readonly terminator: ProcessTerminator;
|
||||
private readonly identityProvider: LocalProcessIdentityProvider;
|
||||
private readonly durableLauncherPath?: string;
|
||||
private readonly states = new WeakMap<ExecutionHandle, LocalExecutionState>();
|
||||
|
||||
constructor(options: LocalProcessExecutorOptions = {}) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.createHandleId = options.createHandleId ?? uuidV7;
|
||||
this.terminator = options.terminator ?? new PosixProcessTerminator();
|
||||
this.identityProvider =
|
||||
options.identityProvider ?? new LinuxProcProcessIdentityProvider();
|
||||
this.durableLauncherPath = options.durableLauncherPath;
|
||||
}
|
||||
|
||||
capabilities(): ExecutorCapabilities {
|
||||
return LOCAL_PROCESS_CAPABILITIES;
|
||||
}
|
||||
|
||||
async start(
|
||||
spec: ExecutionSpec,
|
||||
context: ExecutionContext,
|
||||
): Promise<ExecutionHandle> {
|
||||
assertExecutionSpec(spec);
|
||||
const environment = buildEnvironment(
|
||||
spec.environmentPolicy,
|
||||
context.environment,
|
||||
);
|
||||
const handleId = this.createHandleId();
|
||||
assertHandleIdentifier(handleId);
|
||||
if (context.signal?.aborted) {
|
||||
throw new ExecutorStartError(
|
||||
new Error('Execution was aborted before spawn'),
|
||||
);
|
||||
}
|
||||
if (
|
||||
context.signal &&
|
||||
(!context.signal.addEventListener || !context.signal.removeEventListener)
|
||||
) {
|
||||
throw new InvalidExecutionSpecError(
|
||||
'Execution abort signal must support event listeners',
|
||||
);
|
||||
}
|
||||
|
||||
const processGroup = process.platform !== 'win32';
|
||||
const durableOutput = durableLocalProcessOutput(context.output);
|
||||
let durableLaunch: DurableLocalProcessLaunch | undefined;
|
||||
if (durableOutput) {
|
||||
durableLaunch = await prepareDurableLocalProcessLaunch(
|
||||
spec,
|
||||
context,
|
||||
environment,
|
||||
durableOutput,
|
||||
this.durableLauncherPath,
|
||||
this.clock.now(),
|
||||
);
|
||||
}
|
||||
const options: SpawnOptions = {
|
||||
cwd: spec.workingDirectory,
|
||||
env: durableLaunch?.environment ?? environment,
|
||||
detached: processGroup,
|
||||
stdio: durableLaunch
|
||||
? [
|
||||
'ignore',
|
||||
durableLaunch.outputDescriptor,
|
||||
durableLaunch.outputDescriptor,
|
||||
]
|
||||
: ['ignore', 'pipe', 'pipe'],
|
||||
};
|
||||
let child: ChildProcess;
|
||||
try {
|
||||
child = durableLaunch
|
||||
? spawn(durableLaunch.file, [...durableLaunch.args], options)
|
||||
: spec.command.kind === 'argv'
|
||||
? spawn(spec.command.file, [...spec.command.args], options)
|
||||
: spawn(spec.command.command, {
|
||||
...options,
|
||||
shell: spec.command.shell ?? DEFAULT_POSIX_SHELL,
|
||||
});
|
||||
} catch (error) {
|
||||
await durableLaunch?.closeParentOutput().catch(() => undefined);
|
||||
throw new ExecutorStartError(error);
|
||||
}
|
||||
|
||||
const lifecycle: LocalExecutionLifecycle = {
|
||||
startedAtMs: 0,
|
||||
closedObserved: false,
|
||||
finished: false,
|
||||
runtimeError: false,
|
||||
diagnostics: resourcePolicyDiagnostics(spec),
|
||||
};
|
||||
let resolveClosed: () => void = () => undefined;
|
||||
const closed = new Promise<void>((resolve) => {
|
||||
resolveClosed = resolve;
|
||||
});
|
||||
const outputPumps = durableLaunch
|
||||
? []
|
||||
: [
|
||||
this.pumpOutput(child.stdout, 'stdout', context, lifecycle),
|
||||
this.pumpOutput(child.stderr, 'stderr', context, lifecycle),
|
||||
];
|
||||
|
||||
let spawnConfirmed = false;
|
||||
const spawned = new Promise<void>((resolve, reject) => {
|
||||
child.once('spawn', () => {
|
||||
spawnConfirmed = true;
|
||||
lifecycle.startedAtMs = this.clock.now();
|
||||
resolve();
|
||||
});
|
||||
child.on('error', (error) => {
|
||||
if (!spawnConfirmed) reject(error);
|
||||
else lifecycle.runtimeError = true;
|
||||
});
|
||||
});
|
||||
|
||||
const completion = new Promise<ExecutionResult>((resolve) => {
|
||||
child.once('close', (code, signal) => {
|
||||
lifecycle.closedObserved = true;
|
||||
resolveClosed();
|
||||
void Promise.all(outputPumps).then(() => {
|
||||
if (lifecycle.timeout) clearTimeout(lifecycle.timeout);
|
||||
lifecycle.removeAbortListener?.();
|
||||
const result = createResult(
|
||||
lifecycle,
|
||||
code,
|
||||
signal,
|
||||
this.clock.now(),
|
||||
);
|
||||
lifecycle.result = result;
|
||||
lifecycle.finished = true;
|
||||
resolve(result);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
await spawned;
|
||||
} catch (error) {
|
||||
await durableLaunch?.closeParentOutput().catch(() => undefined);
|
||||
throw new ExecutorStartError(error);
|
||||
}
|
||||
await durableLaunch?.closeParentOutput().catch(() => undefined);
|
||||
if (!child.pid) {
|
||||
throw new ExecutorStartError(new Error('Spawn did not return a PID'));
|
||||
}
|
||||
|
||||
let durableHandle: string | undefined;
|
||||
try {
|
||||
const identity = await this.identityProvider.capture(child.pid);
|
||||
if (identity) {
|
||||
durableHandle = createLocalProcessDurableHandle(handleId, identity);
|
||||
}
|
||||
} catch {
|
||||
// Recovery identity is optional; the Reconciler will conservatively mark
|
||||
// an unprovable execution lost and will never signal by PID alone.
|
||||
}
|
||||
|
||||
const handle: ExecutionHandle = {
|
||||
id: handleId,
|
||||
...(durableHandle === undefined ? {} : { durableHandle }),
|
||||
executorType: this.type,
|
||||
runId: spec.runId,
|
||||
attemptId: spec.attemptId,
|
||||
startedAtMs: lifecycle.startedAtMs,
|
||||
pid: child.pid,
|
||||
completion,
|
||||
};
|
||||
const state: LocalExecutionState = {
|
||||
child,
|
||||
processGroup,
|
||||
graceMs: spec.terminationGraceMs,
|
||||
closed,
|
||||
lifecycle,
|
||||
};
|
||||
this.states.set(handle, state);
|
||||
|
||||
if (!lifecycle.closedObserved && spec.timeoutMs !== undefined) {
|
||||
lifecycle.timeout = setTimeout(() => {
|
||||
void this.stop(handle, {
|
||||
kind: 'timeout',
|
||||
requestedAtMs: this.clock.now(),
|
||||
}).catch(() => {
|
||||
diagnosticOnce(lifecycle, {
|
||||
code: 'TIMEOUT_STOP_FAILED',
|
||||
summary: 'Executor could not stop the process after timeout',
|
||||
});
|
||||
});
|
||||
}, spec.timeoutMs);
|
||||
lifecycle.timeout.unref?.();
|
||||
}
|
||||
|
||||
if (!lifecycle.closedObserved && context.signal) {
|
||||
const onAbort = () => {
|
||||
void this.stop(handle, {
|
||||
kind: 'user',
|
||||
requestedAtMs: this.clock.now(),
|
||||
}).catch(() => {
|
||||
diagnosticOnce(lifecycle, {
|
||||
code: 'ABORT_STOP_FAILED',
|
||||
summary: 'Executor could not stop the process after abort',
|
||||
});
|
||||
});
|
||||
};
|
||||
context.signal.addEventListener!('abort', onAbort, { once: true });
|
||||
lifecycle.removeAbortListener = () =>
|
||||
context.signal?.removeEventListener?.('abort', onAbort);
|
||||
if (context.signal.aborted) onAbort();
|
||||
}
|
||||
|
||||
return handle;
|
||||
}
|
||||
|
||||
async stop(
|
||||
handle: ExecutionHandle,
|
||||
reason: ExecutionStopReason,
|
||||
): Promise<ExecutionStopResult> {
|
||||
const state = this.states.get(handle);
|
||||
if (!state) throw new ExecutorHandleNotFoundError(handle.id);
|
||||
if (state.lifecycle.closedObserved) {
|
||||
return {
|
||||
status: 'already_exited',
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
if (state.stopPromise) return state.stopPromise;
|
||||
|
||||
state.lifecycle.terminationReason = reason;
|
||||
state.stopPromise = this.terminator
|
||||
.terminate({
|
||||
pid: state.child.pid!,
|
||||
processGroup: state.processGroup,
|
||||
graceMs: state.graceMs,
|
||||
closed: state.closed,
|
||||
})
|
||||
.then((result) => ({
|
||||
status: result.alreadyExited
|
||||
? ('already_exited' as const)
|
||||
: ('termination_requested' as const),
|
||||
termSignalSent: result.termSignalSent,
|
||||
killSignalSent: result.killSignalSent,
|
||||
}));
|
||||
return state.stopPromise;
|
||||
}
|
||||
|
||||
async inspect(handle: ExecutionHandle): Promise<ExecutionInspection> {
|
||||
const state = this.states.get(handle);
|
||||
if (!state) throw new ExecutorHandleNotFoundError(handle.id);
|
||||
if (state.lifecycle.closedObserved) {
|
||||
return {
|
||||
status: 'exited',
|
||||
...(state.lifecycle.result === undefined
|
||||
? {}
|
||||
: { result: state.lifecycle.result }),
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: state.stopPromise ? 'stopping' : 'running',
|
||||
};
|
||||
}
|
||||
|
||||
private async pumpOutput(
|
||||
stream: Readable | null,
|
||||
outputStream: ExecutionOutputStream,
|
||||
context: ExecutionContext,
|
||||
lifecycle: LocalExecutionLifecycle,
|
||||
): Promise<void> {
|
||||
if (!stream) return;
|
||||
let sinkAvailable = true;
|
||||
try {
|
||||
for await (const value of stream) {
|
||||
if (!sinkAvailable) continue;
|
||||
try {
|
||||
const chunk =
|
||||
value instanceof Uint8Array ? value : Buffer.from(String(value));
|
||||
await context.output.write({
|
||||
stream: outputStream,
|
||||
chunk,
|
||||
observedAtMs: this.clock.now(),
|
||||
});
|
||||
} catch {
|
||||
sinkAvailable = false;
|
||||
diagnosticOnce(lifecycle, {
|
||||
code: 'OUTPUT_SINK_FAILED',
|
||||
summary: 'Execution output sink failed; output may be incomplete',
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
diagnosticOnce(lifecycle, {
|
||||
code: 'OUTPUT_STREAM_FAILED',
|
||||
summary: 'Execution output stream failed; output may be incomplete',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
import { readFile } from 'fs/promises';
|
||||
import type {
|
||||
PersistedExecutionInspection,
|
||||
PersistedExecutionInspector,
|
||||
} from '../../ports/persistedExecutionInspector';
|
||||
|
||||
export const LOCAL_PROCESS_DURABLE_HANDLE_PREFIX = 'ql3lp1.';
|
||||
export const MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES = 512;
|
||||
|
||||
const LINUX_BOOT_ID_PATH = '/proc/sys/kernel/random/boot_id';
|
||||
|
||||
export interface LinuxProcessIdentity {
|
||||
platform: 'linux';
|
||||
bootId: string;
|
||||
pid: number;
|
||||
processGroupId: number;
|
||||
startTimeTicks: string;
|
||||
}
|
||||
|
||||
interface LinuxProcessSnapshot extends LinuxProcessIdentity {
|
||||
state: string;
|
||||
}
|
||||
|
||||
export interface LocalProcessIdentityProvider {
|
||||
capture(pid: number): Promise<LinuxProcessIdentity | null>;
|
||||
inspect(
|
||||
identity: LinuxProcessIdentity,
|
||||
): Promise<PersistedExecutionInspection>;
|
||||
}
|
||||
|
||||
export interface LinuxProcProcessIdentityProviderOptions {
|
||||
platform?: NodeJS.Platform;
|
||||
readTextFile?: (path: string) => Promise<string>;
|
||||
}
|
||||
|
||||
interface DurableHandlePayload {
|
||||
v: 1;
|
||||
h: string;
|
||||
b: string;
|
||||
p: number;
|
||||
g: number;
|
||||
s: string;
|
||||
}
|
||||
|
||||
function isMissingFileError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
['ENOENT', 'ESRCH'].includes((error as NodeJS.ErrnoException).code ?? '')
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeBootId(value: string): string {
|
||||
const bootId = value.trim();
|
||||
if (!/^[A-Za-z0-9-]{1,64}$/.test(bootId)) {
|
||||
throw new Error('Linux boot id has an invalid format');
|
||||
}
|
||||
return bootId;
|
||||
}
|
||||
|
||||
function assertPositiveSafeInteger(value: number, name: string): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new Error(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertPositiveStartTimeTicks(value: string): void {
|
||||
if (!/^\d{1,32}$/.test(value) || BigInt(value) < BigInt(1)) {
|
||||
throw new Error('Linux process start time has an invalid format');
|
||||
}
|
||||
}
|
||||
|
||||
function parseLinuxProcStat(pid: number, value: string): LinuxProcessSnapshot {
|
||||
assertPositiveSafeInteger(pid, 'pid');
|
||||
const open = value.indexOf('(');
|
||||
const close = value.lastIndexOf(')');
|
||||
if (open < 1 || close <= open) {
|
||||
throw new Error('Linux process stat has an invalid command field');
|
||||
}
|
||||
const observedPid = Number(value.slice(0, open).trim());
|
||||
if (observedPid !== pid) {
|
||||
throw new Error('Linux process stat PID does not match the requested PID');
|
||||
}
|
||||
|
||||
// Values after comm begin at field 3 (state); starttime is field 22.
|
||||
const fields = value
|
||||
.slice(close + 1)
|
||||
.trim()
|
||||
.split(/\s+/);
|
||||
if (fields.length < 20) {
|
||||
throw new Error('Linux process stat is missing identity fields');
|
||||
}
|
||||
const state = fields[0];
|
||||
const processGroupId = Number(fields[2]);
|
||||
const startTimeTicks = fields[19];
|
||||
assertPositiveSafeInteger(processGroupId, 'processGroupId');
|
||||
assertPositiveStartTimeTicks(startTimeTicks);
|
||||
return {
|
||||
platform: 'linux',
|
||||
bootId: '',
|
||||
pid,
|
||||
processGroupId,
|
||||
startTimeTicks,
|
||||
state,
|
||||
};
|
||||
}
|
||||
|
||||
function validateIdentity(identity: LinuxProcessIdentity): void {
|
||||
if (identity.platform !== 'linux') {
|
||||
throw new Error('Local process identity has an unsupported platform');
|
||||
}
|
||||
normalizeBootId(identity.bootId);
|
||||
assertPositiveSafeInteger(identity.pid, 'pid');
|
||||
assertPositiveSafeInteger(identity.processGroupId, 'processGroupId');
|
||||
assertPositiveStartTimeTicks(identity.startTimeTicks);
|
||||
}
|
||||
|
||||
export function createLocalProcessDurableHandle(
|
||||
handleId: string,
|
||||
identity: LinuxProcessIdentity,
|
||||
): string {
|
||||
if (!handleId || handleId.length > 255 || handleId.includes('\0')) {
|
||||
throw new Error('Local process handle id has an invalid format');
|
||||
}
|
||||
validateIdentity(identity);
|
||||
const payload: DurableHandlePayload = {
|
||||
v: 1,
|
||||
h: handleId,
|
||||
b: identity.bootId,
|
||||
p: identity.pid,
|
||||
g: identity.processGroupId,
|
||||
s: identity.startTimeTicks,
|
||||
};
|
||||
const durableHandle = `${LOCAL_PROCESS_DURABLE_HANDLE_PREFIX}${Buffer.from(
|
||||
JSON.stringify(payload),
|
||||
'utf8',
|
||||
).toString('base64url')}`;
|
||||
if (
|
||||
Buffer.byteLength(durableHandle, 'utf8') >
|
||||
MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES
|
||||
) {
|
||||
throw new Error('Local process durable handle exceeds its size limit');
|
||||
}
|
||||
return durableHandle;
|
||||
}
|
||||
|
||||
export function parseLocalProcessDurableHandle(
|
||||
durableHandle: string,
|
||||
): { handleId: string; identity: LinuxProcessIdentity } | null {
|
||||
if (
|
||||
!durableHandle.startsWith(LOCAL_PROCESS_DURABLE_HANDLE_PREFIX) ||
|
||||
Buffer.byteLength(durableHandle, 'utf8') >
|
||||
MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const encoded = durableHandle.slice(
|
||||
LOCAL_PROCESS_DURABLE_HANDLE_PREFIX.length,
|
||||
);
|
||||
if (!encoded || !/^[A-Za-z0-9_-]+$/.test(encoded)) return null;
|
||||
|
||||
try {
|
||||
const payload = JSON.parse(
|
||||
Buffer.from(encoded, 'base64url').toString('utf8'),
|
||||
) as Partial<DurableHandlePayload>;
|
||||
if (
|
||||
payload.v !== 1 ||
|
||||
typeof payload.h !== 'string' ||
|
||||
typeof payload.b !== 'string' ||
|
||||
typeof payload.p !== 'number' ||
|
||||
typeof payload.g !== 'number' ||
|
||||
typeof payload.s !== 'string'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const identity: LinuxProcessIdentity = {
|
||||
platform: 'linux',
|
||||
bootId: payload.b,
|
||||
pid: payload.p,
|
||||
processGroupId: payload.g,
|
||||
startTimeTicks: payload.s,
|
||||
};
|
||||
if (!payload.h || payload.h.length > 255 || payload.h.includes('\0')) {
|
||||
return null;
|
||||
}
|
||||
validateIdentity(identity);
|
||||
return { handleId: payload.h, identity };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class LinuxProcProcessIdentityProvider
|
||||
implements LocalProcessIdentityProvider
|
||||
{
|
||||
private readonly platform: NodeJS.Platform;
|
||||
private readonly readTextFile: (path: string) => Promise<string>;
|
||||
|
||||
constructor(options: LinuxProcProcessIdentityProviderOptions = {}) {
|
||||
this.platform = options.platform ?? process.platform;
|
||||
this.readTextFile =
|
||||
options.readTextFile ?? ((path) => readFile(path, { encoding: 'utf8' }));
|
||||
}
|
||||
|
||||
async capture(pid: number): Promise<LinuxProcessIdentity | null> {
|
||||
if (this.platform !== 'linux') return null;
|
||||
try {
|
||||
const [bootIdValue, statValue] = await Promise.all([
|
||||
this.readTextFile(LINUX_BOOT_ID_PATH),
|
||||
this.readTextFile(`/proc/${pid}/stat`),
|
||||
]);
|
||||
const snapshot = parseLinuxProcStat(pid, statValue);
|
||||
return {
|
||||
platform: 'linux',
|
||||
bootId: normalizeBootId(bootIdValue),
|
||||
pid,
|
||||
processGroupId: snapshot.processGroupId,
|
||||
startTimeTicks: snapshot.startTimeTicks,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isMissingFileError(error)) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async inspect(
|
||||
identity: LinuxProcessIdentity,
|
||||
): Promise<PersistedExecutionInspection> {
|
||||
if (this.platform !== 'linux') return { status: 'unsupported' };
|
||||
validateIdentity(identity);
|
||||
|
||||
let bootId: string;
|
||||
try {
|
||||
bootId = normalizeBootId(await this.readTextFile(LINUX_BOOT_ID_PATH));
|
||||
} catch (error) {
|
||||
if (isMissingFileError(error)) return { status: 'unsupported' };
|
||||
throw error;
|
||||
}
|
||||
if (bootId !== identity.bootId) return { status: 'identity_mismatch' };
|
||||
|
||||
let snapshot: LinuxProcessSnapshot;
|
||||
try {
|
||||
snapshot = parseLinuxProcStat(
|
||||
identity.pid,
|
||||
await this.readTextFile(`/proc/${identity.pid}/stat`),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isMissingFileError(error)) return { status: 'exited' };
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
snapshot.processGroupId !== identity.processGroupId ||
|
||||
snapshot.startTimeTicks !== identity.startTimeTicks
|
||||
) {
|
||||
return { status: 'identity_mismatch' };
|
||||
}
|
||||
if (['Z', 'X', 'x'].includes(snapshot.state)) return { status: 'exited' };
|
||||
return { status: 'running' };
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalProcessPersistedExecutionInspector
|
||||
implements PersistedExecutionInspector
|
||||
{
|
||||
readonly executorType = 'local_process' as const;
|
||||
|
||||
constructor(
|
||||
private readonly identityProvider: LocalProcessIdentityProvider = new LinuxProcProcessIdentityProvider(),
|
||||
) {}
|
||||
|
||||
async inspect(durableHandle: string): Promise<PersistedExecutionInspection> {
|
||||
const parsed = parseLocalProcessDurableHandle(durableHandle);
|
||||
if (!parsed) return { status: 'invalid' };
|
||||
return {
|
||||
...(await this.identityProvider.inspect(parsed.identity)),
|
||||
identityPid: parsed.identity.pid,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
import { ExecutorStopError } from '../../domain/executorErrors';
|
||||
import type {
|
||||
PersistedExecutionController,
|
||||
PersistedExecutionStopResult,
|
||||
PersistedExecutionStopStatus,
|
||||
} from '../../ports/persistedExecutionController';
|
||||
import {
|
||||
LinuxProcProcessIdentityProvider,
|
||||
parseLocalProcessDurableHandle,
|
||||
type LocalProcessIdentityProvider,
|
||||
type LinuxProcessIdentity,
|
||||
} from './localProcessIdentity';
|
||||
|
||||
export const MAX_PERSISTED_LOCAL_STOP_GRACE_MS = 60_000;
|
||||
|
||||
export type PersistedLocalProcessSignalSender = (
|
||||
pid: number,
|
||||
signal: NodeJS.Signals,
|
||||
) => void;
|
||||
|
||||
export interface PersistedLocalProcessControllerOptions {
|
||||
identityProvider?: LocalProcessIdentityProvider;
|
||||
sendSignal?: PersistedLocalProcessSignalSender;
|
||||
graceMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
sleep?: (delayMs: number) => Promise<void>;
|
||||
}
|
||||
|
||||
function isNoSuchProcessError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
(error as NodeJS.ErrnoException).code === 'ESRCH'
|
||||
);
|
||||
}
|
||||
|
||||
function withoutSignal(status: PersistedExecutionStopStatus) {
|
||||
return { status, termSignalSent: false, killSignalSent: false } as const;
|
||||
}
|
||||
|
||||
function mappedInspectionStatus(
|
||||
status: 'identity_mismatch' | 'unsupported' | 'invalid',
|
||||
termSignalSent: boolean,
|
||||
): PersistedExecutionStopResult {
|
||||
return {
|
||||
status,
|
||||
termSignalSent,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
|
||||
export class LocalProcessPersistedExecutionController
|
||||
implements PersistedExecutionController
|
||||
{
|
||||
readonly executorType = 'local_process' as const;
|
||||
private readonly identityProvider: LocalProcessIdentityProvider;
|
||||
private readonly sendSignal: PersistedLocalProcessSignalSender;
|
||||
private readonly graceMs: number;
|
||||
private readonly pollIntervalMs: number;
|
||||
private readonly sleep: (delayMs: number) => Promise<void>;
|
||||
|
||||
constructor(options: PersistedLocalProcessControllerOptions = {}) {
|
||||
this.identityProvider =
|
||||
options.identityProvider ?? new LinuxProcProcessIdentityProvider();
|
||||
this.sendSignal = options.sendSignal ?? process.kill;
|
||||
this.graceMs = options.graceMs ?? 5_000;
|
||||
this.pollIntervalMs = options.pollIntervalMs ?? 50;
|
||||
this.sleep =
|
||||
options.sleep ??
|
||||
((delayMs) =>
|
||||
new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
}));
|
||||
if (
|
||||
!Number.isSafeInteger(this.graceMs) ||
|
||||
this.graceMs < 0 ||
|
||||
this.graceMs > MAX_PERSISTED_LOCAL_STOP_GRACE_MS
|
||||
) {
|
||||
throw new RangeError('Persisted local stop graceMs is invalid');
|
||||
}
|
||||
if (!Number.isSafeInteger(this.pollIntervalMs) || this.pollIntervalMs < 1) {
|
||||
throw new RangeError(
|
||||
'Persisted local stop pollIntervalMs must be a positive integer',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async stop({
|
||||
durableHandle,
|
||||
expectedPid,
|
||||
}: Parameters<
|
||||
PersistedExecutionController['stop']
|
||||
>[0]): Promise<PersistedExecutionStopResult> {
|
||||
const parsed = parseLocalProcessDurableHandle(durableHandle);
|
||||
if (!parsed) return withoutSignal('invalid');
|
||||
const identity = parsed.identity;
|
||||
if (expectedPid !== undefined && expectedPid !== identity.pid) {
|
||||
return withoutSignal('pid_mismatch');
|
||||
}
|
||||
// LocalProcessExecutor uses a detached child as process-group leader.
|
||||
if (identity.processGroupId !== identity.pid) {
|
||||
return withoutSignal('identity_mismatch');
|
||||
}
|
||||
|
||||
const initial = await this.identityProvider.inspect(identity);
|
||||
if (initial.status === 'exited') return withoutSignal('already_exited');
|
||||
if (initial.status !== 'running') {
|
||||
return mappedInspectionStatus(initial.status, false);
|
||||
}
|
||||
|
||||
if (!this.trySignal(identity, 'SIGTERM')) {
|
||||
return withoutSignal('already_exited');
|
||||
}
|
||||
|
||||
let waitedMs = 0;
|
||||
while (waitedMs < this.graceMs) {
|
||||
const delayMs = Math.min(this.pollIntervalMs, this.graceMs - waitedMs);
|
||||
await this.sleep(delayMs);
|
||||
waitedMs += delayMs;
|
||||
const inspection = await this.identityProvider.inspect(identity);
|
||||
if (inspection.status === 'exited') {
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
if (inspection.status !== 'running') {
|
||||
return mappedInspectionStatus(inspection.status, true);
|
||||
}
|
||||
}
|
||||
|
||||
const finalInspection = await this.identityProvider.inspect(identity);
|
||||
if (finalInspection.status === 'exited') {
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
if (finalInspection.status !== 'running') {
|
||||
return mappedInspectionStatus(finalInspection.status, true);
|
||||
}
|
||||
const killSignalSent = this.trySignal(identity, 'SIGKILL');
|
||||
return {
|
||||
status: 'termination_requested',
|
||||
termSignalSent: true,
|
||||
killSignalSent,
|
||||
};
|
||||
}
|
||||
|
||||
private trySignal(
|
||||
identity: LinuxProcessIdentity,
|
||||
signal: NodeJS.Signals,
|
||||
): boolean {
|
||||
try {
|
||||
this.sendSignal(-identity.processGroupId, signal);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isNoSuchProcessError(error)) return false;
|
||||
throw new ExecutorStopError(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { ExecutorStopError } from '../../domain/executorErrors';
|
||||
|
||||
export interface ProcessTerminationRequest {
|
||||
pid: number;
|
||||
processGroup: boolean;
|
||||
graceMs: number;
|
||||
closed: Promise<void>;
|
||||
}
|
||||
|
||||
export interface ProcessTerminationResult {
|
||||
alreadyExited: boolean;
|
||||
termSignalSent: boolean;
|
||||
killSignalSent: boolean;
|
||||
}
|
||||
|
||||
export interface ProcessTerminator {
|
||||
terminate(
|
||||
request: ProcessTerminationRequest,
|
||||
): Promise<ProcessTerminationResult>;
|
||||
}
|
||||
|
||||
export type ProcessSignalSender = (pid: number, signal: NodeJS.Signals) => void;
|
||||
|
||||
function isNoSuchProcessError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
(error as NodeJS.ErrnoException).code === 'ESRCH'
|
||||
);
|
||||
}
|
||||
|
||||
async function exitsWithin(closed: Promise<void>, timeoutMs: number) {
|
||||
if (timeoutMs === 0) return false;
|
||||
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
closed.then(() => true),
|
||||
new Promise<boolean>((resolve) => {
|
||||
timer = setTimeout(() => resolve(false), timeoutMs);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
export class PosixProcessTerminator implements ProcessTerminator {
|
||||
constructor(
|
||||
private readonly sendSignal: ProcessSignalSender = process.kill,
|
||||
) {}
|
||||
|
||||
async terminate(
|
||||
request: ProcessTerminationRequest,
|
||||
): Promise<ProcessTerminationResult> {
|
||||
const targetPid = request.processGroup ? -request.pid : request.pid;
|
||||
try {
|
||||
this.sendSignal(targetPid, 'SIGTERM');
|
||||
} catch (error) {
|
||||
if (isNoSuchProcessError(error)) {
|
||||
return {
|
||||
alreadyExited: true,
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
throw new ExecutorStopError(error);
|
||||
}
|
||||
|
||||
if (await exitsWithin(request.closed, request.graceMs)) {
|
||||
return {
|
||||
alreadyExited: false,
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
this.sendSignal(targetPid, 'SIGKILL');
|
||||
return {
|
||||
alreadyExited: false,
|
||||
termSignalSent: true,
|
||||
killSignalSent: true,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isNoSuchProcessError(error)) {
|
||||
return {
|
||||
alreadyExited: false,
|
||||
termSignalSent: true,
|
||||
killSignalSent: false,
|
||||
};
|
||||
}
|
||||
throw new ExecutorStopError(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { ArtifactReadAuthorizer } from '../../ports/artifactReadAuthorizer';
|
||||
import type { ArtifactReadAuthorizationEffect } from '../../ports/artifactReadAuthorizer';
|
||||
import type { ProjectPolicyEngine } from '../../application/projectPolicyEngine';
|
||||
|
||||
export class ProjectPolicyArtifactReadAuthorizer
|
||||
implements ArtifactReadAuthorizer
|
||||
{
|
||||
constructor(private readonly policy: Pick<ProjectPolicyEngine, 'decide'>) {}
|
||||
|
||||
async authorize(
|
||||
request: Parameters<ArtifactReadAuthorizer['authorize']>[0],
|
||||
): Promise<ArtifactReadAuthorizationEffect> {
|
||||
if (request.action !== 'artifact.read') {
|
||||
throw new TypeError('Artifact read authorization action is invalid');
|
||||
}
|
||||
const result = await this.policy.decide({
|
||||
subject: request.subject,
|
||||
projectId: request.projectId,
|
||||
permission: 'artifact.read',
|
||||
});
|
||||
return result.effect;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import {
|
||||
activateClusterControlRuntime,
|
||||
type ClusterControlActivationAudit,
|
||||
type ClusterControlActivationStack,
|
||||
type ClusterControlReadinessEvidence,
|
||||
type ClusterControlRuntimeActivationResult,
|
||||
type ClusterControlStopResult,
|
||||
} from '../../application/clusterControlRuntimeActivation';
|
||||
import type { DeploymentProfile } from '../../domain/deploymentProfile';
|
||||
import type {
|
||||
PostgresDatabaseResource,
|
||||
PostgresPool,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
assertPostgresSchemaReady,
|
||||
type PostgresSchemaReadinessReport,
|
||||
} from '../../../migrations/postgresql/schemaReadiness';
|
||||
import { PostgresRunRepository } from './runRepository';
|
||||
|
||||
export type ClusterControlDatabasePool = PostgresPool;
|
||||
|
||||
export type ClusterControlDatabaseResource = PostgresDatabaseResource;
|
||||
|
||||
export interface ClusterControlAssemblyInput {
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
readonly runs: PostgresRunRepository;
|
||||
}
|
||||
|
||||
export interface ClusterControlRuntimeBootstrapOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: DeploymentProfile;
|
||||
readonly openDatabase: () => Promise<ClusterControlDatabaseResource>;
|
||||
readonly create: (
|
||||
input: ClusterControlAssemblyInput,
|
||||
) => ClusterControlActivationStack;
|
||||
readonly audit: (
|
||||
record: ClusterControlActivationAudit,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
function readinessEvidence(
|
||||
report: PostgresSchemaReadinessReport,
|
||||
): ClusterControlReadinessEvidence {
|
||||
return Object.freeze({
|
||||
contractName: report.contractName,
|
||||
contractVersion: report.contractVersion,
|
||||
serverMajor: report.serverMajor,
|
||||
migrationIds: Object.freeze([...report.migrationIds]),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazily owns the cluster database around the readiness-first activation gate.
|
||||
* A concrete cluster package supplies openDatabase() and the pg.Pool binding;
|
||||
* disabled and wrong-profile paths never import or open the database driver.
|
||||
*/
|
||||
export async function bootstrapClusterControlRuntime(
|
||||
options: ClusterControlRuntimeBootstrapOptions,
|
||||
): Promise<ClusterControlRuntimeActivationResult> {
|
||||
let database: ClusterControlDatabaseResource | undefined;
|
||||
let closePromise: Promise<void> | undefined;
|
||||
const closeDatabase = (): Promise<void> => {
|
||||
if (!database) return Promise.resolve();
|
||||
closePromise ??= Promise.resolve().then(() => database!.close());
|
||||
return closePromise;
|
||||
};
|
||||
|
||||
try {
|
||||
const activation = await activateClusterControlRuntime({
|
||||
enabled: options.enabled,
|
||||
profile: options.profile,
|
||||
readiness: {
|
||||
async assertReady() {
|
||||
if (database) {
|
||||
throw new Error(
|
||||
'Cluster-control database was opened more than once',
|
||||
);
|
||||
}
|
||||
database = await options.openDatabase();
|
||||
return readinessEvidence(
|
||||
await assertPostgresSchemaReady(database.pool),
|
||||
);
|
||||
},
|
||||
},
|
||||
create(evidence) {
|
||||
if (!database) {
|
||||
throw new Error(
|
||||
'Cluster-control database is unavailable after readiness',
|
||||
);
|
||||
}
|
||||
return options.create({
|
||||
evidence,
|
||||
runs: new PostgresRunRepository(database.pool),
|
||||
});
|
||||
},
|
||||
audit: options.audit,
|
||||
});
|
||||
if (activation.status === 'disabled') return activation;
|
||||
|
||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
return {
|
||||
...activation,
|
||||
stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
let result: ClusterControlStopResult | undefined;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
result = await activation.stop();
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
await closeDatabase();
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return result!;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
try {
|
||||
await closeDatabase();
|
||||
} catch {
|
||||
// Preserve the readiness/assembly/activation failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,831 @@
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunAttemptStatus,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '../../domain/run';
|
||||
import {
|
||||
EXECUTION_ORIGINS,
|
||||
RUN_ATTEMPT_STATUSES,
|
||||
RUN_CANCELLATION_REASONS,
|
||||
RUN_EVENT_ACTOR_TYPES,
|
||||
RUN_STATUSES,
|
||||
} from '../../domain/run';
|
||||
import {
|
||||
assertRunRetryPolicyRecord,
|
||||
RUN_RETRY_SAFETIES,
|
||||
type RunRetryPolicyRecord,
|
||||
} from '../../domain/runRetryPolicy';
|
||||
import {
|
||||
DuplicateIdempotencyKeyError,
|
||||
DuplicateRunAttemptError,
|
||||
DuplicateRunEventError,
|
||||
RunEventPayloadTooLargeError,
|
||||
RunRepositoryBusyError,
|
||||
RunRepositoryConstraintError,
|
||||
RunRepositoryError,
|
||||
RunRepositoryOperationError,
|
||||
} from '../../domain/repositoryErrors';
|
||||
import type {
|
||||
RunRepository,
|
||||
RunRepositoryReader,
|
||||
RunRepositoryTransaction,
|
||||
} from '../../ports/runRepository';
|
||||
import type {
|
||||
PostgresClient as PostgresRunClient,
|
||||
PostgresPool as PostgresRunPool,
|
||||
PostgresQueryable as PostgresRunQueryable,
|
||||
PostgresQueryResult as PostgresRunQueryResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
|
||||
export type {
|
||||
PostgresClient as PostgresRunClient,
|
||||
PostgresPool as PostgresRunPool,
|
||||
PostgresQueryable as PostgresRunQueryable,
|
||||
PostgresQueryResult as PostgresRunQueryResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
MAX_CANCELLATION_RECOVERY_PAGE_SIZE,
|
||||
MAX_RUN_EVENT_PAGE_SIZE,
|
||||
MAX_RUN_EVENT_PAYLOAD_BYTES,
|
||||
} from '../../ports/runRepository';
|
||||
|
||||
interface ColumnDefinition {
|
||||
readonly column: string;
|
||||
readonly property: string;
|
||||
}
|
||||
|
||||
type QueryRow = Record<string, unknown>;
|
||||
|
||||
const POSTGRES_RUNTIME_STATEMENT_TIMEOUT_MS = 5_000;
|
||||
const POSTGRES_RUNTIME_LOCK_TIMEOUT_MS = 1_000;
|
||||
const POSTGRES_RUNTIME_IDLE_TRANSACTION_TIMEOUT_MS = 10_000;
|
||||
|
||||
const RUN_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
|
||||
{ column: 'id', property: 'id' },
|
||||
{ column: 'project_id', property: 'projectId' },
|
||||
{ column: 'task_id', property: 'taskId' },
|
||||
{ column: 'task_revision', property: 'taskRevision' },
|
||||
{ column: 'task_name', property: 'taskName' },
|
||||
{ column: 'task_snapshot_ref', property: 'taskSnapshotRef' },
|
||||
{ column: 'legacy_cron_id', property: 'legacyCronId' },
|
||||
{ column: 'parent_run_id', property: 'parentRunId' },
|
||||
{ column: 'retry_of_run_id', property: 'retryOfRunId' },
|
||||
{ column: 'trigger_id', property: 'triggerId' },
|
||||
{ column: 'trigger_type', property: 'triggerType' },
|
||||
{ column: 'execution_origin', property: 'executionOrigin' },
|
||||
{ column: 'execution_owner', property: 'executionOwner' },
|
||||
{ column: 'triggered_by', property: 'triggeredBy' },
|
||||
{ column: 'request_id', property: 'requestId' },
|
||||
{ column: 'scheduled_for_ms', property: 'scheduledForMs' },
|
||||
{ column: 'status', property: 'status' },
|
||||
{ column: 'version', property: 'version' },
|
||||
{ column: 'event_sequence', property: 'eventSequence' },
|
||||
{ column: 'priority', property: 'priority' },
|
||||
{ column: 'idempotency_key', property: 'idempotencyKey' },
|
||||
{ column: 'input_ref', property: 'inputRef' },
|
||||
{ column: 'output_ref', property: 'outputRef' },
|
||||
{ column: 'created_at_ms', property: 'createdAtMs' },
|
||||
{ column: 'queued_at_ms', property: 'queuedAtMs' },
|
||||
{ column: 'started_at_ms', property: 'startedAtMs' },
|
||||
{ column: 'finished_at_ms', property: 'finishedAtMs' },
|
||||
{ column: 'cancel_requested_at_ms', property: 'cancelRequestedAtMs' },
|
||||
{ column: 'cancel_reason', property: 'cancelReason' },
|
||||
{ column: 'error_code', property: 'errorCode' },
|
||||
{ column: 'error_summary', property: 'errorSummary' },
|
||||
]);
|
||||
|
||||
const ATTEMPT_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
|
||||
{ column: 'id', property: 'id' },
|
||||
{ column: 'run_id', property: 'runId' },
|
||||
{ column: 'step_run_id', property: 'stepRunId' },
|
||||
{ column: 'attempt', property: 'attempt' },
|
||||
{ column: 'status', property: 'status' },
|
||||
{ column: 'executor_type', property: 'executorType' },
|
||||
{ column: 'worker_id', property: 'workerId' },
|
||||
{ column: 'executor_handle', property: 'executorHandle' },
|
||||
{ column: 'pid', property: 'pid' },
|
||||
{ column: 'log_artifact_id', property: 'logArtifactId' },
|
||||
{ column: 'lease_token', property: 'leaseToken' },
|
||||
{ column: 'lease_expires_at_ms', property: 'leaseExpiresAtMs' },
|
||||
{ column: 'deadline_at_ms', property: 'deadlineAtMs' },
|
||||
{ column: 'callback_token_hash', property: 'callbackTokenHash' },
|
||||
{ column: 'callback_sequence', property: 'callbackSequence' },
|
||||
{ column: 'created_at_ms', property: 'createdAtMs' },
|
||||
{ column: 'started_at_ms', property: 'startedAtMs' },
|
||||
{ column: 'finished_at_ms', property: 'finishedAtMs' },
|
||||
{ column: 'exit_code', property: 'exitCode' },
|
||||
{ column: 'error_code', property: 'errorCode' },
|
||||
{ column: 'error_summary', property: 'errorSummary' },
|
||||
]);
|
||||
|
||||
const EVENT_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
|
||||
{ column: 'id', property: 'id' },
|
||||
{ column: 'run_id', property: 'runId' },
|
||||
{ column: 'sequence', property: 'sequence' },
|
||||
{ column: 'type', property: 'type' },
|
||||
{ column: 'dedupe_key', property: 'dedupeKey' },
|
||||
{ column: 'actor_type', property: 'actorType' },
|
||||
{ column: 'actor_id', property: 'actorId' },
|
||||
{ column: 'attempt_id', property: 'attemptId' },
|
||||
{ column: 'step_run_id', property: 'stepRunId' },
|
||||
{ column: 'payload', property: 'payload' },
|
||||
{ column: 'created_at_ms', property: 'createdAtMs' },
|
||||
]);
|
||||
|
||||
const RETRY_POLICY_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
|
||||
{ column: 'run_id', property: 'runId' },
|
||||
{ column: 'max_attempts', property: 'maxAttempts' },
|
||||
{ column: 'retry_on_lost', property: 'retryOnLost' },
|
||||
{ column: 'safety', property: 'safety' },
|
||||
{ column: 'backoff_base_ms', property: 'backoffBaseMs' },
|
||||
{ column: 'backoff_max_ms', property: 'backoffMaxMs' },
|
||||
{ column: 'next_attempt_at_ms', property: 'nextAttemptAtMs' },
|
||||
{ column: 'version', property: 'version' },
|
||||
{ column: 'created_at_ms', property: 'createdAtMs' },
|
||||
{ column: 'updated_at_ms', property: 'updatedAtMs' },
|
||||
]);
|
||||
|
||||
const TERMINAL_RUN_STATUSES = Object.freeze([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
]);
|
||||
|
||||
const BUSY_SQL_STATES = new Set([
|
||||
'08000',
|
||||
'08001',
|
||||
'08003',
|
||||
'08004',
|
||||
'08006',
|
||||
'08007',
|
||||
'08P01',
|
||||
'40001',
|
||||
'40P01',
|
||||
'55P03',
|
||||
'57014',
|
||||
'57P01',
|
||||
'57P02',
|
||||
'57P03',
|
||||
]);
|
||||
|
||||
const RUN_IDEMPOTENCY_CONSTRAINT = 'ql3_runs_project_idempotency_uidx';
|
||||
const ATTEMPT_NUMBER_CONSTRAINT = 'ql3_run_attempts_run_attempt_uidx';
|
||||
const EVENT_SEQUENCE_CONSTRAINT = 'ql3_run_events_run_sequence_uidx';
|
||||
const EVENT_DEDUPE_CONSTRAINT = 'ql3_run_events_run_dedupe_uidx';
|
||||
|
||||
function quoted(identifier: string): string {
|
||||
return `"${identifier}"`;
|
||||
}
|
||||
|
||||
function selectColumns(columns: readonly ColumnDefinition[]): string {
|
||||
return columns
|
||||
.map(({ column, property }) => `${quoted(column)} AS ${quoted(property)}`)
|
||||
.join(', ');
|
||||
}
|
||||
|
||||
function insertSql(
|
||||
tableName: string,
|
||||
columns: readonly ColumnDefinition[],
|
||||
): string {
|
||||
return `INSERT INTO "ql3".${quoted(tableName)} (${columns
|
||||
.map(({ column }) => quoted(column))
|
||||
.join(', ')}) VALUES (${columns
|
||||
.map((_, index) => `$${index + 1}`)
|
||||
.join(', ')})`;
|
||||
}
|
||||
|
||||
function updateSql(
|
||||
tableName: string,
|
||||
columns: readonly ColumnDefinition[],
|
||||
predicate: string,
|
||||
): string {
|
||||
const mutableColumns = columns.slice(1);
|
||||
return `UPDATE "ql3".${quoted(tableName)} SET ${mutableColumns
|
||||
.map(({ column }, index) => `${quoted(column)} = $${index + 2}`)
|
||||
.join(', ')} WHERE ${predicate} RETURNING ${quoted(columns[0].column)}`;
|
||||
}
|
||||
|
||||
function writeValues(
|
||||
record: object,
|
||||
columns: readonly ColumnDefinition[],
|
||||
): unknown[] {
|
||||
const values = record as Record<string, unknown>;
|
||||
return columns.map(({ property }) => values[property] ?? null);
|
||||
}
|
||||
|
||||
function requiredString(row: QueryRow, property: string): string {
|
||||
const value = row[property];
|
||||
if (typeof value !== 'string' || value.length === 0) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
`PostgreSQL Run row has an invalid ${property}`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalString(row: QueryRow, property: string): string | undefined {
|
||||
const value = row[property];
|
||||
if (value === null || value === undefined) return undefined;
|
||||
if (typeof value !== 'string') {
|
||||
throw new RunRepositoryConstraintError(
|
||||
`PostgreSQL Run row has an invalid ${property}`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredInteger(row: QueryRow, property: string): number {
|
||||
const value = row[property];
|
||||
if (typeof value === 'number' && Number.isSafeInteger(value)) return value;
|
||||
if (typeof value === 'string' && /^-?(0|[1-9]\d*)$/.test(value)) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isSafeInteger(parsed)) return parsed;
|
||||
}
|
||||
throw new RunRepositoryConstraintError(
|
||||
`PostgreSQL Run row has an invalid ${property}`,
|
||||
);
|
||||
}
|
||||
|
||||
function optionalInteger(row: QueryRow, property: string): number | undefined {
|
||||
if (row[property] === null || row[property] === undefined) return undefined;
|
||||
return requiredInteger(row, property);
|
||||
}
|
||||
|
||||
function requiredBoolean(row: QueryRow, property: string): boolean {
|
||||
const value = row[property];
|
||||
if (typeof value !== 'boolean') {
|
||||
throw new RunRepositoryConstraintError(
|
||||
`PostgreSQL Run row has an invalid ${property}`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredEnum<T extends string>(
|
||||
row: QueryRow,
|
||||
property: string,
|
||||
allowed: readonly T[],
|
||||
): T {
|
||||
const value = requiredString(row, property);
|
||||
if (!allowed.includes(value as T)) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
`PostgreSQL Run row has an unsupported ${property}`,
|
||||
);
|
||||
}
|
||||
return value as T;
|
||||
}
|
||||
|
||||
function assignOptional<T extends object, K extends keyof T>(
|
||||
record: T,
|
||||
key: K,
|
||||
value: T[K] | undefined,
|
||||
): void {
|
||||
if (value !== undefined) record[key] = value;
|
||||
}
|
||||
|
||||
function rowToRun(row: QueryRow): RunRecord {
|
||||
const run: RunRecord = {
|
||||
id: requiredString(row, 'id'),
|
||||
projectId: requiredString(row, 'projectId'),
|
||||
taskId: requiredString(row, 'taskId'),
|
||||
taskRevision: requiredString(row, 'taskRevision'),
|
||||
triggerType: requiredString(row, 'triggerType'),
|
||||
executionOrigin: requiredEnum(row, 'executionOrigin', EXECUTION_ORIGINS),
|
||||
executionOwner: requiredEnum(row, 'executionOwner', [
|
||||
'legacy',
|
||||
'runtime',
|
||||
] as const),
|
||||
status: requiredEnum(row, 'status', RUN_STATUSES),
|
||||
version: requiredInteger(row, 'version'),
|
||||
eventSequence: requiredInteger(row, 'eventSequence'),
|
||||
priority: requiredInteger(row, 'priority'),
|
||||
createdAtMs: requiredInteger(row, 'createdAtMs'),
|
||||
};
|
||||
assignOptional(run, 'taskName', optionalString(row, 'taskName'));
|
||||
assignOptional(
|
||||
run,
|
||||
'taskSnapshotRef',
|
||||
optionalString(row, 'taskSnapshotRef'),
|
||||
);
|
||||
assignOptional(run, 'legacyCronId', optionalInteger(row, 'legacyCronId'));
|
||||
assignOptional(run, 'parentRunId', optionalString(row, 'parentRunId'));
|
||||
assignOptional(run, 'retryOfRunId', optionalString(row, 'retryOfRunId'));
|
||||
assignOptional(run, 'triggerId', optionalString(row, 'triggerId'));
|
||||
assignOptional(run, 'triggeredBy', optionalString(row, 'triggeredBy'));
|
||||
assignOptional(run, 'requestId', optionalString(row, 'requestId'));
|
||||
assignOptional(run, 'scheduledForMs', optionalInteger(row, 'scheduledForMs'));
|
||||
assignOptional(run, 'idempotencyKey', optionalString(row, 'idempotencyKey'));
|
||||
assignOptional(run, 'inputRef', optionalString(row, 'inputRef'));
|
||||
assignOptional(run, 'outputRef', optionalString(row, 'outputRef'));
|
||||
assignOptional(run, 'queuedAtMs', optionalInteger(row, 'queuedAtMs'));
|
||||
assignOptional(run, 'startedAtMs', optionalInteger(row, 'startedAtMs'));
|
||||
assignOptional(run, 'finishedAtMs', optionalInteger(row, 'finishedAtMs'));
|
||||
assignOptional(
|
||||
run,
|
||||
'cancelRequestedAtMs',
|
||||
optionalInteger(row, 'cancelRequestedAtMs'),
|
||||
);
|
||||
if (row.cancelReason !== null && row.cancelReason !== undefined) {
|
||||
run.cancelReason = requiredEnum(
|
||||
row,
|
||||
'cancelReason',
|
||||
RUN_CANCELLATION_REASONS,
|
||||
);
|
||||
}
|
||||
assignOptional(run, 'errorCode', optionalString(row, 'errorCode'));
|
||||
assignOptional(run, 'errorSummary', optionalString(row, 'errorSummary'));
|
||||
return run;
|
||||
}
|
||||
|
||||
function rowToAttempt(row: QueryRow): RunAttemptRecord {
|
||||
const attempt: RunAttemptRecord = {
|
||||
id: requiredString(row, 'id'),
|
||||
runId: requiredString(row, 'runId'),
|
||||
attempt: requiredInteger(row, 'attempt'),
|
||||
status: requiredEnum(row, 'status', RUN_ATTEMPT_STATUSES),
|
||||
executorType: requiredString(row, 'executorType'),
|
||||
callbackSequence: requiredInteger(row, 'callbackSequence'),
|
||||
createdAtMs: requiredInteger(row, 'createdAtMs'),
|
||||
};
|
||||
assignOptional(attempt, 'stepRunId', optionalString(row, 'stepRunId'));
|
||||
assignOptional(attempt, 'workerId', optionalString(row, 'workerId'));
|
||||
assignOptional(
|
||||
attempt,
|
||||
'executorHandle',
|
||||
optionalString(row, 'executorHandle'),
|
||||
);
|
||||
assignOptional(attempt, 'pid', optionalInteger(row, 'pid'));
|
||||
assignOptional(
|
||||
attempt,
|
||||
'logArtifactId',
|
||||
optionalString(row, 'logArtifactId'),
|
||||
);
|
||||
assignOptional(attempt, 'leaseToken', optionalString(row, 'leaseToken'));
|
||||
assignOptional(
|
||||
attempt,
|
||||
'leaseExpiresAtMs',
|
||||
optionalInteger(row, 'leaseExpiresAtMs'),
|
||||
);
|
||||
assignOptional(attempt, 'deadlineAtMs', optionalInteger(row, 'deadlineAtMs'));
|
||||
assignOptional(
|
||||
attempt,
|
||||
'callbackTokenHash',
|
||||
optionalString(row, 'callbackTokenHash'),
|
||||
);
|
||||
assignOptional(attempt, 'startedAtMs', optionalInteger(row, 'startedAtMs'));
|
||||
assignOptional(attempt, 'finishedAtMs', optionalInteger(row, 'finishedAtMs'));
|
||||
assignOptional(attempt, 'exitCode', optionalInteger(row, 'exitCode'));
|
||||
assignOptional(attempt, 'errorCode', optionalString(row, 'errorCode'));
|
||||
assignOptional(attempt, 'errorSummary', optionalString(row, 'errorSummary'));
|
||||
return attempt;
|
||||
}
|
||||
|
||||
function normalizePayload(payload: unknown): Readonly<Record<string, unknown>> {
|
||||
let value = payload;
|
||||
if (typeof value === 'string') {
|
||||
try {
|
||||
value = JSON.parse(value);
|
||||
} catch (error) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'PostgreSQL RunEvent payload is invalid JSON',
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'PostgreSQL RunEvent payload is not a JSON object',
|
||||
);
|
||||
}
|
||||
return value as Readonly<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
function rowToEvent(row: QueryRow): RunEventRecord {
|
||||
const event: RunEventRecord = {
|
||||
id: requiredString(row, 'id'),
|
||||
runId: requiredString(row, 'runId'),
|
||||
sequence: requiredInteger(row, 'sequence'),
|
||||
type: requiredString(row, 'type'),
|
||||
actorType: requiredEnum(row, 'actorType', RUN_EVENT_ACTOR_TYPES),
|
||||
payload: normalizePayload(row.payload),
|
||||
createdAtMs: requiredInteger(row, 'createdAtMs'),
|
||||
};
|
||||
assignOptional(event, 'dedupeKey', optionalString(row, 'dedupeKey'));
|
||||
assignOptional(event, 'actorId', optionalString(row, 'actorId'));
|
||||
assignOptional(event, 'attemptId', optionalString(row, 'attemptId'));
|
||||
assignOptional(event, 'stepRunId', optionalString(row, 'stepRunId'));
|
||||
return event;
|
||||
}
|
||||
|
||||
function rowToRetryPolicy(row: QueryRow): RunRetryPolicyRecord {
|
||||
const policy: RunRetryPolicyRecord = {
|
||||
runId: requiredString(row, 'runId'),
|
||||
maxAttempts: requiredInteger(row, 'maxAttempts'),
|
||||
retryOnLost: requiredBoolean(row, 'retryOnLost'),
|
||||
safety: requiredEnum(row, 'safety', RUN_RETRY_SAFETIES),
|
||||
backoffBaseMs: requiredInteger(row, 'backoffBaseMs'),
|
||||
backoffMaxMs: requiredInteger(row, 'backoffMaxMs'),
|
||||
version: requiredInteger(row, 'version'),
|
||||
createdAtMs: requiredInteger(row, 'createdAtMs'),
|
||||
updatedAtMs: requiredInteger(row, 'updatedAtMs'),
|
||||
};
|
||||
assignOptional(
|
||||
policy,
|
||||
'nextAttemptAtMs',
|
||||
optionalInteger(row, 'nextAttemptAtMs'),
|
||||
);
|
||||
assertRunRetryPolicyRecord(policy);
|
||||
return policy;
|
||||
}
|
||||
|
||||
function sqlState(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
const value = (error as { code?: unknown }).code;
|
||||
return typeof value === 'string' ? value : undefined;
|
||||
}
|
||||
|
||||
function constraintName(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
const value = (error as { constraint?: unknown }).constraint;
|
||||
return typeof value === 'string' ? value : undefined;
|
||||
}
|
||||
|
||||
function mapPostgresError(error: unknown): RunRepositoryError {
|
||||
if (error instanceof RunRepositoryError) return error;
|
||||
const state = sqlState(error);
|
||||
if (state && BUSY_SQL_STATES.has(state)) {
|
||||
return new RunRepositoryBusyError(error);
|
||||
}
|
||||
if (state?.startsWith('23')) {
|
||||
return new RunRepositoryConstraintError(
|
||||
'PostgreSQL Run repository constraint violation',
|
||||
error,
|
||||
);
|
||||
}
|
||||
return new RunRepositoryOperationError(error);
|
||||
}
|
||||
|
||||
function affectedOneOrNone(result: PostgresRunQueryResult): boolean {
|
||||
const count = result.rowCount ?? result.rows.length;
|
||||
if (count === 0) return false;
|
||||
if (count === 1) return true;
|
||||
throw new RunRepositoryConstraintError(
|
||||
'PostgreSQL compare-and-set affected more than one row',
|
||||
);
|
||||
}
|
||||
|
||||
function assertEventPayloadSize(event: RunEventRecord): void {
|
||||
let serialized: string;
|
||||
try {
|
||||
serialized = JSON.stringify(event.payload);
|
||||
} catch (error) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'RunEvent payload is not JSON serializable',
|
||||
error,
|
||||
);
|
||||
}
|
||||
const bytes = Buffer.byteLength(serialized, 'utf8');
|
||||
if (bytes > MAX_RUN_EVENT_PAYLOAD_BYTES) {
|
||||
throw new RunEventPayloadTooLargeError(bytes, MAX_RUN_EVENT_PAYLOAD_BYTES);
|
||||
}
|
||||
}
|
||||
|
||||
async function queryMapped<TRow extends QueryRow = QueryRow>(
|
||||
queryable: PostgresRunQueryable,
|
||||
text: string,
|
||||
values?: readonly unknown[],
|
||||
): Promise<PostgresRunQueryResult<TRow>> {
|
||||
try {
|
||||
return await queryable.query<TRow>(text, values);
|
||||
} catch (error) {
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
|
||||
function singleRow<TRow extends QueryRow>(
|
||||
result: PostgresRunQueryResult<TRow>,
|
||||
): TRow | null {
|
||||
if (result.rows.length === 0) return null;
|
||||
if (result.rows.length !== 1) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'PostgreSQL Run repository returned duplicate identity rows',
|
||||
);
|
||||
}
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
const RUN_SELECT = selectColumns(RUN_COLUMNS);
|
||||
const ATTEMPT_SELECT = selectColumns(ATTEMPT_COLUMNS);
|
||||
const EVENT_SELECT = selectColumns(EVENT_COLUMNS);
|
||||
const RETRY_POLICY_SELECT = selectColumns(RETRY_POLICY_COLUMNS);
|
||||
|
||||
const INSERT_RUN_SQL = insertSql('runs', RUN_COLUMNS);
|
||||
const INSERT_ATTEMPT_SQL = insertSql('run_attempts', ATTEMPT_COLUMNS);
|
||||
const INSERT_EVENT_SQL = insertSql('run_events', EVENT_COLUMNS);
|
||||
const INSERT_RETRY_POLICY_SQL = insertSql(
|
||||
'run_retry_policies',
|
||||
RETRY_POLICY_COLUMNS,
|
||||
);
|
||||
const UPDATE_RUN_SQL = updateSql(
|
||||
'runs',
|
||||
RUN_COLUMNS,
|
||||
`"id" = $1 AND "version" = $${RUN_COLUMNS.length + 1}`,
|
||||
);
|
||||
const UPDATE_ATTEMPT_SQL = updateSql(
|
||||
'run_attempts',
|
||||
ATTEMPT_COLUMNS,
|
||||
`"id" = $1 AND "status" = $${
|
||||
ATTEMPT_COLUMNS.length + 1
|
||||
} AND "callback_sequence" = $${ATTEMPT_COLUMNS.length + 2}`,
|
||||
);
|
||||
const UPDATE_RETRY_POLICY_SQL = updateSql(
|
||||
'run_retry_policies',
|
||||
RETRY_POLICY_COLUMNS,
|
||||
`"run_id" = $1 AND "version" = $${RETRY_POLICY_COLUMNS.length + 1}`,
|
||||
);
|
||||
|
||||
class PostgresRunReader implements RunRepositoryReader {
|
||||
constructor(protected readonly queryable: PostgresRunQueryable) {}
|
||||
|
||||
async findRunById(runId: string): Promise<RunRecord | null> {
|
||||
const row = singleRow(
|
||||
await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${RUN_SELECT} FROM "ql3"."runs" WHERE "id" = $1`,
|
||||
[runId],
|
||||
),
|
||||
);
|
||||
return row ? rowToRun(row) : null;
|
||||
}
|
||||
|
||||
async findAttemptById(attemptId: string): Promise<RunAttemptRecord | null> {
|
||||
const row = singleRow(
|
||||
await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${ATTEMPT_SELECT} FROM "ql3"."run_attempts" WHERE "id" = $1`,
|
||||
[attemptId],
|
||||
),
|
||||
);
|
||||
return row ? rowToAttempt(row) : null;
|
||||
}
|
||||
|
||||
async findLatestAttemptByRunId(
|
||||
runId: string,
|
||||
): Promise<RunAttemptRecord | null> {
|
||||
const row = singleRow(
|
||||
await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${ATTEMPT_SELECT} FROM "ql3"."run_attempts" WHERE "run_id" = $1 ORDER BY "attempt" DESC, "id" DESC LIMIT 1`,
|
||||
[runId],
|
||||
),
|
||||
);
|
||||
return row ? rowToAttempt(row) : null;
|
||||
}
|
||||
|
||||
async findRetryPolicyByRunId(
|
||||
runId: string,
|
||||
): Promise<RunRetryPolicyRecord | null> {
|
||||
const row = singleRow(
|
||||
await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${RETRY_POLICY_SELECT} FROM "ql3"."run_retry_policies" WHERE "run_id" = $1`,
|
||||
[runId],
|
||||
),
|
||||
);
|
||||
return row ? rowToRetryPolicy(row) : null;
|
||||
}
|
||||
|
||||
async listEvents(
|
||||
runId: string,
|
||||
options: { afterSequence?: number; limit?: number } = {},
|
||||
): Promise<RunEventRecord[]> {
|
||||
const afterSequence = options.afterSequence ?? 0;
|
||||
const limit = options.limit ?? 100;
|
||||
if (!Number.isInteger(afterSequence) || afterSequence < 0) {
|
||||
throw new RangeError('afterSequence must be a non-negative integer');
|
||||
}
|
||||
if (
|
||||
!Number.isInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_RUN_EVENT_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_RUN_EVENT_PAGE_SIZE',
|
||||
);
|
||||
}
|
||||
const result = await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${EVENT_SELECT} FROM "ql3"."run_events" WHERE "run_id" = $1 AND "sequence" > $2 ORDER BY "sequence", "id" LIMIT $3`,
|
||||
[runId, afterSequence, limit],
|
||||
);
|
||||
return result.rows.map(rowToEvent);
|
||||
}
|
||||
|
||||
async listCancellationRequested(
|
||||
options: { beforeMs?: number; limit?: number } = {},
|
||||
): Promise<RunRecord[]> {
|
||||
const beforeMs = options.beforeMs;
|
||||
const limit = options.limit ?? 100;
|
||||
if (
|
||||
beforeMs !== undefined &&
|
||||
(!Number.isSafeInteger(beforeMs) || beforeMs < 0)
|
||||
) {
|
||||
throw new RangeError('beforeMs must be a non-negative safe integer');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_CANCELLATION_RECOVERY_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_CANCELLATION_RECOVERY_PAGE_SIZE',
|
||||
);
|
||||
}
|
||||
const result = await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${RUN_SELECT} FROM "ql3"."runs" WHERE "status" <> ALL($1::text[]) AND "cancel_requested_at_ms" IS NOT NULL AND ($2::bigint IS NULL OR "cancel_requested_at_ms" <= $2) ORDER BY "cancel_requested_at_ms", "id" LIMIT $3`,
|
||||
[TERMINAL_RUN_STATUSES, beforeMs ?? null, limit],
|
||||
);
|
||||
return result.rows.map(rowToRun);
|
||||
}
|
||||
}
|
||||
|
||||
export class PostgresRunTransaction
|
||||
extends PostgresRunReader
|
||||
implements RunRepositoryTransaction
|
||||
{
|
||||
async insertRun(run: RunRecord): Promise<void> {
|
||||
try {
|
||||
await this.queryable.query(INSERT_RUN_SQL, writeValues(run, RUN_COLUMNS));
|
||||
} catch (error) {
|
||||
if (
|
||||
sqlState(error) === '23505' &&
|
||||
constraintName(error) === RUN_IDEMPOTENCY_CONSTRAINT &&
|
||||
run.idempotencyKey
|
||||
) {
|
||||
throw new DuplicateIdempotencyKeyError(
|
||||
run.projectId,
|
||||
run.idempotencyKey,
|
||||
);
|
||||
}
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async insertAttempt(attempt: RunAttemptRecord): Promise<void> {
|
||||
try {
|
||||
await this.queryable.query(
|
||||
INSERT_ATTEMPT_SQL,
|
||||
writeValues(attempt, ATTEMPT_COLUMNS),
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
sqlState(error) === '23505' &&
|
||||
constraintName(error) === ATTEMPT_NUMBER_CONSTRAINT
|
||||
) {
|
||||
throw new DuplicateRunAttemptError(attempt.runId, attempt.attempt);
|
||||
}
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async insertRetryPolicy(policy: RunRetryPolicyRecord): Promise<void> {
|
||||
assertRunRetryPolicyRecord(policy);
|
||||
try {
|
||||
await this.queryable.query(
|
||||
INSERT_RETRY_POLICY_SQL,
|
||||
writeValues(policy, RETRY_POLICY_COLUMNS),
|
||||
);
|
||||
} catch (error) {
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async compareAndSetRun(
|
||||
run: RunRecord,
|
||||
expectedVersion: number,
|
||||
): Promise<boolean> {
|
||||
if (run.version !== expectedVersion + 1) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'A compare-and-set Run write must increment version exactly once',
|
||||
);
|
||||
}
|
||||
const result = await queryMapped(this.queryable, UPDATE_RUN_SQL, [
|
||||
...writeValues(run, RUN_COLUMNS),
|
||||
expectedVersion,
|
||||
]);
|
||||
return affectedOneOrNone(result);
|
||||
}
|
||||
|
||||
async compareAndSetAttempt(
|
||||
attempt: RunAttemptRecord,
|
||||
expected: {
|
||||
status: RunAttemptStatus;
|
||||
callbackSequence: number;
|
||||
},
|
||||
): Promise<boolean> {
|
||||
const result = await queryMapped(this.queryable, UPDATE_ATTEMPT_SQL, [
|
||||
...writeValues(attempt, ATTEMPT_COLUMNS),
|
||||
expected.status,
|
||||
expected.callbackSequence,
|
||||
]);
|
||||
return affectedOneOrNone(result);
|
||||
}
|
||||
|
||||
async compareAndSetRetryPolicy(
|
||||
policy: RunRetryPolicyRecord,
|
||||
expectedVersion: number,
|
||||
): Promise<boolean> {
|
||||
if (policy.version !== expectedVersion + 1) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'A compare-and-set retry policy write must increment version exactly once',
|
||||
);
|
||||
}
|
||||
assertRunRetryPolicyRecord(policy);
|
||||
const result = await queryMapped(this.queryable, UPDATE_RETRY_POLICY_SQL, [
|
||||
...writeValues(policy, RETRY_POLICY_COLUMNS),
|
||||
expectedVersion,
|
||||
]);
|
||||
return affectedOneOrNone(result);
|
||||
}
|
||||
|
||||
async appendEvent(event: RunEventRecord): Promise<void> {
|
||||
assertEventPayloadSize(event);
|
||||
try {
|
||||
await this.queryable.query(
|
||||
INSERT_EVENT_SQL,
|
||||
writeValues(event, EVENT_COLUMNS),
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
sqlState(error) === '23505' &&
|
||||
(constraintName(error) === EVENT_SEQUENCE_CONSTRAINT ||
|
||||
constraintName(error) === EVENT_DEDUPE_CONSTRAINT)
|
||||
) {
|
||||
throw new DuplicateRunEventError(event.runId, event.dedupeKey);
|
||||
}
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Driver-neutral PostgreSQL Run Repository. The cluster-only package owns the
|
||||
* concrete pg.Pool binding; edge/standalone builds never import the driver.
|
||||
*/
|
||||
export class PostgresRunRepository
|
||||
extends PostgresRunReader
|
||||
implements RunRepository
|
||||
{
|
||||
constructor(private readonly pool: PostgresRunPool) {
|
||||
super(pool);
|
||||
}
|
||||
|
||||
async transaction<T>(
|
||||
work: (transaction: RunRepositoryTransaction) => Promise<T>,
|
||||
): Promise<T> {
|
||||
let client: PostgresRunClient;
|
||||
try {
|
||||
client = await this.pool.connect();
|
||||
} catch (error) {
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
let began = false;
|
||||
let phase: 'begin' | 'work' | 'commit' = 'begin';
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
began = true;
|
||||
await client.query('SET TRANSACTION ISOLATION LEVEL READ COMMITTED');
|
||||
await client.query(`SELECT set_config('statement_timeout', $1, true)`, [
|
||||
`${POSTGRES_RUNTIME_STATEMENT_TIMEOUT_MS}ms`,
|
||||
]);
|
||||
await client.query(`SELECT set_config('lock_timeout', $1, true)`, [
|
||||
`${POSTGRES_RUNTIME_LOCK_TIMEOUT_MS}ms`,
|
||||
]);
|
||||
await client.query(
|
||||
`SELECT set_config('idle_in_transaction_session_timeout', $1, true)`,
|
||||
[`${POSTGRES_RUNTIME_IDLE_TRANSACTION_TIMEOUT_MS}ms`],
|
||||
);
|
||||
phase = 'work';
|
||||
const result = await work(new PostgresRunTransaction(client));
|
||||
phase = 'commit';
|
||||
await client.query('COMMIT');
|
||||
began = false;
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (began) {
|
||||
try {
|
||||
await client.query('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the work/commit failure; release discards broken clients.
|
||||
}
|
||||
}
|
||||
if (phase === 'work') throw error;
|
||||
throw mapPostgresError(error);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
import {
|
||||
ApprovalHumanDecisionRequiredError,
|
||||
ApprovalPolicyDeniedError,
|
||||
ApprovalRequestNotFoundError,
|
||||
ApprovalSelfDecisionError,
|
||||
approvalRequestEffectiveStatus,
|
||||
assertApprovalMutationId,
|
||||
assertApprovalReasonCode,
|
||||
assertApprovalRequestId,
|
||||
assertApprovalRequestVersion,
|
||||
assertApprovalTimestamp,
|
||||
normalizeApprovalActionBinding,
|
||||
normalizeApprovalRequestRecord,
|
||||
normalizeApprovalPolicyFence,
|
||||
sameApprovalSubject,
|
||||
type ApprovalActionBinding,
|
||||
type ApprovalDecision,
|
||||
type ApprovalRequestEffectiveStatus,
|
||||
type ApprovalRequestRecord,
|
||||
type ApprovedActionDispatchRecord,
|
||||
type ApprovalRisk,
|
||||
} from '../domain/approvalRequest';
|
||||
import {
|
||||
assertProjectPolicyProjectId,
|
||||
normalizePolicySubject,
|
||||
type PolicySubject,
|
||||
} from '../domain/projectPolicy';
|
||||
import type { ApprovalRequestRepository } from '../ports/approvalRequestRepository';
|
||||
import type { ProjectPolicyEngine } from './projectPolicyEngine';
|
||||
|
||||
export interface CreateApprovalRequestInput {
|
||||
id: string;
|
||||
projectId: string;
|
||||
action: ApprovalActionBinding;
|
||||
risk: ApprovalRisk;
|
||||
requestedBy: PolicySubject;
|
||||
requestedAtMs: number;
|
||||
expiresAtMs: number;
|
||||
}
|
||||
|
||||
export interface DecideApprovalRequestInput {
|
||||
requestId: string;
|
||||
expectedVersion: number;
|
||||
decisionId: string;
|
||||
decision: ApprovalDecision;
|
||||
reasonCode: string;
|
||||
decidedBy: PolicySubject;
|
||||
decidedAtMs: number;
|
||||
}
|
||||
|
||||
export interface ConsumeApprovalRequestInput {
|
||||
requestId: string;
|
||||
expectedVersion: number;
|
||||
consumptionId: string;
|
||||
dispatchId: string;
|
||||
action: ApprovalActionBinding;
|
||||
requestedBy: PolicySubject;
|
||||
consumedBy: PolicySubject;
|
||||
consumedAtMs: number;
|
||||
}
|
||||
|
||||
export interface ApprovalRequestView {
|
||||
request: Readonly<ApprovalRequestRecord>;
|
||||
effectiveStatus: ApprovalRequestEffectiveStatus;
|
||||
}
|
||||
|
||||
function assertExactKeys(
|
||||
name: string,
|
||||
value: object,
|
||||
expected: readonly string[],
|
||||
): void {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
keys.length !== canonical.length ||
|
||||
keys.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
throw new TypeError(`${name} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertInput(name: string, value: unknown): asserts value is object {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError(`${name} must be an object`);
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovalRequestService {
|
||||
constructor(
|
||||
private readonly repository: ApprovalRequestRepository,
|
||||
private readonly policy: ProjectPolicyEngine,
|
||||
) {}
|
||||
|
||||
async create(
|
||||
input: CreateApprovalRequestInput,
|
||||
): Promise<Readonly<ApprovalRequestRecord>> {
|
||||
assertInput('Approval create input', input);
|
||||
assertExactKeys('Approval create input', input, [
|
||||
'id',
|
||||
'projectId',
|
||||
'action',
|
||||
'risk',
|
||||
'requestedBy',
|
||||
'requestedAtMs',
|
||||
'expiresAtMs',
|
||||
]);
|
||||
const action = normalizeApprovalActionBinding(input.action);
|
||||
const requestedBy = normalizePolicySubject(input.requestedBy);
|
||||
const request = normalizeApprovalRequestRecord({
|
||||
id: input.id,
|
||||
projectId: input.projectId,
|
||||
version: 1,
|
||||
state: 'pending',
|
||||
action,
|
||||
risk: input.risk,
|
||||
requestedBy,
|
||||
requestedAtMs: input.requestedAtMs,
|
||||
expiresAtMs: input.expiresAtMs,
|
||||
decisionId: null,
|
||||
decision: null,
|
||||
decisionReasonCode: null,
|
||||
decidedBy: null,
|
||||
decidedAtMs: null,
|
||||
consumptionId: null,
|
||||
dispatchId: null,
|
||||
consumedBy: null,
|
||||
consumedAtMs: null,
|
||||
});
|
||||
const authorization = await this.policy.decideWithFence({
|
||||
projectId: request.projectId,
|
||||
subject: requestedBy,
|
||||
permission: action.permission,
|
||||
});
|
||||
if (
|
||||
authorization.decision.effect !== 'require_approval' ||
|
||||
!authorization.fence
|
||||
) {
|
||||
throw new ApprovalPolicyDeniedError();
|
||||
}
|
||||
const result = await this.repository.create({
|
||||
request,
|
||||
authorizationFence: normalizeApprovalPolicyFence(authorization.fence),
|
||||
});
|
||||
return result.request;
|
||||
}
|
||||
|
||||
async decide(
|
||||
input: DecideApprovalRequestInput,
|
||||
): Promise<Readonly<ApprovalRequestRecord>> {
|
||||
assertInput('Approval decision input', input);
|
||||
assertExactKeys('Approval decision input', input, [
|
||||
'requestId',
|
||||
'expectedVersion',
|
||||
'decisionId',
|
||||
'decision',
|
||||
'reasonCode',
|
||||
'decidedBy',
|
||||
'decidedAtMs',
|
||||
]);
|
||||
assertApprovalRequestId(input.requestId);
|
||||
assertApprovalRequestVersion(input.expectedVersion);
|
||||
assertApprovalMutationId(input.decisionId);
|
||||
assertApprovalReasonCode(input.reasonCode);
|
||||
assertApprovalTimestamp('decidedAtMs', input.decidedAtMs);
|
||||
const decidedBy = normalizePolicySubject(input.decidedBy);
|
||||
if (decidedBy.type !== 'user') {
|
||||
throw new ApprovalHumanDecisionRequiredError();
|
||||
}
|
||||
const existing = await this.repository.findById(input.requestId);
|
||||
if (!existing) throw new ApprovalRequestNotFoundError();
|
||||
const request = normalizeApprovalRequestRecord(existing);
|
||||
if (sameApprovalSubject(request.requestedBy, decidedBy)) {
|
||||
throw new ApprovalSelfDecisionError();
|
||||
}
|
||||
const authorization = await this.policy.decideWithFence({
|
||||
projectId: request.projectId,
|
||||
subject: decidedBy,
|
||||
permission: 'approval.decide',
|
||||
});
|
||||
if (authorization.decision.effect !== 'allow' || !authorization.fence) {
|
||||
throw new ApprovalPolicyDeniedError();
|
||||
}
|
||||
const result = await this.repository.decide({
|
||||
requestId: input.requestId,
|
||||
expectedVersion: input.expectedVersion,
|
||||
decisionId: input.decisionId,
|
||||
decision: input.decision,
|
||||
reasonCode: input.reasonCode,
|
||||
decidedBy,
|
||||
decidedAtMs: input.decidedAtMs,
|
||||
authorizationFence: normalizeApprovalPolicyFence(authorization.fence),
|
||||
});
|
||||
return result.request;
|
||||
}
|
||||
|
||||
async consume(input: ConsumeApprovalRequestInput): Promise<{
|
||||
request: Readonly<ApprovalRequestRecord>;
|
||||
dispatch: Readonly<ApprovedActionDispatchRecord>;
|
||||
}> {
|
||||
assertInput('Approval consumption input', input);
|
||||
assertExactKeys('Approval consumption input', input, [
|
||||
'requestId',
|
||||
'expectedVersion',
|
||||
'consumptionId',
|
||||
'dispatchId',
|
||||
'action',
|
||||
'requestedBy',
|
||||
'consumedBy',
|
||||
'consumedAtMs',
|
||||
]);
|
||||
assertApprovalRequestId(input.requestId);
|
||||
assertApprovalRequestVersion(input.expectedVersion);
|
||||
assertApprovalMutationId(input.consumptionId);
|
||||
assertApprovalMutationId(input.dispatchId);
|
||||
assertApprovalTimestamp('consumedAtMs', input.consumedAtMs);
|
||||
const action = normalizeApprovalActionBinding(input.action);
|
||||
const requestedBy = normalizePolicySubject(input.requestedBy);
|
||||
const consumedBy = normalizePolicySubject(input.consumedBy);
|
||||
if (consumedBy.type !== 'system' && consumedBy.type !== 'worker') {
|
||||
throw new ApprovalPolicyDeniedError();
|
||||
}
|
||||
const existing = await this.repository.findById(input.requestId);
|
||||
if (!existing) throw new ApprovalRequestNotFoundError();
|
||||
const request = normalizeApprovalRequestRecord(existing);
|
||||
const authorization = await this.policy.decideWithFence({
|
||||
projectId: request.projectId,
|
||||
subject: requestedBy,
|
||||
permission: action.permission,
|
||||
});
|
||||
if (
|
||||
(authorization.decision.effect !== 'allow' &&
|
||||
authorization.decision.effect !== 'require_approval') ||
|
||||
!authorization.fence
|
||||
) {
|
||||
throw new ApprovalPolicyDeniedError();
|
||||
}
|
||||
const result = await this.repository.consume({
|
||||
requestId: input.requestId,
|
||||
expectedVersion: input.expectedVersion,
|
||||
consumptionId: input.consumptionId,
|
||||
dispatchId: input.dispatchId,
|
||||
action,
|
||||
requestedBy,
|
||||
consumedBy,
|
||||
consumedAtMs: input.consumedAtMs,
|
||||
authorizationFence: normalizeApprovalPolicyFence(authorization.fence),
|
||||
});
|
||||
return Object.freeze({
|
||||
request: result.request,
|
||||
dispatch: result.dispatch,
|
||||
});
|
||||
}
|
||||
|
||||
async get(requestId: string, nowMs: number): Promise<ApprovalRequestView> {
|
||||
assertApprovalRequestId(requestId);
|
||||
assertApprovalTimestamp('nowMs', nowMs);
|
||||
const request = await this.repository.findById(requestId);
|
||||
if (!request) throw new ApprovalRequestNotFoundError();
|
||||
const normalized = normalizeApprovalRequestRecord(request);
|
||||
return Object.freeze({
|
||||
request: normalized,
|
||||
effectiveStatus: approvalRequestEffectiveStatus(normalized, nowMs),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import {
|
||||
assertApprovedActionLeaseDuration,
|
||||
assertApprovedActionLeaseIdentity,
|
||||
assertApprovedActionPageSize,
|
||||
assertApprovedActionResultCode,
|
||||
type ApprovedActionDispatchCursor,
|
||||
type ApprovedActionDispatchExecutionRecord,
|
||||
} from '../domain/approvedActionDispatchExecution';
|
||||
import type { ApprovedActionHandler } from '../ports/approvedActionHandler';
|
||||
import type { ApprovedActionDispatchRepository } from '../ports/approvedActionDispatchRepository';
|
||||
|
||||
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
||||
const DEFAULT_RETRY_BASE_MS = 1_000;
|
||||
const DEFAULT_RETRY_MAX_MS = 60_000;
|
||||
|
||||
export interface ApprovedActionDispatcherOptions {
|
||||
owner: string;
|
||||
leaseDurationMs?: number;
|
||||
retryBaseMs?: number;
|
||||
retryMaxMs?: number;
|
||||
clock?: () => number;
|
||||
createId?: () => string;
|
||||
}
|
||||
|
||||
export interface ApprovedActionDispatchBatchSummary {
|
||||
scanned: number;
|
||||
claimed: number;
|
||||
started: number;
|
||||
succeeded: number;
|
||||
failed: number;
|
||||
blocked: number;
|
||||
retrying: number;
|
||||
deferred: number;
|
||||
recoveryRequired: number;
|
||||
alreadyTerminal: number;
|
||||
unavailable: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: Readonly<ApprovedActionDispatchCursor>;
|
||||
}
|
||||
|
||||
function assertPositiveInteger(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
keys.length === canonical.length &&
|
||||
keys.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
export class ApprovedActionDispatcher {
|
||||
private readonly handlers = new Map<string, ApprovedActionHandler>();
|
||||
private readonly owner: string;
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly retryBaseMs: number;
|
||||
private readonly retryMaxMs: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: ApprovedActionDispatchRepository,
|
||||
handlers: readonly ApprovedActionHandler[],
|
||||
options: ApprovedActionDispatcherOptions,
|
||||
) {
|
||||
assertApprovedActionLeaseIdentity(options.owner);
|
||||
this.owner = options.owner;
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
||||
this.retryBaseMs = options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS;
|
||||
this.retryMaxMs = options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS;
|
||||
this.clock = options.clock ?? Date.now;
|
||||
this.createId = options.createId ?? uuidV7;
|
||||
assertApprovedActionLeaseDuration(this.leaseDurationMs);
|
||||
assertPositiveInteger('retryBaseMs', this.retryBaseMs);
|
||||
assertPositiveInteger('retryMaxMs', this.retryMaxMs);
|
||||
if (this.retryMaxMs < this.retryBaseMs) {
|
||||
throw new RangeError(
|
||||
'retryMaxMs must be greater than or equal to retryBaseMs',
|
||||
);
|
||||
}
|
||||
for (const handler of handlers) {
|
||||
if (
|
||||
!handler ||
|
||||
typeof handler !== 'object' ||
|
||||
typeof handler.actionType !== 'string' ||
|
||||
handler.actionType.length < 1 ||
|
||||
handler.actionType.length > 64 ||
|
||||
typeof handler.inspect !== 'function' ||
|
||||
typeof handler.execute !== 'function'
|
||||
) {
|
||||
throw new TypeError('Approved action handler is invalid');
|
||||
}
|
||||
if (this.handlers.has(handler.actionType)) {
|
||||
throw new TypeError(
|
||||
`Duplicate approved action handler: ${handler.actionType}`,
|
||||
);
|
||||
}
|
||||
this.handlers.set(handler.actionType, handler);
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchBatch(
|
||||
options: { cursor?: ApprovedActionDispatchCursor; limit?: number } = {},
|
||||
): Promise<Readonly<ApprovedActionDispatchBatchSummary>> {
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
||||
throw new TypeError('Approved action dispatch options must be an object');
|
||||
}
|
||||
if (
|
||||
!exactKeys(
|
||||
options,
|
||||
options.cursor === undefined && options.limit === undefined
|
||||
? []
|
||||
: [
|
||||
...(options.cursor === undefined ? [] : ['cursor']),
|
||||
...(options.limit === undefined ? [] : ['limit']),
|
||||
],
|
||||
)
|
||||
) {
|
||||
throw new TypeError('Approved action dispatch options shape is invalid');
|
||||
}
|
||||
const limit = options.limit ?? 16;
|
||||
assertApprovedActionPageSize(limit);
|
||||
const observedAtMs = this.now();
|
||||
const page = await this.repository.listDue({
|
||||
nowMs: observedAtMs,
|
||||
limit,
|
||||
...(options.cursor ? { cursor: options.cursor } : {}),
|
||||
});
|
||||
const summary: ApprovedActionDispatchBatchSummary = {
|
||||
scanned: page.dispatches.length,
|
||||
claimed: 0,
|
||||
started: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
blocked: 0,
|
||||
retrying: 0,
|
||||
deferred: 0,
|
||||
recoveryRequired: 0,
|
||||
alreadyTerminal: 0,
|
||||
unavailable: 0,
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor ? { nextCursor: page.nextCursor } : {}),
|
||||
};
|
||||
for (const candidate of page.dispatches) {
|
||||
await this.dispatchOne(candidate.dispatch.id, summary);
|
||||
}
|
||||
return Object.freeze(summary);
|
||||
}
|
||||
|
||||
private async dispatchOne(
|
||||
dispatchId: string,
|
||||
summary: ApprovedActionDispatchBatchSummary,
|
||||
): Promise<void> {
|
||||
const claimedAtMs = this.now();
|
||||
let claim;
|
||||
try {
|
||||
claim = await this.repository.claim({
|
||||
dispatchId,
|
||||
owner: this.owner,
|
||||
leaseToken: this.createId(),
|
||||
nowMs: claimedAtMs,
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
});
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
if (claim.status === 'not_found') {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
if (claim.status !== 'claimed') {
|
||||
if (claim.status === 'recovery_required') summary.recoveryRequired += 1;
|
||||
else if (
|
||||
claim.status === 'succeeded' ||
|
||||
claim.status === 'failed' ||
|
||||
claim.status === 'blocked'
|
||||
) {
|
||||
summary.alreadyTerminal += 1;
|
||||
} else summary.deferred += 1;
|
||||
return;
|
||||
}
|
||||
summary.claimed += 1;
|
||||
const handler = this.handlers.get(
|
||||
claim.snapshot.dispatch.action.actionType,
|
||||
);
|
||||
if (!handler) {
|
||||
await this.releasePreflight(
|
||||
claim.snapshot.execution,
|
||||
'handler_unavailable',
|
||||
true,
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
let inspection;
|
||||
try {
|
||||
inspection = await handler.inspect(claim.snapshot.dispatch);
|
||||
this.assertInspection(inspection);
|
||||
} catch {
|
||||
await this.releasePreflight(
|
||||
claim.snapshot.execution,
|
||||
'handler_inspection_failed',
|
||||
true,
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (inspection.status !== 'ready') {
|
||||
await this.releasePreflight(
|
||||
claim.snapshot.execution,
|
||||
inspection.resultCode,
|
||||
inspection.status === 'retry',
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
inspection.actionDigest !== claim.snapshot.dispatch.action.actionDigest
|
||||
) {
|
||||
await this.releasePreflight(
|
||||
claim.snapshot.execution,
|
||||
'action_digest_mismatch',
|
||||
false,
|
||||
summary,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let started;
|
||||
try {
|
||||
const startedAtMs = this.now();
|
||||
started = await this.repository.start({
|
||||
dispatchId,
|
||||
approvalRequestId: claim.snapshot.dispatch.approvalRequestId,
|
||||
actionDigest: inspection.actionDigest,
|
||||
owner: this.owner,
|
||||
leaseToken: claim.snapshot.execution.leaseToken!,
|
||||
expectedVersion: claim.snapshot.execution.version,
|
||||
startedAtMs,
|
||||
});
|
||||
summary.started += 1;
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
let outcome: 'succeeded' | 'failed' | 'indeterminate';
|
||||
let resultCode: string;
|
||||
try {
|
||||
const result = await handler.execute(
|
||||
Object.freeze({
|
||||
dispatch: started.dispatch,
|
||||
execution: started.execution,
|
||||
idempotencyKey: started.dispatch.id,
|
||||
fence: Object.freeze({
|
||||
owner: this.owner,
|
||||
leaseToken: started.execution.leaseToken!,
|
||||
version: started.execution.version,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
this.assertExecutionResult(result);
|
||||
outcome = result.outcome;
|
||||
resultCode = result.resultCode;
|
||||
} catch {
|
||||
outcome = 'indeterminate';
|
||||
resultCode = 'handler_failed_after_start';
|
||||
}
|
||||
try {
|
||||
const completed = await this.repository.complete({
|
||||
dispatchId,
|
||||
owner: this.owner,
|
||||
leaseToken: started.execution.leaseToken!,
|
||||
expectedVersion: started.execution.version,
|
||||
resultMutationId: this.createId(),
|
||||
outcome,
|
||||
resultCode,
|
||||
completedAtMs: this.now(),
|
||||
});
|
||||
if (completed.execution.status === 'succeeded') summary.succeeded += 1;
|
||||
else if (completed.execution.status === 'failed') summary.failed += 1;
|
||||
else summary.blocked += 1;
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
summary.recoveryRequired += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private async releasePreflight(
|
||||
execution: Readonly<ApprovedActionDispatchExecutionRecord>,
|
||||
resultCode: string,
|
||||
retry: boolean,
|
||||
summary: ApprovedActionDispatchBatchSummary,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const atMs = this.now();
|
||||
const released = await this.repository.releaseBeforeStart({
|
||||
dispatchId: execution.dispatchId,
|
||||
owner: this.owner,
|
||||
leaseToken: execution.leaseToken!,
|
||||
expectedVersion: execution.version,
|
||||
resultMutationId: this.createId(),
|
||||
resultCode,
|
||||
atMs,
|
||||
...(retry
|
||||
? { retryAtMs: this.nextRetryAt(atMs, execution.attemptCount) }
|
||||
: {}),
|
||||
});
|
||||
if (released.execution.status === 'retry_wait') summary.retrying += 1;
|
||||
else summary.blocked += 1;
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private assertInspection(
|
||||
value: unknown,
|
||||
): asserts value is Awaited<ReturnType<ApprovedActionHandler['inspect']>> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError('Approved action inspection is invalid');
|
||||
}
|
||||
if (
|
||||
'status' in value &&
|
||||
value.status === 'ready' &&
|
||||
exactKeys(value, ['status', 'actionDigest']) &&
|
||||
'actionDigest' in value &&
|
||||
typeof value.actionDigest === 'string' &&
|
||||
/^[0-9a-f]{64}$/.test(value.actionDigest)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
'status' in value &&
|
||||
(value.status === 'retry' || value.status === 'blocked') &&
|
||||
exactKeys(value, ['status', 'resultCode']) &&
|
||||
'resultCode' in value &&
|
||||
typeof value.resultCode === 'string'
|
||||
) {
|
||||
assertApprovedActionResultCode(value.resultCode);
|
||||
return;
|
||||
}
|
||||
throw new TypeError('Approved action inspection is invalid');
|
||||
}
|
||||
|
||||
private assertExecutionResult(
|
||||
value: unknown,
|
||||
): asserts value is Awaited<ReturnType<ApprovedActionHandler['execute']>> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!exactKeys(value, ['outcome', 'resultCode']) ||
|
||||
!('outcome' in value) ||
|
||||
!['succeeded', 'failed', 'indeterminate'].includes(
|
||||
value.outcome as string,
|
||||
) ||
|
||||
!('resultCode' in value) ||
|
||||
typeof value.resultCode !== 'string'
|
||||
) {
|
||||
throw new TypeError('Approved action execution result is invalid');
|
||||
}
|
||||
assertApprovedActionResultCode(value.resultCode);
|
||||
}
|
||||
|
||||
private nextRetryAt(atMs: number, attemptCount: number): number {
|
||||
const exponent = Math.max(0, Math.min(attemptCount - 1, 30));
|
||||
const delay = Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** exponent);
|
||||
return Math.min(Number.MAX_SAFE_INTEGER, atMs + delay);
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('clock must return a non-negative safe integer');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import {
|
||||
assertAuthenticatedPrincipalActive,
|
||||
normalizeAuthenticatedPrincipal,
|
||||
type AuthenticatedPrincipal,
|
||||
} from '../domain/authenticatedPrincipal';
|
||||
import {
|
||||
APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES,
|
||||
ApprovedActionRecoveryAuthorizationDeniedError,
|
||||
ApprovedActionRecoveryHumanRequiredError,
|
||||
ApprovedActionRecoveryNotFoundError,
|
||||
ApprovedActionRecoveryStrongAuthenticationRequiredError,
|
||||
MAX_APPROVED_ACTION_RECOVERY_AUTH_AGE_MS,
|
||||
createApprovedActionRecoveryAuthorizationFact,
|
||||
} from '../domain/approvedActionRecoveryAuthorization';
|
||||
import {
|
||||
assertApprovedActionEvidenceDigest,
|
||||
type ApprovedActionRecoveryDecision,
|
||||
} from '../domain/approvedActionRecovery';
|
||||
import { assertApprovedActionResultCode } from '../domain/approvedActionDispatchExecution';
|
||||
import { assertApprovalMutationId } from '../domain/approvalRequest';
|
||||
import type {
|
||||
ApprovedActionRecoveryRepository,
|
||||
ResolveApprovedActionRecoveryResult,
|
||||
} from '../ports/approvedActionRecoveryRepository';
|
||||
import type { ProjectPolicyEngine } from './projectPolicyEngine';
|
||||
|
||||
export interface ManuallyResolveApprovedActionRecoveryInput {
|
||||
dispatchId: string;
|
||||
expectedExecutionVersion: number;
|
||||
expectedRecoveryVersion: number;
|
||||
mutationId: string;
|
||||
decision: ApprovedActionRecoveryDecision;
|
||||
evidenceDigest?: string;
|
||||
reasonCode: string;
|
||||
principal: AuthenticatedPrincipal;
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): void {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
keys.length !== canonical.length ||
|
||||
keys.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
throw new TypeError('Manual recovery input shape is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function assertVersion(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0 || value > 2_147_483_647) {
|
||||
throw new RangeError(`${name} is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
export class ApprovedActionManualRecoveryService {
|
||||
constructor(
|
||||
private readonly repository: ApprovedActionRecoveryRepository,
|
||||
private readonly policy: ProjectPolicyEngine,
|
||||
private readonly clock: () => number = Date.now,
|
||||
) {}
|
||||
|
||||
async resolve(
|
||||
input: ManuallyResolveApprovedActionRecoveryInput,
|
||||
): Promise<ResolveApprovedActionRecoveryResult> {
|
||||
if (!input || typeof input !== 'object' || Array.isArray(input)) {
|
||||
throw new TypeError('Manual recovery input must be an object');
|
||||
}
|
||||
exactKeys(input, [
|
||||
'dispatchId',
|
||||
'expectedExecutionVersion',
|
||||
'expectedRecoveryVersion',
|
||||
'mutationId',
|
||||
'decision',
|
||||
...(input.evidenceDigest === undefined ? [] : ['evidenceDigest']),
|
||||
'reasonCode',
|
||||
'principal',
|
||||
]);
|
||||
assertApprovalMutationId(input.dispatchId);
|
||||
assertVersion('expectedExecutionVersion', input.expectedExecutionVersion);
|
||||
assertVersion('expectedRecoveryVersion', input.expectedRecoveryVersion);
|
||||
assertApprovalMutationId(input.mutationId);
|
||||
if (
|
||||
!['confirm_succeeded', 'confirm_failed', 'abandon_unknown'].includes(
|
||||
input.decision,
|
||||
)
|
||||
) {
|
||||
throw new TypeError('Manual recovery decision is invalid');
|
||||
}
|
||||
if (input.evidenceDigest !== undefined) {
|
||||
assertApprovedActionEvidenceDigest(input.evidenceDigest);
|
||||
}
|
||||
assertApprovedActionResultCode(input.reasonCode);
|
||||
const resolvedAtMs = this.now();
|
||||
const principal = normalizeAuthenticatedPrincipal(input.principal);
|
||||
assertAuthenticatedPrincipalActive(principal, resolvedAtMs);
|
||||
if (principal.subject.type !== 'user') {
|
||||
throw new ApprovedActionRecoveryHumanRequiredError();
|
||||
}
|
||||
if (
|
||||
!APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES.includes(
|
||||
principal.assurance as (typeof APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES)[number],
|
||||
) ||
|
||||
resolvedAtMs - principal.authenticatedAtMs >
|
||||
MAX_APPROVED_ACTION_RECOVERY_AUTH_AGE_MS
|
||||
) {
|
||||
throw new ApprovedActionRecoveryStrongAuthenticationRequiredError();
|
||||
}
|
||||
const snapshot = await this.repository.findById(input.dispatchId);
|
||||
if (!snapshot) throw new ApprovedActionRecoveryNotFoundError();
|
||||
const authorization = await this.policy.decideWithFence({
|
||||
projectId: snapshot.action.dispatch.projectId,
|
||||
subject: principal.subject,
|
||||
permission: 'approval.recover',
|
||||
});
|
||||
if (
|
||||
authorization.decision.effect !== 'allow' ||
|
||||
!authorization.fence ||
|
||||
authorization.fence.bindingVersion === null
|
||||
) {
|
||||
throw new ApprovedActionRecoveryAuthorizationDeniedError();
|
||||
}
|
||||
const authorizationFact = createApprovedActionRecoveryAuthorizationFact({
|
||||
dispatchId: input.dispatchId,
|
||||
projectId: snapshot.action.dispatch.projectId,
|
||||
mutationId: input.mutationId,
|
||||
resolvedBy: principal.subject,
|
||||
authenticationId: principal.authenticationId,
|
||||
assurance:
|
||||
principal.assurance as (typeof APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES)[number],
|
||||
authenticatedAtMs: principal.authenticatedAtMs,
|
||||
projectVersion: authorization.fence.projectVersion,
|
||||
bindingVersion: authorization.fence.bindingVersion,
|
||||
authorizedAtMs: resolvedAtMs,
|
||||
});
|
||||
return this.repository.resolve({
|
||||
dispatchId: input.dispatchId,
|
||||
expectedExecutionVersion: input.expectedExecutionVersion,
|
||||
expectedRecoveryVersion: input.expectedRecoveryVersion,
|
||||
mutationId: input.mutationId,
|
||||
source: 'human',
|
||||
decision: input.decision,
|
||||
...(input.evidenceDigest === undefined
|
||||
? {}
|
||||
: { evidenceDigest: input.evidenceDigest }),
|
||||
reasonCode: input.reasonCode,
|
||||
resolvedBy: principal.subject,
|
||||
resolvedAtMs,
|
||||
authorizationFact,
|
||||
});
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('clock must return a non-negative safe integer');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import {
|
||||
assertApprovedActionLeaseIdentity,
|
||||
assertApprovedActionResultCode,
|
||||
} from '../domain/approvedActionDispatchExecution';
|
||||
import {
|
||||
assertApprovedActionEvidenceDigest,
|
||||
assertApprovedActionRecoveryLeaseDuration,
|
||||
assertApprovedActionRecoveryPageSize,
|
||||
type ApprovedActionRecoveryCursor,
|
||||
} from '../domain/approvedActionRecovery';
|
||||
import type {
|
||||
ApprovedActionRecoveryEvidence,
|
||||
ApprovedActionRecoveryEvidenceProvider,
|
||||
} from '../ports/approvedActionRecoveryEvidenceProvider';
|
||||
import type { ApprovedActionRecoveryRepository } from '../ports/approvedActionRecoveryRepository';
|
||||
|
||||
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
||||
const DEFAULT_RETRY_BASE_MS = 5_000;
|
||||
const DEFAULT_RETRY_MAX_MS = 5 * 60_000;
|
||||
|
||||
export interface ApprovedActionRecoveryReconcilerOptions {
|
||||
owner: string;
|
||||
leaseDurationMs?: number;
|
||||
retryBaseMs?: number;
|
||||
retryMaxMs?: number;
|
||||
clock?: () => number;
|
||||
createId?: () => string;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRecoveryBatchSummary {
|
||||
scanned: number;
|
||||
claimed: number;
|
||||
verifiedSucceeded: number;
|
||||
verifiedFailed: number;
|
||||
deferred: number;
|
||||
manualRequired: number;
|
||||
executionActive: number;
|
||||
alreadyResolved: number;
|
||||
unavailable: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: Readonly<ApprovedActionRecoveryCursor>;
|
||||
}
|
||||
|
||||
function assertPositiveInteger(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
keys.length === canonical.length &&
|
||||
keys.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
export class ApprovedActionRecoveryReconciler {
|
||||
private readonly providers = new Map<
|
||||
string,
|
||||
ApprovedActionRecoveryEvidenceProvider
|
||||
>();
|
||||
private readonly owner: string;
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly retryBaseMs: number;
|
||||
private readonly retryMaxMs: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: ApprovedActionRecoveryRepository,
|
||||
providers: readonly ApprovedActionRecoveryEvidenceProvider[],
|
||||
options: ApprovedActionRecoveryReconcilerOptions,
|
||||
) {
|
||||
assertApprovedActionLeaseIdentity(options.owner);
|
||||
this.owner = options.owner;
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
||||
this.retryBaseMs = options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS;
|
||||
this.retryMaxMs = options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS;
|
||||
this.clock = options.clock ?? Date.now;
|
||||
this.createId = options.createId ?? uuidV7;
|
||||
assertApprovedActionRecoveryLeaseDuration(this.leaseDurationMs);
|
||||
assertPositiveInteger('retryBaseMs', this.retryBaseMs);
|
||||
assertPositiveInteger('retryMaxMs', this.retryMaxMs);
|
||||
if (this.retryMaxMs < this.retryBaseMs) {
|
||||
throw new RangeError(
|
||||
'retryMaxMs must be greater than or equal to retryBaseMs',
|
||||
);
|
||||
}
|
||||
for (const provider of providers) {
|
||||
if (
|
||||
!provider ||
|
||||
typeof provider !== 'object' ||
|
||||
typeof provider.actionType !== 'string' ||
|
||||
provider.actionType.length < 1 ||
|
||||
provider.actionType.length > 64 ||
|
||||
!['automatic', 'manual_only'].includes(provider.capability) ||
|
||||
typeof provider.inspect !== 'function'
|
||||
) {
|
||||
throw new TypeError('Approved action recovery provider is invalid');
|
||||
}
|
||||
if (this.providers.has(provider.actionType)) {
|
||||
throw new TypeError(
|
||||
`Duplicate approved action recovery provider: ${provider.actionType}`,
|
||||
);
|
||||
}
|
||||
this.providers.set(provider.actionType, provider);
|
||||
}
|
||||
}
|
||||
|
||||
async reconcileBatch(
|
||||
options: { cursor?: ApprovedActionRecoveryCursor; limit?: number } = {},
|
||||
): Promise<Readonly<ApprovedActionRecoveryBatchSummary>> {
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
||||
throw new TypeError('Approved action recovery options must be an object');
|
||||
}
|
||||
const expectedKeys = [
|
||||
...(options.cursor === undefined ? [] : ['cursor']),
|
||||
...(options.limit === undefined ? [] : ['limit']),
|
||||
];
|
||||
if (!exactKeys(options, expectedKeys)) {
|
||||
throw new TypeError('Approved action recovery options shape is invalid');
|
||||
}
|
||||
const limit = options.limit ?? 16;
|
||||
assertApprovedActionRecoveryPageSize(limit);
|
||||
const observedAtMs = this.now();
|
||||
const page = await this.repository.listDue({
|
||||
nowMs: observedAtMs,
|
||||
limit,
|
||||
...(options.cursor ? { cursor: options.cursor } : {}),
|
||||
});
|
||||
const summary: ApprovedActionRecoveryBatchSummary = {
|
||||
scanned: page.recoveries.length,
|
||||
claimed: 0,
|
||||
verifiedSucceeded: 0,
|
||||
verifiedFailed: 0,
|
||||
deferred: 0,
|
||||
manualRequired: 0,
|
||||
executionActive: 0,
|
||||
alreadyResolved: 0,
|
||||
unavailable: 0,
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor ? { nextCursor: page.nextCursor } : {}),
|
||||
};
|
||||
for (const candidate of page.recoveries) {
|
||||
await this.reconcileOne(candidate.action.dispatch.id, summary);
|
||||
}
|
||||
return Object.freeze(summary);
|
||||
}
|
||||
|
||||
private async reconcileOne(
|
||||
dispatchId: string,
|
||||
summary: ApprovedActionRecoveryBatchSummary,
|
||||
): Promise<void> {
|
||||
let claim;
|
||||
try {
|
||||
claim = await this.repository.claim({
|
||||
dispatchId,
|
||||
owner: this.owner,
|
||||
leaseToken: this.createId(),
|
||||
nowMs: this.now(),
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
});
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
if (claim.status === 'not_found') {
|
||||
summary.unavailable += 1;
|
||||
return;
|
||||
}
|
||||
if (claim.status !== 'claimed') {
|
||||
if (claim.status === 'manual_required') summary.manualRequired += 1;
|
||||
else if (claim.status === 'resolved') summary.alreadyResolved += 1;
|
||||
else if (claim.status === 'execution_active')
|
||||
summary.executionActive += 1;
|
||||
else summary.deferred += 1;
|
||||
return;
|
||||
}
|
||||
summary.claimed += 1;
|
||||
const snapshot = claim.snapshot;
|
||||
const provider = this.providers.get(
|
||||
snapshot.action.dispatch.action.actionType,
|
||||
);
|
||||
let evidence: ApprovedActionRecoveryEvidence;
|
||||
if (!provider || provider.capability === 'manual_only') {
|
||||
evidence = {
|
||||
finding: 'unsupported',
|
||||
resultCode: 'automatic_recovery_unsupported',
|
||||
};
|
||||
} else {
|
||||
try {
|
||||
evidence = await provider.inspect(
|
||||
Object.freeze({
|
||||
snapshot,
|
||||
idempotencyKey: snapshot.action.dispatch.id,
|
||||
observedAtMs: this.now(),
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
evidence = {
|
||||
finding: 'unavailable',
|
||||
resultCode: 'recovery_evidence_unavailable',
|
||||
};
|
||||
}
|
||||
try {
|
||||
this.assertEvidence(evidence);
|
||||
} catch {
|
||||
evidence = {
|
||||
finding: 'conflict',
|
||||
resultCode: 'recovery_evidence_invalid',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
evidence.finding === 'verified_succeeded' ||
|
||||
evidence.finding === 'verified_failed'
|
||||
) {
|
||||
try {
|
||||
const resolved = await this.repository.resolve({
|
||||
dispatchId,
|
||||
expectedExecutionVersion: snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: snapshot.recovery.version,
|
||||
owner: this.owner,
|
||||
leaseToken: snapshot.recovery.leaseToken!,
|
||||
mutationId: this.createId(),
|
||||
source: 'automatic_evidence',
|
||||
decision:
|
||||
evidence.finding === 'verified_succeeded'
|
||||
? 'confirm_succeeded'
|
||||
: 'confirm_failed',
|
||||
evidenceDigest: evidence.evidenceDigest,
|
||||
reasonCode: evidence.resultCode,
|
||||
resolvedAtMs: this.now(),
|
||||
});
|
||||
if (resolved.status === 'not_found') {
|
||||
summary.unavailable += 1;
|
||||
} else if (resolved.status === 'already_terminal') {
|
||||
summary.alreadyResolved += 1;
|
||||
} else if (resolved.snapshot.action.execution.status === 'succeeded') {
|
||||
summary.verifiedSucceeded += 1;
|
||||
} else {
|
||||
summary.verifiedFailed += 1;
|
||||
}
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const retry = ['still_running', 'missing', 'unavailable'].includes(
|
||||
evidence.finding,
|
||||
);
|
||||
try {
|
||||
const observedAtMs = this.now();
|
||||
const recorded = await this.repository.recordFinding({
|
||||
dispatchId,
|
||||
expectedExecutionVersion: snapshot.action.execution.version,
|
||||
expectedRecoveryVersion: snapshot.recovery.version,
|
||||
owner: this.owner,
|
||||
leaseToken: snapshot.recovery.leaseToken!,
|
||||
findingMutationId: this.createId(),
|
||||
finding: evidence.finding,
|
||||
resultCode: evidence.resultCode,
|
||||
...(evidence.evidenceDigest
|
||||
? { evidenceDigest: evidence.evidenceDigest }
|
||||
: {}),
|
||||
observedAtMs,
|
||||
...(retry
|
||||
? {
|
||||
retryAtMs: this.nextRetryAt(
|
||||
observedAtMs,
|
||||
snapshot.recovery.findingCount + 1,
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
if (recorded.recovery.status === 'manual_required') {
|
||||
summary.manualRequired += 1;
|
||||
} else {
|
||||
summary.deferred += 1;
|
||||
}
|
||||
} catch {
|
||||
summary.unavailable += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private assertEvidence(
|
||||
value: unknown,
|
||||
): asserts value is ApprovedActionRecoveryEvidence {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError('Approved action recovery evidence is invalid');
|
||||
}
|
||||
if (
|
||||
!('finding' in value) ||
|
||||
![
|
||||
'verified_succeeded',
|
||||
'verified_failed',
|
||||
'still_running',
|
||||
'missing',
|
||||
'conflict',
|
||||
'unsupported',
|
||||
'unavailable',
|
||||
].includes(value.finding as string) ||
|
||||
!('resultCode' in value) ||
|
||||
typeof value.resultCode !== 'string'
|
||||
) {
|
||||
throw new TypeError('Approved action recovery evidence is invalid');
|
||||
}
|
||||
assertApprovedActionResultCode(value.resultCode);
|
||||
const verified =
|
||||
value.finding === 'verified_succeeded' ||
|
||||
value.finding === 'verified_failed';
|
||||
if (
|
||||
!exactKeys(
|
||||
value,
|
||||
verified || 'evidenceDigest' in value
|
||||
? ['finding', 'resultCode', 'evidenceDigest']
|
||||
: ['finding', 'resultCode'],
|
||||
)
|
||||
) {
|
||||
throw new TypeError('Approved action recovery evidence is invalid');
|
||||
}
|
||||
if (verified && !('evidenceDigest' in value)) {
|
||||
throw new TypeError('Verified recovery evidence has no digest');
|
||||
}
|
||||
if ('evidenceDigest' in value) {
|
||||
if (typeof value.evidenceDigest !== 'string') {
|
||||
throw new TypeError('Approved action recovery evidence is invalid');
|
||||
}
|
||||
assertApprovedActionEvidenceDigest(value.evidenceDigest);
|
||||
}
|
||||
}
|
||||
|
||||
private nextRetryAt(atMs: number, findingCount: number): number {
|
||||
const exponent = Math.max(0, Math.min(findingCount - 1, 30));
|
||||
const delay = Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** exponent);
|
||||
return Math.min(Number.MAX_SAFE_INTEGER, atMs + delay);
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('clock must return a non-negative safe integer');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
import type { ApprovedActionDispatchCursor } from '../domain/approvedActionDispatchExecution';
|
||||
import type { ApprovedActionRecoveryCursor } from '../domain/approvedActionRecovery';
|
||||
import type {
|
||||
ApprovedActionDispatchCycleOptions,
|
||||
ApprovedActionRecoveryCycleOptions,
|
||||
ApprovedActionRuntimeCycleSummary,
|
||||
ApprovedActionRuntimeSupervisor,
|
||||
} from './approvedActionRuntimeSupervisor';
|
||||
|
||||
export const MIN_APPROVED_ACTION_RUNTIME_INTERVAL_MS = 250;
|
||||
export const MAX_APPROVED_ACTION_RUNTIME_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_APPROVED_ACTION_RUNTIME_INITIAL_DELAY_MS =
|
||||
24 * 60 * 60 * 1_000;
|
||||
export const MAX_APPROVED_ACTION_RUNTIME_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRuntimeLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRuntimeLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
cycle?: {
|
||||
dispatch?: ApprovedActionDispatchCycleOptions;
|
||||
recovery?: ApprovedActionRecoveryCycleOptions;
|
||||
};
|
||||
scheduler?: ApprovedActionRuntimeLifecycleScheduler;
|
||||
onCycle?: (summary: Readonly<ApprovedActionRuntimeCycleSummary>) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type ApprovedActionRuntimeStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: ApprovedActionRuntimeLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One timer serializes recovery and new dispatch work. It remains inert until
|
||||
* start(), never overlaps a slow cycle, resumes bounded keyset cursors, and
|
||||
* waits only a bounded time during shutdown.
|
||||
*/
|
||||
export class ApprovedActionRuntimeLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly dispatchOptions: Omit<
|
||||
ApprovedActionDispatchCycleOptions,
|
||||
'cursor'
|
||||
>;
|
||||
private readonly recoveryOptions: Omit<
|
||||
ApprovedActionRecoveryCycleOptions,
|
||||
'cursor'
|
||||
>;
|
||||
private readonly scheduler: ApprovedActionRuntimeLifecycleScheduler;
|
||||
private readonly onCycle?: (
|
||||
summary: Readonly<ApprovedActionRuntimeCycleSummary>,
|
||||
) => void;
|
||||
private readonly onError?: (error: unknown) => void;
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
private dispatchCursor?: ApprovedActionDispatchCursor;
|
||||
private recoveryCursor?: ApprovedActionRecoveryCursor;
|
||||
|
||||
constructor(
|
||||
private readonly supervisor: Pick<
|
||||
ApprovedActionRuntimeSupervisor,
|
||||
'runCycle'
|
||||
>,
|
||||
options: ApprovedActionRuntimeLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.dispatchOptions = {
|
||||
...(options.cycle?.dispatch?.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.cycle.dispatch.pageSize }),
|
||||
...(options.cycle?.dispatch?.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.cycle.dispatch.maxPages }),
|
||||
};
|
||||
this.recoveryOptions = {
|
||||
...(options.cycle?.recovery?.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.cycle.recovery.pageSize }),
|
||||
...(options.cycle?.recovery?.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.cycle.recovery.maxPages }),
|
||||
};
|
||||
this.dispatchCursor =
|
||||
options.cycle?.dispatch?.cursor === undefined
|
||||
? undefined
|
||||
: { ...options.cycle.dispatch.cursor };
|
||||
this.recoveryCursor =
|
||||
options.cycle?.recovery?.cursor === undefined
|
||||
? undefined
|
||||
: { ...options.cycle.recovery.cursor };
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_APPROVED_ACTION_RUNTIME_INTERVAL_MS,
|
||||
MAX_APPROVED_ACTION_RUNTIME_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_APPROVED_ACTION_RUNTIME_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_APPROVED_ACTION_RUNTIME_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<ApprovedActionRuntimeStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<ApprovedActionRuntimeStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.supervisor
|
||||
.runCycle({
|
||||
recovery: {
|
||||
...this.recoveryOptions,
|
||||
...(this.recoveryCursor === undefined
|
||||
? {}
|
||||
: { cursor: { ...this.recoveryCursor } }),
|
||||
},
|
||||
dispatch: {
|
||||
...this.dispatchOptions,
|
||||
...(this.dispatchCursor === undefined
|
||||
? {}
|
||||
: { cursor: { ...this.dispatchCursor } }),
|
||||
},
|
||||
})
|
||||
.then((summary) => {
|
||||
this.recoveryCursor =
|
||||
summary.recovery.remaining && summary.recovery.nextCursor
|
||||
? { ...summary.recovery.nextCursor }
|
||||
: undefined;
|
||||
this.dispatchCursor =
|
||||
summary.dispatch.remaining && summary.dispatch.nextCursor
|
||||
? { ...summary.dispatch.nextCursor }
|
||||
: undefined;
|
||||
this.notifyCycle(summary);
|
||||
})
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private notifyCycle(
|
||||
summary: Readonly<ApprovedActionRuntimeCycleSummary>,
|
||||
): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must not create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
import {
|
||||
assertApprovedActionPageSize,
|
||||
type ApprovedActionDispatchCursor,
|
||||
} from '../domain/approvedActionDispatchExecution';
|
||||
import {
|
||||
assertApprovedActionRecoveryPageSize,
|
||||
type ApprovedActionRecoveryCursor,
|
||||
} from '../domain/approvedActionRecovery';
|
||||
import type {
|
||||
ApprovedActionDispatchBatchSummary,
|
||||
ApprovedActionDispatcher,
|
||||
} from './approvedActionDispatcher';
|
||||
import type {
|
||||
ApprovedActionRecoveryBatchSummary,
|
||||
ApprovedActionRecoveryReconciler,
|
||||
} from './approvedActionRecoveryReconciler';
|
||||
|
||||
export const MAX_APPROVED_ACTION_RUNTIME_PAGES_PER_PHASE = 64;
|
||||
|
||||
export type ApprovedActionRuntimePhaseStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface ApprovedActionDispatchCycleOptions {
|
||||
cursor?: ApprovedActionDispatchCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRecoveryCycleOptions {
|
||||
cursor?: ApprovedActionRecoveryCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRuntimeCycleOptions {
|
||||
dispatch?: ApprovedActionDispatchCycleOptions;
|
||||
recovery?: ApprovedActionRecoveryCycleOptions;
|
||||
}
|
||||
|
||||
export interface ApprovedActionDispatchCycleSummary
|
||||
extends Omit<ApprovedActionDispatchBatchSummary, 'truncated' | 'nextCursor'> {
|
||||
pages: number;
|
||||
stopReason: ApprovedActionRuntimePhaseStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: Readonly<ApprovedActionDispatchCursor>;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRecoveryCycleSummary
|
||||
extends Omit<ApprovedActionRecoveryBatchSummary, 'truncated' | 'nextCursor'> {
|
||||
pages: number;
|
||||
stopReason: ApprovedActionRuntimePhaseStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: Readonly<ApprovedActionRecoveryCursor>;
|
||||
}
|
||||
|
||||
export interface ApprovedActionRuntimeCycleSummary {
|
||||
recovery: Readonly<ApprovedActionRecoveryCycleSummary>;
|
||||
dispatch: Readonly<ApprovedActionDispatchCycleSummary>;
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
keys.length === canonical.length &&
|
||||
keys.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
function assertOptionsObject(
|
||||
name: string,
|
||||
value: unknown,
|
||||
): asserts value is object {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError(`${name} must be an object`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertMaxPages(value: number): void {
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < 1 ||
|
||||
value > MAX_APPROVED_ACTION_RUNTIME_PAGES_PER_PHASE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_APPROVED_ACTION_RUNTIME_PAGES_PER_PHASE',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function sameDispatchCursor(
|
||||
left: ApprovedActionDispatchCursor | undefined,
|
||||
right: Readonly<ApprovedActionDispatchCursor>,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.eligibleAtMs === right.eligibleAtMs &&
|
||||
left.dispatchId === right.dispatchId
|
||||
);
|
||||
}
|
||||
|
||||
function sameRecoveryCursor(
|
||||
left: ApprovedActionRecoveryCursor | undefined,
|
||||
right: Readonly<ApprovedActionRecoveryCursor>,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.nextScanAtMs === right.nextScanAtMs &&
|
||||
left.dispatchId === right.dispatchId
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeDispatchOptions(
|
||||
value: ApprovedActionDispatchCycleOptions = {},
|
||||
): Required<Pick<ApprovedActionDispatchCycleOptions, 'pageSize' | 'maxPages'>> &
|
||||
Pick<ApprovedActionDispatchCycleOptions, 'cursor'> {
|
||||
assertOptionsObject('dispatch cycle options', value);
|
||||
const expectedKeys = [
|
||||
...(value.cursor === undefined ? [] : ['cursor']),
|
||||
...(value.pageSize === undefined ? [] : ['pageSize']),
|
||||
...(value.maxPages === undefined ? [] : ['maxPages']),
|
||||
];
|
||||
if (!exactKeys(value, expectedKeys)) {
|
||||
throw new TypeError('dispatch cycle options shape is invalid');
|
||||
}
|
||||
const pageSize = value.pageSize ?? 16;
|
||||
const maxPages = value.maxPages ?? 4;
|
||||
assertApprovedActionPageSize(pageSize);
|
||||
assertMaxPages(maxPages);
|
||||
return {
|
||||
pageSize,
|
||||
maxPages,
|
||||
...(value.cursor === undefined ? {} : { cursor: { ...value.cursor } }),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeRecoveryOptions(
|
||||
value: ApprovedActionRecoveryCycleOptions = {},
|
||||
): Required<Pick<ApprovedActionRecoveryCycleOptions, 'pageSize' | 'maxPages'>> &
|
||||
Pick<ApprovedActionRecoveryCycleOptions, 'cursor'> {
|
||||
assertOptionsObject('recovery cycle options', value);
|
||||
const expectedKeys = [
|
||||
...(value.cursor === undefined ? [] : ['cursor']),
|
||||
...(value.pageSize === undefined ? [] : ['pageSize']),
|
||||
...(value.maxPages === undefined ? [] : ['maxPages']),
|
||||
];
|
||||
if (!exactKeys(value, expectedKeys)) {
|
||||
throw new TypeError('recovery cycle options shape is invalid');
|
||||
}
|
||||
const pageSize = value.pageSize ?? 16;
|
||||
const maxPages = value.maxPages ?? 4;
|
||||
assertApprovedActionRecoveryPageSize(pageSize);
|
||||
assertMaxPages(maxPages);
|
||||
return {
|
||||
pageSize,
|
||||
maxPages,
|
||||
...(value.cursor === undefined ? {} : { cursor: { ...value.cursor } }),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one bounded SQLite control-plane cycle. Recovery is deliberately first:
|
||||
* if the recovery index cannot be read, the cycle does not create more action
|
||||
* side effects. The class owns no timer and is safe to embed in other profiles.
|
||||
*/
|
||||
export class ApprovedActionRuntimeSupervisor {
|
||||
constructor(
|
||||
private readonly dispatcher: Pick<
|
||||
ApprovedActionDispatcher,
|
||||
'dispatchBatch'
|
||||
>,
|
||||
private readonly reconciler: Pick<
|
||||
ApprovedActionRecoveryReconciler,
|
||||
'reconcileBatch'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async runCycle(
|
||||
options: ApprovedActionRuntimeCycleOptions = {},
|
||||
): Promise<Readonly<ApprovedActionRuntimeCycleSummary>> {
|
||||
assertOptionsObject('approved action runtime options', options);
|
||||
const expectedKeys = [
|
||||
...(options.dispatch === undefined ? [] : ['dispatch']),
|
||||
...(options.recovery === undefined ? [] : ['recovery']),
|
||||
];
|
||||
if (!exactKeys(options, expectedKeys)) {
|
||||
throw new TypeError('approved action runtime options shape is invalid');
|
||||
}
|
||||
const recoveryOptions = normalizeRecoveryOptions(options.recovery);
|
||||
const dispatchOptions = normalizeDispatchOptions(options.dispatch);
|
||||
const recovery = await this.runRecovery(recoveryOptions);
|
||||
const dispatch = await this.runDispatch(dispatchOptions);
|
||||
return Object.freeze({ recovery, dispatch });
|
||||
}
|
||||
|
||||
private async runDispatch(
|
||||
options: ReturnType<typeof normalizeDispatchOptions>,
|
||||
): Promise<Readonly<ApprovedActionDispatchCycleSummary>> {
|
||||
const total: ApprovedActionDispatchCycleSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
claimed: 0,
|
||||
started: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
blocked: 0,
|
||||
retrying: 0,
|
||||
deferred: 0,
|
||||
recoveryRequired: 0,
|
||||
alreadyTerminal: 0,
|
||||
unavailable: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
for (let pageNumber = 0; pageNumber < options.maxPages; pageNumber += 1) {
|
||||
const page = await this.dispatcher.dispatchBatch({
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: options.pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.claimed += page.claimed;
|
||||
total.started += page.started;
|
||||
total.succeeded += page.succeeded;
|
||||
total.failed += page.failed;
|
||||
total.blocked += page.blocked;
|
||||
total.retrying += page.retrying;
|
||||
total.deferred += page.deferred;
|
||||
total.recoveryRequired += page.recoveryRequired;
|
||||
total.alreadyTerminal += page.alreadyTerminal;
|
||||
total.unavailable += page.unavailable;
|
||||
if (!page.truncated) return Object.freeze(total);
|
||||
if (!page.nextCursor || sameDispatchCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
if (page.nextCursor) total.nextCursor = { ...page.nextCursor };
|
||||
return Object.freeze(total);
|
||||
}
|
||||
cursor = { ...page.nextCursor };
|
||||
if (pageNumber === options.maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return Object.freeze(total);
|
||||
}
|
||||
}
|
||||
return Object.freeze(total);
|
||||
}
|
||||
|
||||
private async runRecovery(
|
||||
options: ReturnType<typeof normalizeRecoveryOptions>,
|
||||
): Promise<Readonly<ApprovedActionRecoveryCycleSummary>> {
|
||||
const total: ApprovedActionRecoveryCycleSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
claimed: 0,
|
||||
verifiedSucceeded: 0,
|
||||
verifiedFailed: 0,
|
||||
deferred: 0,
|
||||
manualRequired: 0,
|
||||
executionActive: 0,
|
||||
alreadyResolved: 0,
|
||||
unavailable: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
for (let pageNumber = 0; pageNumber < options.maxPages; pageNumber += 1) {
|
||||
const page = await this.reconciler.reconcileBatch({
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: options.pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.claimed += page.claimed;
|
||||
total.verifiedSucceeded += page.verifiedSucceeded;
|
||||
total.verifiedFailed += page.verifiedFailed;
|
||||
total.deferred += page.deferred;
|
||||
total.manualRequired += page.manualRequired;
|
||||
total.executionActive += page.executionActive;
|
||||
total.alreadyResolved += page.alreadyResolved;
|
||||
total.unavailable += page.unavailable;
|
||||
if (!page.truncated) return Object.freeze(total);
|
||||
if (!page.nextCursor || sameRecoveryCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
if (page.nextCursor) total.nextCursor = { ...page.nextCursor };
|
||||
return Object.freeze(total);
|
||||
}
|
||||
cursor = { ...page.nextCursor };
|
||||
if (pageNumber === options.maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return Object.freeze(total);
|
||||
}
|
||||
}
|
||||
return Object.freeze(total);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import {
|
||||
APPROVED_RUN_ACTION_TYPE,
|
||||
APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
ApprovedRunActionBindingConflictError,
|
||||
InvalidApprovedRunActionError,
|
||||
digestApprovedRunCreationPlan,
|
||||
normalizeApprovedRunCreationPlan,
|
||||
type ApprovedRunCreationPlan,
|
||||
} from '../domain/approvedRunAction';
|
||||
import type { ApprovedActionDispatchRecord } from '../domain/approvalRequest';
|
||||
import type {
|
||||
ApprovedActionExecutionContext,
|
||||
ApprovedActionExecutionResult,
|
||||
ApprovedActionHandler,
|
||||
ApprovedActionInspectionResult,
|
||||
} from '../ports/approvedActionHandler';
|
||||
import type { ApprovedRunActionPlanResolver } from '../ports/approvedRunActionPlanResolver';
|
||||
import type { ApprovedRunActionRepository } from '../ports/approvedRunActionRepository';
|
||||
|
||||
export class ApprovedRunActionHandler implements ApprovedActionHandler {
|
||||
readonly actionType = APPROVED_RUN_ACTION_TYPE;
|
||||
|
||||
constructor(
|
||||
private readonly plans: ApprovedRunActionPlanResolver,
|
||||
private readonly repository: ApprovedRunActionRepository,
|
||||
) {}
|
||||
|
||||
async inspect(
|
||||
dispatch: Readonly<ApprovedActionDispatchRecord>,
|
||||
): Promise<ApprovedActionInspectionResult> {
|
||||
if (dispatch.action.actionType !== this.actionType) {
|
||||
return { status: 'blocked', resultCode: 'approved_run_type_mismatch' };
|
||||
}
|
||||
const plan = await this.plans.resolve(dispatch.action.actionRef);
|
||||
if (!plan) {
|
||||
return { status: 'retry', resultCode: 'approved_run_plan_missing' };
|
||||
}
|
||||
try {
|
||||
const normalized = normalizeApprovedRunCreationPlan(plan);
|
||||
if (
|
||||
normalized.actionRef !== dispatch.action.actionRef ||
|
||||
normalized.projectId !== dispatch.projectId
|
||||
) {
|
||||
return {
|
||||
status: 'blocked',
|
||||
resultCode: 'approved_run_plan_binding_invalid',
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: 'ready',
|
||||
actionDigest: digestApprovedRunCreationPlan(normalized),
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidApprovedRunActionError) {
|
||||
return { status: 'blocked', resultCode: 'approved_run_plan_invalid' };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async execute(
|
||||
context: Readonly<ApprovedActionExecutionContext>,
|
||||
): Promise<ApprovedActionExecutionResult> {
|
||||
if (!this.contextIsBound(context)) {
|
||||
return { outcome: 'failed', resultCode: 'approved_run_fence_invalid' };
|
||||
}
|
||||
const plan = await this.plans.resolve(context.dispatch.action.actionRef);
|
||||
if (!plan) {
|
||||
return {
|
||||
outcome: 'failed',
|
||||
resultCode: 'approved_run_plan_disappeared',
|
||||
};
|
||||
}
|
||||
let normalized: Readonly<ApprovedRunCreationPlan>;
|
||||
try {
|
||||
normalized = normalizeApprovedRunCreationPlan(plan);
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidApprovedRunActionError) {
|
||||
return { outcome: 'failed', resultCode: 'approved_run_plan_changed' };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
normalized.actionRef !== context.dispatch.action.actionRef ||
|
||||
normalized.projectId !== context.dispatch.projectId ||
|
||||
digestApprovedRunCreationPlan(normalized) !==
|
||||
context.dispatch.action.actionDigest
|
||||
) {
|
||||
return { outcome: 'failed', resultCode: 'approved_run_plan_changed' };
|
||||
}
|
||||
try {
|
||||
await this.repository.create({
|
||||
snapshot: Object.freeze({
|
||||
dispatch: context.dispatch,
|
||||
execution: context.execution,
|
||||
}),
|
||||
plan: normalized,
|
||||
});
|
||||
return {
|
||||
outcome: 'succeeded',
|
||||
resultCode: APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof ApprovedRunActionBindingConflictError) {
|
||||
return {
|
||||
outcome: 'failed',
|
||||
resultCode: 'approved_run_receipt_conflict',
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private contextIsBound(
|
||||
context: Readonly<ApprovedActionExecutionContext>,
|
||||
): boolean {
|
||||
return (
|
||||
context.dispatch.action.actionType === this.actionType &&
|
||||
context.execution.status === 'executing' &&
|
||||
context.execution.dispatchId === context.dispatch.id &&
|
||||
context.execution.projectId === context.dispatch.projectId &&
|
||||
context.execution.startedAtMs !== null &&
|
||||
context.execution.leaseOwner === context.fence.owner &&
|
||||
context.execution.leaseToken === context.fence.leaseToken &&
|
||||
context.execution.version === context.fence.version &&
|
||||
context.idempotencyKey === context.dispatch.id
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import type {
|
||||
AcknowledgeRemoteRunRunningCommand,
|
||||
AcknowledgeRemoteRunStartingCommand,
|
||||
FailRemoteRunStartCommand,
|
||||
RemoteRunActivationResult,
|
||||
} from './remoteRunActivationService';
|
||||
import { RemoteRunActivationService } from './remoteRunActivationService';
|
||||
import type { AuthenticatedWorkerPrincipal } from './workerControlService';
|
||||
import type { WorkerRemoteRunActivationClient } from '../ports/workerRemoteRunActivationClient';
|
||||
|
||||
/** The transport authenticates once; Worker request bodies cannot select a principal. */
|
||||
export class BoundWorkerRemoteRunActivationClient
|
||||
implements WorkerRemoteRunActivationClient
|
||||
{
|
||||
constructor(
|
||||
private readonly service: RemoteRunActivationService,
|
||||
private readonly principal: AuthenticatedWorkerPrincipal,
|
||||
) {}
|
||||
|
||||
acknowledgeStarting(
|
||||
command: AcknowledgeRemoteRunStartingCommand,
|
||||
): Promise<RemoteRunActivationResult> {
|
||||
return this.service.acknowledgeStarting(this.principal, command);
|
||||
}
|
||||
|
||||
acknowledgeRunning(
|
||||
command: AcknowledgeRemoteRunRunningCommand,
|
||||
): Promise<RemoteRunActivationResult> {
|
||||
return this.service.acknowledgeRunning(this.principal, command);
|
||||
}
|
||||
|
||||
failStart(
|
||||
command: FailRemoteRunStartCommand,
|
||||
): Promise<RemoteRunActivationResult> {
|
||||
return this.service.failStart(this.principal, command);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { WorkerRemoteRunCompletionClient } from '../ports/workerRemoteRunCompletionClient';
|
||||
import type { PrimaryRunCompletionResult } from './primaryRunCompletionService';
|
||||
import {
|
||||
RemoteRunCompletionService,
|
||||
type RemoteRunCompletionCommand,
|
||||
} from './remoteRunCompletionService';
|
||||
import type { AuthenticatedWorkerPrincipal } from './workerControlService';
|
||||
|
||||
/** The transport authenticates once; Worker request bodies cannot select a principal. */
|
||||
export class BoundWorkerRemoteRunCompletionClient
|
||||
implements WorkerRemoteRunCompletionClient
|
||||
{
|
||||
constructor(
|
||||
private readonly service: RemoteRunCompletionService,
|
||||
private readonly principal: AuthenticatedWorkerPrincipal,
|
||||
) {}
|
||||
|
||||
complete(
|
||||
command: RemoteRunCompletionCommand,
|
||||
): Promise<PrimaryRunCompletionResult> {
|
||||
return this.service.complete(this.principal, command);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { RunDispatchLeaseRecord } from '../domain/runDispatchLease';
|
||||
import type { ReleaseRunDispatchLeaseResult } from '../ports/runDispatchLeaseRepository';
|
||||
import type { WorkerRunLeaseClient } from '../ports/workerRunLeaseClient';
|
||||
import type {
|
||||
FencedRunDispatchLeaseRequest,
|
||||
ReleaseRunDispatchLeaseRequest,
|
||||
} from './runDispatchLeaseService';
|
||||
import { RunDispatchLeaseService } from './runDispatchLeaseService';
|
||||
import type { AuthenticatedWorkerPrincipal } from './workerControlService';
|
||||
|
||||
/**
|
||||
* Transport seam: the authenticated principal is fixed when the client is
|
||||
* constructed and can never be supplied by a Worker request body.
|
||||
*/
|
||||
export class BoundWorkerRunLeaseClient implements WorkerRunLeaseClient {
|
||||
constructor(
|
||||
private readonly service: RunDispatchLeaseService,
|
||||
private readonly principal: AuthenticatedWorkerPrincipal,
|
||||
) {}
|
||||
|
||||
renew(
|
||||
request: FencedRunDispatchLeaseRequest,
|
||||
): Promise<RunDispatchLeaseRecord> {
|
||||
return this.service.renew(this.principal, request);
|
||||
}
|
||||
|
||||
release(
|
||||
request: ReleaseRunDispatchLeaseRequest,
|
||||
): Promise<ReleaseRunDispatchLeaseResult> {
|
||||
return this.service.release(this.principal, request);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
import type { DeploymentProfile } from '../domain/deploymentProfile';
|
||||
|
||||
export type ClusterControlActivationState =
|
||||
| 'disabled'
|
||||
| 'schema_ready'
|
||||
| 'reconciled'
|
||||
| 'active'
|
||||
| 'failed'
|
||||
| 'stopped';
|
||||
|
||||
export interface ClusterControlReadinessEvidence {
|
||||
readonly contractName: string;
|
||||
readonly contractVersion: number;
|
||||
readonly serverMajor: number;
|
||||
readonly migrationIds: readonly string[];
|
||||
}
|
||||
|
||||
export interface ClusterControlReadinessProbe {
|
||||
assertReady(): Promise<ClusterControlReadinessEvidence>;
|
||||
}
|
||||
|
||||
export interface ClusterControlStartupRecoverySummary {
|
||||
readonly safe: boolean;
|
||||
readonly remaining: number;
|
||||
readonly failed: number;
|
||||
}
|
||||
|
||||
export type ClusterControlStopResult = 'stopped' | 'timed_out';
|
||||
|
||||
export interface ClusterControlActivationStack {
|
||||
reconcile(): Promise<ClusterControlStartupRecoverySummary>;
|
||||
startLifecycles(): Promise<boolean>;
|
||||
installAdmission(): () => void;
|
||||
stop(): Promise<ClusterControlStopResult>;
|
||||
}
|
||||
|
||||
export interface ClusterControlActivationAudit {
|
||||
readonly state: ClusterControlActivationState;
|
||||
readonly contractName?: string;
|
||||
readonly contractVersion?: number;
|
||||
readonly serverMajor?: number;
|
||||
readonly migrationCount?: number;
|
||||
readonly recovery?: ClusterControlStartupRecoverySummary;
|
||||
}
|
||||
|
||||
export interface ClusterControlRuntimeActivationOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: DeploymentProfile;
|
||||
readonly readiness: ClusterControlReadinessProbe;
|
||||
readonly create: (
|
||||
evidence: ClusterControlReadinessEvidence,
|
||||
) => ClusterControlActivationStack;
|
||||
readonly audit: (
|
||||
record: ClusterControlActivationAudit,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export type ClusterControlRuntimeActivationResult =
|
||||
| { readonly status: 'disabled'; stop(): Promise<'stopped'> }
|
||||
| {
|
||||
readonly status: 'active';
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
readonly recovery: ClusterControlStartupRecoverySummary;
|
||||
stop(): Promise<ClusterControlStopResult>;
|
||||
};
|
||||
|
||||
const DISABLED_STOP = async (): Promise<'stopped'> => 'stopped';
|
||||
|
||||
function auditEvidence(
|
||||
evidence: ClusterControlReadinessEvidence,
|
||||
): Pick<
|
||||
ClusterControlActivationAudit,
|
||||
'contractName' | 'contractVersion' | 'serverMajor' | 'migrationCount'
|
||||
> {
|
||||
return {
|
||||
contractName: evidence.contractName,
|
||||
contractVersion: evidence.contractVersion,
|
||||
serverMajor: evidence.serverMajor,
|
||||
migrationCount: evidence.migrationIds.length,
|
||||
};
|
||||
}
|
||||
|
||||
function assertSafeRecovery(
|
||||
recovery: ClusterControlStartupRecoverySummary,
|
||||
): void {
|
||||
if (!recovery.safe || recovery.remaining !== 0 || recovery.failed !== 0) {
|
||||
throw new Error('Cluster-control startup recovery did not converge safely');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enforces readiness -> assembly -> recovery -> lifecycle -> admission order.
|
||||
* The factory is deliberately called only after schema/role readiness, so an
|
||||
* invalid cluster database cannot even construct business repositories.
|
||||
*/
|
||||
export async function activateClusterControlRuntime(
|
||||
options: ClusterControlRuntimeActivationOptions,
|
||||
): Promise<ClusterControlRuntimeActivationResult> {
|
||||
const enabled = options.enabled ?? false;
|
||||
if (!enabled) {
|
||||
await options.audit({ state: 'disabled' });
|
||||
return { status: 'disabled', stop: DISABLED_STOP };
|
||||
}
|
||||
if (options.profile !== 'cluster-control') {
|
||||
throw new TypeError(
|
||||
`Deployment profile ${options.profile} cannot activate cluster-control`,
|
||||
);
|
||||
}
|
||||
|
||||
let evidence: ClusterControlReadinessEvidence | undefined;
|
||||
let stack: ClusterControlActivationStack | undefined;
|
||||
let disposeAdmission: (() => void) | undefined;
|
||||
try {
|
||||
evidence = await options.readiness.assertReady();
|
||||
await options.audit({ state: 'schema_ready', ...auditEvidence(evidence) });
|
||||
stack = options.create(evidence);
|
||||
const recovery = await stack.reconcile();
|
||||
assertSafeRecovery(recovery);
|
||||
await options.audit({
|
||||
state: 'reconciled',
|
||||
...auditEvidence(evidence),
|
||||
recovery,
|
||||
});
|
||||
if (!(await stack.startLifecycles())) {
|
||||
throw new Error('Cluster-control lifecycles did not start');
|
||||
}
|
||||
disposeAdmission = stack.installAdmission();
|
||||
await options.audit({
|
||||
state: 'active',
|
||||
...auditEvidence(evidence),
|
||||
recovery,
|
||||
});
|
||||
|
||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
return {
|
||||
status: 'active',
|
||||
evidence,
|
||||
recovery,
|
||||
stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
let admissionError: unknown;
|
||||
try {
|
||||
disposeAdmission?.();
|
||||
} catch (error) {
|
||||
admissionError = error;
|
||||
}
|
||||
disposeAdmission = undefined;
|
||||
const result = await stack!.stop();
|
||||
if (admissionError) {
|
||||
try {
|
||||
await options.audit({
|
||||
state: 'failed',
|
||||
...auditEvidence(evidence!),
|
||||
});
|
||||
} catch {
|
||||
// Preserve the admission cleanup failure after stopping the stack.
|
||||
}
|
||||
throw admissionError;
|
||||
}
|
||||
try {
|
||||
await options.audit({
|
||||
state: 'stopped',
|
||||
...auditEvidence(evidence!),
|
||||
recovery,
|
||||
});
|
||||
} catch {
|
||||
// Diagnostic failure cannot reverse stopped ownership.
|
||||
}
|
||||
return result;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
try {
|
||||
disposeAdmission?.();
|
||||
} catch {
|
||||
// Preserve the activation failure and continue stopping the stack.
|
||||
}
|
||||
if (stack) {
|
||||
try {
|
||||
await stack.stop();
|
||||
} catch {
|
||||
// Preserve the activation failure after best-effort cleanup.
|
||||
}
|
||||
}
|
||||
try {
|
||||
await options.audit({
|
||||
state: 'failed',
|
||||
...(evidence ? auditEvidence(evidence) : {}),
|
||||
});
|
||||
} catch {
|
||||
// Diagnostic failure cannot replace the activation failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { RunAttemptStatus } from '../domain/run';
|
||||
|
||||
export class RunCommandError extends Error {
|
||||
constructor(message: string, public readonly code: string) {
|
||||
super(message);
|
||||
this.name = new.target.name;
|
||||
}
|
||||
}
|
||||
|
||||
export class RunNotFoundError extends RunCommandError {
|
||||
constructor(public readonly runId: string) {
|
||||
super('Run does not exist', 'RUN_NOT_FOUND');
|
||||
}
|
||||
}
|
||||
|
||||
export class RunAttemptNotFoundError extends RunCommandError {
|
||||
constructor(public readonly attemptId: string) {
|
||||
super('RunAttempt does not exist', 'RUN_ATTEMPT_NOT_FOUND');
|
||||
}
|
||||
}
|
||||
|
||||
export class RunAttemptConcurrentWriteError extends RunCommandError {
|
||||
constructor(
|
||||
public readonly attemptId: string,
|
||||
public readonly expectedStatus: RunAttemptStatus,
|
||||
public readonly expectedCallbackSequence: number,
|
||||
) {
|
||||
super(
|
||||
'RunAttempt changed while applying the command',
|
||||
'RUN_ATTEMPT_CONCURRENT_WRITE',
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
import { isTerminalRunAttemptStatus } from '../domain/runStateMachine';
|
||||
import type {
|
||||
CompletionReceiptDirectoryEntry,
|
||||
CompletionReceiptOrphanDirectory,
|
||||
CompletionReceiptOwnership,
|
||||
CompletionReceiptOwnershipSource,
|
||||
} from '../ports/completionReceiptOrphanMaintenance';
|
||||
|
||||
export const MAX_ORPHAN_AUDIT_SHARDS = 32;
|
||||
export const MAX_ORPHAN_AUDIT_ENTRIES_PER_SHARD = 64;
|
||||
|
||||
export type CompletionReceiptOrphanAuditMode = 'audit' | 'quarantine';
|
||||
export type CompletionReceiptOrphanCategory =
|
||||
| 'journaled'
|
||||
| 'active_attempt'
|
||||
| 'young_terminal_attempt'
|
||||
| 'terminal_orphan'
|
||||
| 'young_unknown_receipt'
|
||||
| 'unknown_receipt'
|
||||
| 'young_temporary'
|
||||
| 'stale_temporary'
|
||||
| 'young_unknown_entry'
|
||||
| 'unknown_entry'
|
||||
| 'unsafe_entry';
|
||||
export type CompletionReceiptOrphanAction =
|
||||
| 'retained'
|
||||
| 'eligible'
|
||||
| 'blocked_overflow'
|
||||
| 'quarantined'
|
||||
| 'changed';
|
||||
|
||||
export interface CompletionReceiptOrphanAuditEntry {
|
||||
shard: string;
|
||||
name: string;
|
||||
category: CompletionReceiptOrphanCategory;
|
||||
action: CompletionReceiptOrphanAction;
|
||||
ageMs: number;
|
||||
attemptId?: string;
|
||||
attemptStatus?: string;
|
||||
quarantineRef?: string;
|
||||
}
|
||||
|
||||
export interface CompletionReceiptOrphanAuditReport {
|
||||
schemaVersion: 1;
|
||||
mode: CompletionReceiptOrphanAuditMode;
|
||||
observedAtMs: number;
|
||||
minimumAgeMs: number;
|
||||
startShard: string;
|
||||
nextShard: string;
|
||||
wrapped: boolean;
|
||||
shardCount: number;
|
||||
maxEntriesPerShard: number;
|
||||
scannedEntries: number;
|
||||
overflowShards: readonly string[];
|
||||
entries: readonly CompletionReceiptOrphanAuditEntry[];
|
||||
counts: Readonly<Record<CompletionReceiptOrphanCategory, number>>;
|
||||
}
|
||||
|
||||
export interface CompletionReceiptOrphanAuditorOptions {
|
||||
mode?: CompletionReceiptOrphanAuditMode;
|
||||
observedAtMs?: number;
|
||||
minimumAgeMs?: number;
|
||||
startShard?: number;
|
||||
shardCount?: number;
|
||||
maxEntriesPerShard?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
const CATEGORIES: readonly CompletionReceiptOrphanCategory[] = [
|
||||
'journaled',
|
||||
'active_attempt',
|
||||
'young_terminal_attempt',
|
||||
'terminal_orphan',
|
||||
'young_unknown_receipt',
|
||||
'unknown_receipt',
|
||||
'young_temporary',
|
||||
'stale_temporary',
|
||||
'young_unknown_entry',
|
||||
'unknown_entry',
|
||||
'unsafe_entry',
|
||||
];
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
function shardName(value: number): string {
|
||||
return value.toString(16).padStart(2, '0');
|
||||
}
|
||||
|
||||
function classify(
|
||||
entry: CompletionReceiptDirectoryEntry,
|
||||
ownership: CompletionReceiptOwnership | undefined,
|
||||
oldEnough: boolean,
|
||||
): { category: CompletionReceiptOrphanCategory; eligible: boolean } {
|
||||
if (entry.kind === 'unsafe') {
|
||||
return { category: 'unsafe_entry', eligible: false };
|
||||
}
|
||||
if (entry.kind === 'temporary') {
|
||||
return oldEnough
|
||||
? { category: 'stale_temporary', eligible: true }
|
||||
: { category: 'young_temporary', eligible: false };
|
||||
}
|
||||
if (entry.kind === 'unknown') {
|
||||
return oldEnough
|
||||
? { category: 'unknown_entry', eligible: true }
|
||||
: { category: 'young_unknown_entry', eligible: false };
|
||||
}
|
||||
if (ownership?.journalState) {
|
||||
return { category: 'journaled', eligible: false };
|
||||
}
|
||||
if (ownership?.attemptStatus) {
|
||||
if (!isTerminalRunAttemptStatus(ownership.attemptStatus)) {
|
||||
return { category: 'active_attempt', eligible: false };
|
||||
}
|
||||
return oldEnough
|
||||
? { category: 'terminal_orphan', eligible: true }
|
||||
: { category: 'young_terminal_attempt', eligible: false };
|
||||
}
|
||||
return oldEnough
|
||||
? { category: 'unknown_receipt', eligible: true }
|
||||
: { category: 'young_unknown_receipt', eligible: false };
|
||||
}
|
||||
|
||||
export class CompletionReceiptOrphanAuditor {
|
||||
constructor(
|
||||
private readonly directory: CompletionReceiptOrphanDirectory,
|
||||
private readonly ownership: CompletionReceiptOwnershipSource,
|
||||
) {}
|
||||
|
||||
async run(
|
||||
options: CompletionReceiptOrphanAuditorOptions = {},
|
||||
): Promise<CompletionReceiptOrphanAuditReport> {
|
||||
const mode = options.mode ?? 'audit';
|
||||
if (mode !== 'audit' && mode !== 'quarantine') {
|
||||
throw new RangeError('mode must be audit or quarantine');
|
||||
}
|
||||
const observedAtMs =
|
||||
options.observedAtMs ?? options.clock?.now() ?? Date.now();
|
||||
const minimumAgeMs = options.minimumAgeMs ?? 5 * 60_000;
|
||||
const startShard = options.startShard ?? 0;
|
||||
const shardCount = options.shardCount ?? 8;
|
||||
const maxEntriesPerShard = options.maxEntriesPerShard ?? 32;
|
||||
assertIntegerBetween(
|
||||
'observedAtMs',
|
||||
observedAtMs,
|
||||
0,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'minimumAgeMs',
|
||||
minimumAgeMs,
|
||||
0,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
);
|
||||
assertIntegerBetween('startShard', startShard, 0, 255);
|
||||
assertIntegerBetween('shardCount', shardCount, 1, MAX_ORPHAN_AUDIT_SHARDS);
|
||||
assertIntegerBetween(
|
||||
'maxEntriesPerShard',
|
||||
maxEntriesPerShard,
|
||||
1,
|
||||
MAX_ORPHAN_AUDIT_ENTRIES_PER_SHARD,
|
||||
);
|
||||
|
||||
const counts = Object.fromEntries(
|
||||
CATEGORIES.map((category) => [category, 0]),
|
||||
) as Record<CompletionReceiptOrphanCategory, number>;
|
||||
const overflowShards: string[] = [];
|
||||
const entries: CompletionReceiptOrphanAuditEntry[] = [];
|
||||
|
||||
for (let offset = 0; offset < shardCount; offset += 1) {
|
||||
const shard = shardName((startShard + offset) % 256);
|
||||
const snapshot = await this.directory.inspectShard(
|
||||
shard,
|
||||
maxEntriesPerShard,
|
||||
);
|
||||
if (snapshot.shard !== shard) {
|
||||
throw new Error('Completion receipt directory returned another shard');
|
||||
}
|
||||
if (snapshot.entries.length > maxEntriesPerShard) {
|
||||
throw new Error('Completion receipt directory exceeded its hard limit');
|
||||
}
|
||||
if (snapshot.overflow) overflowShards.push(shard);
|
||||
|
||||
const attemptIds = snapshot.entries.flatMap((entry) =>
|
||||
entry.kind === 'receipt' && entry.attemptId ? [entry.attemptId] : [],
|
||||
);
|
||||
const ownership = await this.ownership.lookup(attemptIds);
|
||||
for (const entry of snapshot.entries) {
|
||||
const ageMs = Math.max(0, observedAtMs - entry.modifiedAtMs);
|
||||
const classification = classify(
|
||||
entry,
|
||||
entry.attemptId ? ownership.get(entry.attemptId) : undefined,
|
||||
ageMs >= minimumAgeMs,
|
||||
);
|
||||
counts[classification.category] += 1;
|
||||
let action: CompletionReceiptOrphanAction = classification.eligible
|
||||
? 'eligible'
|
||||
: 'retained';
|
||||
let quarantineRef: string | undefined;
|
||||
if (classification.eligible && mode === 'quarantine') {
|
||||
if (snapshot.overflow) {
|
||||
action = 'blocked_overflow';
|
||||
} else {
|
||||
const result = await this.directory.quarantine(entry);
|
||||
action = result.status;
|
||||
if (result.status === 'quarantined') {
|
||||
quarantineRef = result.reference;
|
||||
}
|
||||
}
|
||||
}
|
||||
entries.push({
|
||||
shard,
|
||||
name: entry.name,
|
||||
category: classification.category,
|
||||
action,
|
||||
ageMs,
|
||||
...(entry.attemptId ? { attemptId: entry.attemptId } : {}),
|
||||
...(entry.attemptId && ownership.get(entry.attemptId)?.attemptStatus
|
||||
? {
|
||||
attemptStatus: ownership.get(entry.attemptId)!.attemptStatus,
|
||||
}
|
||||
: {}),
|
||||
...(quarantineRef ? { quarantineRef } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const absoluteNextShard = startShard + shardCount;
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
mode,
|
||||
observedAtMs,
|
||||
minimumAgeMs,
|
||||
startShard: shardName(startShard),
|
||||
nextShard: shardName(absoluteNextShard % 256),
|
||||
wrapped: absoluteNextShard > 255,
|
||||
shardCount,
|
||||
maxEntriesPerShard,
|
||||
scannedEntries: entries.length,
|
||||
overflowShards,
|
||||
entries,
|
||||
counts,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
import { timingSafeEqual } from 'crypto';
|
||||
import {
|
||||
decodeLocalSecretPlaintext,
|
||||
decryptLocalSecretEnvelopeToBuffer,
|
||||
encryptLocalSecretEnvelope,
|
||||
type LocalSecretNonceFactory,
|
||||
} from '../adapters/crypto/aes256GcmLocalSecret';
|
||||
import {
|
||||
LOCAL_SECRET_ALGORITHM,
|
||||
LocalSecretMutationConflictError,
|
||||
LocalSecretUnavailableError,
|
||||
LocalSecretVersionConflictError,
|
||||
assertLocalSecretMutationId,
|
||||
assertLocalSecretName,
|
||||
assertLocalSecretPlaintext,
|
||||
assertLocalSecretProjectId,
|
||||
assertLocalSecretKeyId,
|
||||
createLocalSecretRef,
|
||||
parseLocalSecretRef,
|
||||
type LocalSecretEnvelope,
|
||||
} from '../domain/localSecret';
|
||||
import { assertRunDispatchCandidate } from '../domain/runDispatchCandidate';
|
||||
import type { LocalSecretEnvelopeRepository } from '../ports/localSecretEnvelopeRepository';
|
||||
import type {
|
||||
LocalSecretEnvironmentProvider,
|
||||
LocalSecretEnvironmentRequest,
|
||||
} from '../ports/localSecretEnvironmentProvider';
|
||||
import type {
|
||||
LocalSecretKeyMaterial,
|
||||
LocalSecretKeyProvider,
|
||||
} from '../ports/localSecretKeyProvider';
|
||||
|
||||
export interface PutEncryptedLocalSecretCommand {
|
||||
projectId: string;
|
||||
name: string;
|
||||
plaintext: string;
|
||||
mutationId: string;
|
||||
expectedCurrentVersion: number;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
export interface PutEncryptedLocalSecretResult {
|
||||
status: 'inserted' | 'existing';
|
||||
version: number;
|
||||
secretRef: string;
|
||||
}
|
||||
|
||||
export { LocalSecretMutationConflictError, LocalSecretVersionConflictError };
|
||||
|
||||
function assertPutCommand(command: PutEncryptedLocalSecretCommand): void {
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Local Secret write command must be an object');
|
||||
}
|
||||
assertLocalSecretProjectId(command.projectId);
|
||||
assertLocalSecretName(command.name);
|
||||
assertLocalSecretPlaintext(command.plaintext);
|
||||
assertLocalSecretMutationId(command.mutationId);
|
||||
if (
|
||||
!Number.isSafeInteger(command.expectedCurrentVersion) ||
|
||||
command.expectedCurrentVersion < 0 ||
|
||||
command.expectedCurrentVersion >= 2_147_483_647
|
||||
) {
|
||||
throw new TypeError('Local Secret expected current version is invalid');
|
||||
}
|
||||
if (!Number.isSafeInteger(command.createdAtMs) || command.createdAtMs < 0) {
|
||||
throw new TypeError('Local Secret creation time is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function ownedKeyMaterial(
|
||||
material: LocalSecretKeyMaterial | null,
|
||||
expectedKeyId?: string,
|
||||
): { keyId: string; key: Buffer } {
|
||||
if (!material || !(material.key instanceof Uint8Array)) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
try {
|
||||
assertLocalSecretKeyId(material.keyId);
|
||||
if (
|
||||
(expectedKeyId !== undefined && material.keyId !== expectedKeyId) ||
|
||||
material.key.byteLength !== 32
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
return { keyId: material.keyId, key: Buffer.from(material.key) };
|
||||
} catch {
|
||||
throw new LocalSecretUnavailableError();
|
||||
} finally {
|
||||
material.key.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
function plaintextMatches(
|
||||
envelope: LocalSecretEnvelope,
|
||||
key: Uint8Array,
|
||||
expected: string,
|
||||
): boolean {
|
||||
const actual = decryptLocalSecretEnvelopeToBuffer(envelope, key);
|
||||
const wanted = Buffer.from(expected, 'utf8');
|
||||
try {
|
||||
return actual.length === wanted.length && timingSafeEqual(actual, wanted);
|
||||
} finally {
|
||||
actual.fill(0);
|
||||
wanted.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
export class EncryptedLocalSecretService
|
||||
implements LocalSecretEnvironmentProvider
|
||||
{
|
||||
constructor(
|
||||
private readonly envelopes: LocalSecretEnvelopeRepository,
|
||||
private readonly keys: LocalSecretKeyProvider,
|
||||
private readonly nonceFactory?: LocalSecretNonceFactory,
|
||||
) {}
|
||||
|
||||
async put(
|
||||
command: PutEncryptedLocalSecretCommand,
|
||||
): Promise<PutEncryptedLocalSecretResult> {
|
||||
assertPutCommand(command);
|
||||
try {
|
||||
return await this.putValidated(command);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof LocalSecretVersionConflictError ||
|
||||
error instanceof LocalSecretMutationConflictError ||
|
||||
error instanceof LocalSecretUnavailableError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
private async putValidated(
|
||||
command: PutEncryptedLocalSecretCommand,
|
||||
): Promise<PutEncryptedLocalSecretResult> {
|
||||
const existing = await this.envelopes.findByMutation(
|
||||
command.projectId,
|
||||
command.name,
|
||||
command.mutationId,
|
||||
);
|
||||
if (existing) {
|
||||
const material = ownedKeyMaterial(
|
||||
await this.keys.resolve(existing.keyId),
|
||||
existing.keyId,
|
||||
);
|
||||
try {
|
||||
if (
|
||||
existing.version !== command.expectedCurrentVersion + 1 ||
|
||||
!plaintextMatches(existing, material.key, command.plaintext)
|
||||
) {
|
||||
throw new LocalSecretMutationConflictError();
|
||||
}
|
||||
} finally {
|
||||
material.key.fill(0);
|
||||
}
|
||||
return this.result('existing', existing);
|
||||
}
|
||||
|
||||
const material = ownedKeyMaterial(await this.keys.active());
|
||||
try {
|
||||
const envelope = encryptLocalSecretEnvelope(
|
||||
{
|
||||
projectId: command.projectId,
|
||||
name: command.name,
|
||||
version: command.expectedCurrentVersion + 1,
|
||||
mutationId: command.mutationId,
|
||||
keyId: material.keyId,
|
||||
algorithm: LOCAL_SECRET_ALGORITHM,
|
||||
createdAtMs: command.createdAtMs,
|
||||
},
|
||||
command.plaintext,
|
||||
material.key,
|
||||
this.nonceFactory,
|
||||
);
|
||||
const appended = await this.envelopes.append({
|
||||
envelope,
|
||||
expectedCurrentVersion: command.expectedCurrentVersion,
|
||||
});
|
||||
if (appended.status === 'existing') {
|
||||
const existingMaterial =
|
||||
appended.envelope.keyId === material.keyId
|
||||
? material
|
||||
: ownedKeyMaterial(
|
||||
await this.keys.resolve(appended.envelope.keyId),
|
||||
appended.envelope.keyId,
|
||||
);
|
||||
try {
|
||||
if (
|
||||
appended.envelope.version !== command.expectedCurrentVersion + 1 ||
|
||||
!plaintextMatches(
|
||||
appended.envelope,
|
||||
existingMaterial.key,
|
||||
command.plaintext,
|
||||
)
|
||||
) {
|
||||
throw new LocalSecretMutationConflictError();
|
||||
}
|
||||
} finally {
|
||||
if (existingMaterial !== material) existingMaterial.key.fill(0);
|
||||
}
|
||||
}
|
||||
return this.result(appended.status, appended.envelope);
|
||||
} finally {
|
||||
material.key.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
async resolve(
|
||||
request: Readonly<LocalSecretEnvironmentRequest>,
|
||||
): Promise<readonly string[] | null> {
|
||||
const cachedKeys = new Map<string, Buffer>();
|
||||
try {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
assertRunDispatchCandidate(request.candidate);
|
||||
if (
|
||||
!Array.isArray(request.secretRefs) ||
|
||||
request.secretRefs.length > 64
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
const references = request.secretRefs.map(parseLocalSecretRef);
|
||||
if (
|
||||
references.some(
|
||||
(reference) => reference.projectId !== request.candidate.projectId,
|
||||
)
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
const envelopes = await this.envelopes.resolveMany(references);
|
||||
if (
|
||||
envelopes.length !== references.length ||
|
||||
envelopes.some((item) => !item)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const plaintext: string[] = [];
|
||||
for (const envelope of envelopes as readonly LocalSecretEnvelope[]) {
|
||||
let key = cachedKeys.get(envelope.keyId);
|
||||
if (!key) {
|
||||
const material = ownedKeyMaterial(
|
||||
await this.keys.resolve(envelope.keyId),
|
||||
envelope.keyId,
|
||||
);
|
||||
key = material.key;
|
||||
cachedKeys.set(envelope.keyId, key);
|
||||
}
|
||||
const bytes = decryptLocalSecretEnvelopeToBuffer(envelope, key);
|
||||
try {
|
||||
plaintext.push(decodeLocalSecretPlaintext(bytes));
|
||||
} finally {
|
||||
bytes.fill(0);
|
||||
}
|
||||
}
|
||||
return Object.freeze(plaintext);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalSecretUnavailableError) throw error;
|
||||
throw new LocalSecretUnavailableError();
|
||||
} finally {
|
||||
for (const key of cachedKeys.values()) key.fill(0);
|
||||
cachedKeys.clear();
|
||||
}
|
||||
}
|
||||
|
||||
private result(
|
||||
status: PutEncryptedLocalSecretResult['status'],
|
||||
envelope: LocalSecretEnvelope,
|
||||
): PutEncryptedLocalSecretResult {
|
||||
return Object.freeze({
|
||||
status,
|
||||
version: envelope.version,
|
||||
secretRef: createLocalSecretRef({
|
||||
projectId: envelope.projectId,
|
||||
name: envelope.name,
|
||||
version: envelope.version,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import type { DeploymentProfile } from '../domain/deploymentProfile';
|
||||
import type { WorkerRecord } from '../domain/worker';
|
||||
import type { WorkerExecutionDrainer } from '../ports/workerExecutionDrainer';
|
||||
import type {
|
||||
WorkerHeartbeatLifecycle,
|
||||
WorkerHeartbeatStopResult,
|
||||
} from './workerHeartbeatLifecycle';
|
||||
|
||||
export type HeadlessWorkerStopResult =
|
||||
| 'stopped'
|
||||
| 'not_started'
|
||||
| 'executions_timed_out'
|
||||
| 'heartbeat_timed_out'
|
||||
| 'heartbeat_disconnect_failed';
|
||||
|
||||
export interface HeadlessWorkerBootstrapOptions {
|
||||
enabled?: boolean;
|
||||
profile: DeploymentProfile;
|
||||
heartbeat: WorkerHeartbeatLifecycle;
|
||||
executions: WorkerExecutionDrainer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Independent Worker boot topology. It owns no HTTP panel, Scheduler, SQLite
|
||||
* control-plane repository, or local Primary router. Shutdown first advertises
|
||||
* zero capacity, then waits for the execution plane, and only then marks the
|
||||
* Worker session offline.
|
||||
*/
|
||||
export class HeadlessWorkerRuntime {
|
||||
private started = false;
|
||||
|
||||
constructor(
|
||||
private readonly heartbeat: WorkerHeartbeatLifecycle,
|
||||
private readonly executions: WorkerExecutionDrainer,
|
||||
) {}
|
||||
|
||||
currentSession(): WorkerRecord | undefined {
|
||||
return this.heartbeat.currentSession();
|
||||
}
|
||||
|
||||
async start(): Promise<boolean> {
|
||||
if (this.started) return false;
|
||||
const started = await this.heartbeat.start();
|
||||
this.started = started;
|
||||
return started;
|
||||
}
|
||||
|
||||
async drainAndStop(): Promise<HeadlessWorkerStopResult> {
|
||||
if (!this.started) return 'not_started';
|
||||
await this.heartbeat.drain();
|
||||
if ((await this.executions.drain()) === 'timed_out') {
|
||||
return 'executions_timed_out';
|
||||
}
|
||||
const heartbeatResult: WorkerHeartbeatStopResult =
|
||||
await this.heartbeat.stop();
|
||||
if (heartbeatResult === 'timed_out') return 'heartbeat_timed_out';
|
||||
if (heartbeatResult === 'disconnect_failed') {
|
||||
return 'heartbeat_disconnect_failed';
|
||||
}
|
||||
this.started = false;
|
||||
return 'stopped';
|
||||
}
|
||||
}
|
||||
|
||||
export type HeadlessWorkerBootstrapResult =
|
||||
| { status: 'disabled' }
|
||||
| { status: 'active'; runtime: HeadlessWorkerRuntime };
|
||||
|
||||
export async function bootstrapHeadlessWorkerRuntime({
|
||||
enabled = false,
|
||||
profile,
|
||||
heartbeat,
|
||||
executions,
|
||||
}: HeadlessWorkerBootstrapOptions): Promise<HeadlessWorkerBootstrapResult> {
|
||||
if (!enabled) return { status: 'disabled' };
|
||||
if (profile !== 'worker') {
|
||||
throw new TypeError(
|
||||
`Deployment profile ${profile} cannot activate the headless Worker runtime`,
|
||||
);
|
||||
}
|
||||
const runtime = new HeadlessWorkerRuntime(heartbeat, executions);
|
||||
if (!(await runtime.start())) {
|
||||
throw new Error('Headless Worker runtime did not start');
|
||||
}
|
||||
return { status: 'active', runtime };
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { createHash } from 'crypto';
|
||||
import jwt, { type JwtPayload } from 'jsonwebtoken';
|
||||
import {
|
||||
assertAuthenticatedPrincipalActive,
|
||||
normalizeAuthenticatedPrincipal,
|
||||
type AuthenticatedPrincipal,
|
||||
} from '../domain/authenticatedPrincipal';
|
||||
import {
|
||||
IdentityDirectoryUnavailableError,
|
||||
LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
} from '../domain/identityDirectory';
|
||||
import type { IdentityDirectoryRepository } from '../ports/identityDirectoryRepository';
|
||||
import type {
|
||||
LegacyPanelPlatform,
|
||||
LegacyPanelSessionSource,
|
||||
} from '../ports/legacyPanelSessionSource';
|
||||
|
||||
const LEGACY_PANEL_JWT_PATTERN =
|
||||
/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/;
|
||||
const MAX_LEGACY_PANEL_TOKEN_LENGTH = 4096;
|
||||
const MAX_LEGACY_PANEL_JWT_DATA_LENGTH = 256;
|
||||
|
||||
export interface AuthenticateLegacyPanelSessionRequest {
|
||||
token: string;
|
||||
platform: LegacyPanelPlatform;
|
||||
nowMs: number;
|
||||
}
|
||||
|
||||
export class LegacyPanelAuthenticationRejectedError extends Error {
|
||||
readonly code = 'LEGACY_PANEL_AUTHENTICATION_REJECTED';
|
||||
|
||||
constructor() {
|
||||
super('Legacy panel authentication was rejected');
|
||||
this.name = 'LegacyPanelAuthenticationRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacyPanelAuthenticationUnavailableError extends Error {
|
||||
readonly code = 'LEGACY_PANEL_AUTHENTICATION_UNAVAILABLE';
|
||||
|
||||
constructor() {
|
||||
super('Legacy panel authentication is unavailable');
|
||||
this.name = 'LegacyPanelAuthenticationUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function assertExactRequest(request: AuthenticateLegacyPanelSessionRequest) {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError('Legacy panel authentication request is invalid');
|
||||
}
|
||||
const keys = Object.keys(request).sort();
|
||||
const expected = ['nowMs', 'platform', 'token'];
|
||||
if (
|
||||
keys.length !== expected.length ||
|
||||
keys.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
throw new TypeError('Legacy panel authentication request shape is invalid');
|
||||
}
|
||||
if (
|
||||
typeof request.token !== 'string' ||
|
||||
request.token.length < 1 ||
|
||||
request.token.length > MAX_LEGACY_PANEL_TOKEN_LENGTH ||
|
||||
!LEGACY_PANEL_JWT_PATTERN.test(request.token)
|
||||
) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
if (request.platform !== 'desktop' && request.platform !== 'mobile') {
|
||||
throw new TypeError('Legacy panel platform is invalid');
|
||||
}
|
||||
if (!Number.isSafeInteger(request.nowMs) || request.nowMs < 0) {
|
||||
throw new TypeError('Legacy panel authentication time is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function normalizePayload(value: string | JwtPayload): {
|
||||
authenticatedAtMs: number;
|
||||
expiresAtMs: number;
|
||||
} {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
const keys = Object.keys(value).sort();
|
||||
const expected = ['data', 'exp', 'iat'];
|
||||
if (
|
||||
keys.length !== expected.length ||
|
||||
keys.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
if (
|
||||
typeof value.data !== 'string' ||
|
||||
value.data.length < 1 ||
|
||||
value.data.length > MAX_LEGACY_PANEL_JWT_DATA_LENGTH ||
|
||||
!Number.isSafeInteger(value.iat) ||
|
||||
value.iat! < 0 ||
|
||||
!Number.isSafeInteger(value.exp) ||
|
||||
value.exp! <= value.iat!
|
||||
) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
const authenticatedAtMs = value.iat! * 1000;
|
||||
const expiresAtMs = value.exp! * 1000;
|
||||
if (
|
||||
!Number.isSafeInteger(authenticatedAtMs) ||
|
||||
!Number.isSafeInteger(expiresAtMs)
|
||||
) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
return { authenticatedAtMs, expiresAtMs };
|
||||
}
|
||||
|
||||
export class LegacyPanelAuthenticationService {
|
||||
constructor(
|
||||
private readonly identityDirectory: IdentityDirectoryRepository,
|
||||
private readonly sessions: LegacyPanelSessionSource,
|
||||
private readonly jwtSecret: string,
|
||||
) {
|
||||
if (
|
||||
typeof jwtSecret !== 'string' ||
|
||||
jwtSecret.length < 1 ||
|
||||
jwtSecret.length > 4096
|
||||
) {
|
||||
throw new TypeError('Legacy panel JWT secret is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async authenticate(
|
||||
request: AuthenticateLegacyPanelSessionRequest,
|
||||
): Promise<Readonly<AuthenticatedPrincipal>> {
|
||||
assertExactRequest(request);
|
||||
let payload: { authenticatedAtMs: number; expiresAtMs: number };
|
||||
try {
|
||||
payload = normalizePayload(
|
||||
jwt.verify(request.token, this.jwtSecret, {
|
||||
algorithms: ['HS384'],
|
||||
clockTimestamp: Math.floor(request.nowMs / 1000),
|
||||
clockTolerance: 0,
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
if (
|
||||
payload.authenticatedAtMs > request.nowMs ||
|
||||
payload.expiresAtMs <= request.nowMs
|
||||
) {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
|
||||
let active: boolean;
|
||||
try {
|
||||
active = await this.sessions.isActive(request.token, request.platform);
|
||||
} catch {
|
||||
throw new LegacyPanelAuthenticationUnavailableError();
|
||||
}
|
||||
if (!active) throw new LegacyPanelAuthenticationRejectedError();
|
||||
|
||||
let subject;
|
||||
try {
|
||||
subject = await this.identityDirectory.resolveAuthenticationSubject(
|
||||
LEGACY_PANEL_IDENTITY_PROVIDER,
|
||||
LEGACY_PANEL_PROVIDER_SUBJECT,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof IdentityDirectoryUnavailableError) {
|
||||
throw new LegacyPanelAuthenticationUnavailableError();
|
||||
}
|
||||
throw new LegacyPanelAuthenticationUnavailableError();
|
||||
}
|
||||
if (!subject || subject.type !== 'user') {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
|
||||
const principal = normalizeAuthenticatedPrincipal({
|
||||
subject,
|
||||
authenticationId: `legacy_panel:${createHash('sha256')
|
||||
.update(request.token, 'utf8')
|
||||
.digest('hex')}`,
|
||||
authenticatedAtMs: payload.authenticatedAtMs,
|
||||
expiresAtMs: payload.expiresAtMs,
|
||||
assurance: 'single_factor',
|
||||
});
|
||||
try {
|
||||
assertAuthenticatedPrincipalActive(principal, request.nowMs);
|
||||
} catch {
|
||||
throw new LegacyPanelAuthenticationRejectedError();
|
||||
}
|
||||
return principal;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
import type { ExecutionOrigin } from '../domain/run';
|
||||
import { selectOneLegacyExecution } from '../domain/legacyExecutionSelection';
|
||||
import type {
|
||||
LegacyExecutionCallbackFact,
|
||||
LegacyExecutionCancellationFact,
|
||||
LegacyExecutionSelector,
|
||||
} from '../ports/legacyExecutionCorrelation';
|
||||
import type {
|
||||
ActiveLegacyShadowRun,
|
||||
LegacyShadowRunLocator,
|
||||
} from '../ports/legacyShadowRunLocator';
|
||||
import type { LegacyShadowRunWriter } from './legacyShadowRunWriter';
|
||||
|
||||
export type LegacyCorrelationOperation =
|
||||
| 'cancel_all'
|
||||
| 'cancel_one'
|
||||
| 'callback';
|
||||
export type LegacyCorrelationFailureReason =
|
||||
| 'ambiguous'
|
||||
| 'truncated'
|
||||
| 'unmatched'
|
||||
| 'write_failed';
|
||||
|
||||
export interface LegacyCorrelationFailure {
|
||||
operation: LegacyCorrelationOperation;
|
||||
reason: LegacyCorrelationFailureReason;
|
||||
legacyCronId: number;
|
||||
candidateCount: number;
|
||||
}
|
||||
|
||||
export interface LegacyCorrelationReporter {
|
||||
failure(failure: LegacyCorrelationFailure): void;
|
||||
}
|
||||
|
||||
export interface LegacyCorrelationResult {
|
||||
matched: number;
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
export class LegacyShadowRunCorrelator {
|
||||
constructor(
|
||||
private readonly locator: LegacyShadowRunLocator,
|
||||
private readonly writer: LegacyShadowRunWriter,
|
||||
private readonly reporter: LegacyCorrelationReporter,
|
||||
) {}
|
||||
|
||||
async cancel(
|
||||
fact: LegacyExecutionCancellationFact,
|
||||
origins: readonly ExecutionOrigin[],
|
||||
): Promise<LegacyCorrelationResult> {
|
||||
const lookup = await this.locator.listActiveByLegacyCron({
|
||||
legacyCronId: fact.legacyCronId,
|
||||
origins,
|
||||
});
|
||||
const candidates =
|
||||
fact.scope === 'all'
|
||||
? [...lookup.candidates]
|
||||
: this.selectOne(lookup.candidates, fact);
|
||||
if (lookup.truncated) {
|
||||
this.report({
|
||||
operation: fact.scope === 'all' ? 'cancel_all' : 'cancel_one',
|
||||
reason: 'truncated',
|
||||
legacyCronId: fact.legacyCronId,
|
||||
candidateCount: lookup.candidates.length,
|
||||
});
|
||||
}
|
||||
if (fact.scope === 'one' && candidates.length !== 1) {
|
||||
this.reportSelectionFailure('cancel_one', fact, lookup.candidates);
|
||||
return { matched: 0, truncated: lookup.truncated };
|
||||
}
|
||||
|
||||
let matched = 0;
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
await this.writer.cancelled(
|
||||
{ runId: candidate.runId, attemptId: candidate.attemptId },
|
||||
{ atMs: fact.atMs, reason: fact.reason },
|
||||
);
|
||||
matched += 1;
|
||||
} catch {
|
||||
this.report({
|
||||
operation: fact.scope === 'all' ? 'cancel_all' : 'cancel_one',
|
||||
reason: 'write_failed',
|
||||
legacyCronId: fact.legacyCronId,
|
||||
candidateCount: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
return { matched, truncated: lookup.truncated };
|
||||
}
|
||||
|
||||
async callback(
|
||||
fact: LegacyExecutionCallbackFact,
|
||||
origins: readonly ExecutionOrigin[],
|
||||
): Promise<LegacyCorrelationResult> {
|
||||
const lookup = await this.locator.listActiveByLegacyCron({
|
||||
legacyCronId: fact.legacyCronId,
|
||||
origins,
|
||||
});
|
||||
const candidates = this.selectOne(lookup.candidates, fact);
|
||||
if (lookup.truncated) {
|
||||
this.report({
|
||||
operation: 'callback',
|
||||
reason: 'truncated',
|
||||
legacyCronId: fact.legacyCronId,
|
||||
candidateCount: lookup.candidates.length,
|
||||
});
|
||||
}
|
||||
if (candidates.length !== 1) {
|
||||
this.reportSelectionFailure('callback', fact, lookup.candidates);
|
||||
return { matched: 0, truncated: lookup.truncated };
|
||||
}
|
||||
|
||||
const [candidate] = candidates;
|
||||
const reference = {
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
};
|
||||
try {
|
||||
if (fact.phase === 'running') {
|
||||
await this.writer.spawned(reference, {
|
||||
atMs: fact.atMs,
|
||||
...(fact.pid === undefined ? {} : { pid: fact.pid }),
|
||||
...(fact.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: fact.logArtifactId }),
|
||||
});
|
||||
await this.writer.running(reference, fact.atMs);
|
||||
} else {
|
||||
await this.writer.spawned(reference, {
|
||||
atMs: fact.atMs,
|
||||
...(fact.pid === undefined ? {} : { pid: fact.pid }),
|
||||
...(fact.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: fact.logArtifactId }),
|
||||
});
|
||||
await this.writer.exited(reference, {
|
||||
atMs: fact.atMs,
|
||||
exitCode: fact.exitCode ?? 0,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
this.report({
|
||||
operation: 'callback',
|
||||
reason: 'write_failed',
|
||||
legacyCronId: fact.legacyCronId,
|
||||
candidateCount: 1,
|
||||
});
|
||||
return { matched: 0, truncated: lookup.truncated };
|
||||
}
|
||||
return { matched: 1, truncated: lookup.truncated };
|
||||
}
|
||||
|
||||
private selectOne(
|
||||
candidates: readonly ActiveLegacyShadowRun[],
|
||||
selector: LegacyExecutionSelector,
|
||||
): ActiveLegacyShadowRun[] {
|
||||
return selectOneLegacyExecution(candidates, selector);
|
||||
}
|
||||
|
||||
private reportSelectionFailure(
|
||||
operation: LegacyCorrelationOperation,
|
||||
selector: LegacyExecutionSelector,
|
||||
candidates: readonly ActiveLegacyShadowRun[],
|
||||
): void {
|
||||
this.report({
|
||||
operation,
|
||||
reason: candidates.length === 0 ? 'unmatched' : 'ambiguous',
|
||||
legacyCronId: selector.legacyCronId,
|
||||
candidateCount: candidates.length,
|
||||
});
|
||||
}
|
||||
|
||||
private report(failure: LegacyCorrelationFailure): void {
|
||||
try {
|
||||
this.reporter.failure(failure);
|
||||
} catch {
|
||||
// Correlation diagnostics must not affect legacy execution.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import type { ExecutionOrigin } from '../domain/run';
|
||||
import type { RuntimeRolloutPolicy } from '../domain/runtimeRollout';
|
||||
import type {
|
||||
LegacyExecutionAcceptedFact,
|
||||
LegacyExecutionCancelledFact,
|
||||
LegacyExecutionExitedFact,
|
||||
LegacyExecutionObservation,
|
||||
LegacyExecutionObserver,
|
||||
LegacyExecutionRunningFact,
|
||||
LegacyExecutionSpawnedFact,
|
||||
LegacyExecutionStartFailedFact,
|
||||
} from '../ports/legacyExecutionObserver';
|
||||
import type {
|
||||
LegacyShadowRunReference,
|
||||
LegacyShadowRunWriter,
|
||||
} from './legacyShadowRunWriter';
|
||||
|
||||
export type ShadowObservationOperation =
|
||||
| 'accept'
|
||||
| 'spawned'
|
||||
| 'running'
|
||||
| 'start_failed'
|
||||
| 'exited'
|
||||
| 'cancelled';
|
||||
|
||||
export interface ShadowObservationFailure {
|
||||
origin: ExecutionOrigin;
|
||||
operation: ShadowObservationOperation;
|
||||
errorCode: string;
|
||||
runId?: string;
|
||||
attemptId?: string;
|
||||
}
|
||||
|
||||
export interface ShadowObservationReporter {
|
||||
failure(failure: ShadowObservationFailure): void;
|
||||
}
|
||||
|
||||
export interface TrackedLegacyExecutionObservation
|
||||
extends LegacyExecutionObservation {
|
||||
settled(): Promise<void>;
|
||||
}
|
||||
|
||||
const NOOP_OBSERVATION: TrackedLegacyExecutionObservation = Object.freeze({
|
||||
spawned() {},
|
||||
running() {},
|
||||
startFailed() {},
|
||||
exited() {},
|
||||
cancelled() {},
|
||||
async settled() {},
|
||||
});
|
||||
|
||||
function classifyError(error: unknown): string {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
'code' in error &&
|
||||
typeof (error as Error & { code?: unknown }).code === 'string'
|
||||
) {
|
||||
const code = (error as Error & { code: string }).code;
|
||||
return /^[A-Z0-9_]{1,64}$/.test(code) ? code : 'SHADOW_UNKNOWN';
|
||||
}
|
||||
return 'SHADOW_UNKNOWN';
|
||||
}
|
||||
|
||||
class SerialLegacyExecutionObservation
|
||||
implements TrackedLegacyExecutionObservation
|
||||
{
|
||||
private chain: Promise<LegacyShadowRunReference | null>;
|
||||
|
||||
constructor(
|
||||
private readonly origin: ExecutionOrigin,
|
||||
private readonly writer: LegacyShadowRunWriter,
|
||||
accepted: LegacyExecutionAcceptedFact,
|
||||
private readonly reporter: ShadowObservationReporter,
|
||||
) {
|
||||
this.chain = writer.accept(accepted).catch((error) => {
|
||||
this.report('accept', error);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
spawned(fact: LegacyExecutionSpawnedFact): void {
|
||||
this.enqueue('spawned', (reference) =>
|
||||
this.writer.spawned(reference, fact),
|
||||
);
|
||||
}
|
||||
|
||||
running(fact: LegacyExecutionRunningFact): void {
|
||||
this.enqueue('running', (reference) =>
|
||||
this.writer.running(reference, fact.atMs),
|
||||
);
|
||||
}
|
||||
|
||||
startFailed(fact: LegacyExecutionStartFailedFact): void {
|
||||
this.enqueue('start_failed', (reference) =>
|
||||
this.writer.startFailed(reference, fact),
|
||||
);
|
||||
}
|
||||
|
||||
exited(fact: LegacyExecutionExitedFact): void {
|
||||
this.enqueue('exited', (reference) => this.writer.exited(reference, fact));
|
||||
}
|
||||
|
||||
cancelled(fact: LegacyExecutionCancelledFact): void {
|
||||
this.enqueue('cancelled', (reference) =>
|
||||
this.writer.cancelled(reference, fact),
|
||||
);
|
||||
}
|
||||
|
||||
async settled(): Promise<void> {
|
||||
await this.chain;
|
||||
}
|
||||
|
||||
private enqueue(
|
||||
operation: ShadowObservationOperation,
|
||||
write: (reference: LegacyShadowRunReference) => Promise<void>,
|
||||
): void {
|
||||
this.chain = this.chain.then(async (reference) => {
|
||||
if (!reference) return null;
|
||||
try {
|
||||
await write(reference);
|
||||
} catch (error) {
|
||||
this.report(operation, error, reference);
|
||||
}
|
||||
return reference;
|
||||
});
|
||||
}
|
||||
|
||||
private report(
|
||||
operation: ShadowObservationOperation,
|
||||
error: unknown,
|
||||
reference?: LegacyShadowRunReference,
|
||||
): void {
|
||||
try {
|
||||
this.reporter.failure({
|
||||
origin: this.origin,
|
||||
operation,
|
||||
errorCode: classifyError(error),
|
||||
...(reference === undefined ? {} : reference),
|
||||
});
|
||||
} catch {
|
||||
// Shadow reporting must not become a second failure path.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacyShadowRunObserver implements LegacyExecutionObserver {
|
||||
constructor(
|
||||
private readonly policy: RuntimeRolloutPolicy,
|
||||
private readonly writer: LegacyShadowRunWriter,
|
||||
private readonly reporter: ShadowObservationReporter,
|
||||
) {}
|
||||
|
||||
begin(fact: LegacyExecutionAcceptedFact): TrackedLegacyExecutionObservation {
|
||||
const decision = this.policy.decide(fact.origin);
|
||||
if (decision.mode === 'off') return NOOP_OBSERVATION;
|
||||
if (decision.mode === 'primary') {
|
||||
throw new Error(
|
||||
'Legacy observer cannot accept a Runtime-owned primary execution',
|
||||
);
|
||||
}
|
||||
return new SerialLegacyExecutionObservation(
|
||||
fact.origin,
|
||||
this.writer,
|
||||
fact,
|
||||
this.reporter,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,546 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunAttemptStatus,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunStatus,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
isTerminalRunAttemptStatus,
|
||||
isTerminalRunStatus,
|
||||
reserveRunEvent,
|
||||
transitionRun,
|
||||
transitionRunAttempt,
|
||||
type RunAttemptTransitionCommand,
|
||||
type RunAttemptTransitionDecision,
|
||||
type RunDomainEventDraft,
|
||||
type RunTransitionCommand,
|
||||
type RunTransitionDecision,
|
||||
} from '../domain/runStateMachine';
|
||||
import { RunVersionConflictError } from '../domain/stateMachineErrors';
|
||||
import type {
|
||||
LegacyExecutionAcceptedFact,
|
||||
LegacyExecutionCancelledFact,
|
||||
LegacyExecutionExitedFact,
|
||||
LegacyExecutionSpawnedFact,
|
||||
LegacyExecutionStartFailedFact,
|
||||
} from '../ports/legacyExecutionObserver';
|
||||
import type {
|
||||
RunRepository,
|
||||
RunRepositoryTransaction,
|
||||
} from '../ports/runRepository';
|
||||
import {
|
||||
RunAttemptConcurrentWriteError,
|
||||
RunAttemptNotFoundError,
|
||||
RunNotFoundError,
|
||||
} from './commandErrors';
|
||||
|
||||
export interface LegacyShadowRunReference {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
}
|
||||
|
||||
export type ShadowIdFactory = () => string;
|
||||
|
||||
export class LegacyShadowRunWriter {
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly createId: ShadowIdFactory = uuidV7,
|
||||
) {}
|
||||
|
||||
async accept(
|
||||
fact: LegacyExecutionAcceptedFact,
|
||||
): Promise<LegacyShadowRunReference> {
|
||||
const reference = {
|
||||
runId: this.createId(),
|
||||
attemptId: this.createId(),
|
||||
};
|
||||
const initialRun: RunRecord = {
|
||||
id: reference.runId,
|
||||
projectId: fact.projectId,
|
||||
taskId: fact.taskId,
|
||||
taskRevision: fact.taskRevision,
|
||||
...(fact.taskName === undefined ? {} : { taskName: fact.taskName }),
|
||||
...(fact.legacyCronId === undefined
|
||||
? {}
|
||||
: { legacyCronId: fact.legacyCronId }),
|
||||
triggerType: fact.triggerType,
|
||||
executionOrigin: fact.origin,
|
||||
executionOwner: 'legacy',
|
||||
...(fact.triggeredBy === undefined
|
||||
? {}
|
||||
: { triggeredBy: fact.triggeredBy }),
|
||||
...(fact.requestId === undefined ? {} : { requestId: fact.requestId }),
|
||||
...(fact.scheduledForMs === undefined
|
||||
? {}
|
||||
: { scheduledForMs: fact.scheduledForMs }),
|
||||
status: 'created',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: 0,
|
||||
createdAtMs: fact.acceptedAtMs,
|
||||
};
|
||||
const initialAttempt: RunAttemptRecord = {
|
||||
id: reference.attemptId,
|
||||
runId: reference.runId,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executorType: 'legacy_local',
|
||||
callbackSequence: 0,
|
||||
createdAtMs: fact.acceptedAtMs,
|
||||
};
|
||||
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
await transaction.insertRun(initialRun);
|
||||
await transaction.insertAttempt(initialAttempt);
|
||||
|
||||
const created = reserveRunEvent(initialRun, 0);
|
||||
const createdRun = created.run;
|
||||
const createdUpdated = await transaction.compareAndSetRun(createdRun, 0);
|
||||
if (!createdUpdated) {
|
||||
throw new RunVersionConflictError(initialRun.id, 0, initialRun.version);
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(
|
||||
createdRun,
|
||||
{
|
||||
sequence: created.sequence,
|
||||
type: 'run.created',
|
||||
payload: {
|
||||
status: 'created',
|
||||
version: createdRun.version,
|
||||
execution_owner: 'legacy',
|
||||
shadow: true,
|
||||
},
|
||||
},
|
||||
fact.acceptedAtMs,
|
||||
),
|
||||
);
|
||||
|
||||
const queued = transitionRun(createdRun, {
|
||||
to: 'queued',
|
||||
expectedVersion: createdRun.version,
|
||||
atMs: fact.acceptedAtMs,
|
||||
});
|
||||
await this.persistRunDecision(
|
||||
transaction,
|
||||
createdRun,
|
||||
queued,
|
||||
fact.acceptedAtMs,
|
||||
);
|
||||
});
|
||||
return reference;
|
||||
}
|
||||
|
||||
async spawned(
|
||||
reference: LegacyShadowRunReference,
|
||||
fact: LegacyExecutionSpawnedFact,
|
||||
): Promise<void> {
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
const aggregate = await this.load(transaction, reference);
|
||||
await this.ensureSpawned(
|
||||
transaction,
|
||||
aggregate.run,
|
||||
aggregate.attempt,
|
||||
fact,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async running(
|
||||
reference: LegacyShadowRunReference,
|
||||
atMs: number,
|
||||
): Promise<void> {
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
let { run, attempt } = await this.load(transaction, reference);
|
||||
if (isTerminalRunStatus(run.status)) return;
|
||||
({ run, attempt } = await this.ensureSpawned(transaction, run, attempt, {
|
||||
atMs,
|
||||
}));
|
||||
await this.ensureRunning(transaction, run, attempt, atMs);
|
||||
});
|
||||
}
|
||||
|
||||
async startFailed(
|
||||
reference: LegacyShadowRunReference,
|
||||
fact: LegacyExecutionStartFailedFact,
|
||||
): Promise<void> {
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
let { run, attempt } = await this.load(transaction, reference);
|
||||
if (isTerminalRunStatus(run.status)) return;
|
||||
({ run, attempt } = await this.ensureSpawned(transaction, run, attempt, {
|
||||
atMs: fact.atMs,
|
||||
}));
|
||||
|
||||
if (!isTerminalRunAttemptStatus(attempt.status)) {
|
||||
if (attempt.status === 'claimed') {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: 'starting',
|
||||
expectedRunVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
},
|
||||
));
|
||||
}
|
||||
if (attempt.status === 'starting' || attempt.status === 'running') {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: 'failed',
|
||||
expectedRunVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
errorCode: fact.errorCode,
|
||||
errorSummary: 'Legacy process failed to start',
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if (!isTerminalRunStatus(run.status)) {
|
||||
await this.transitionRunStatus(transaction, run, {
|
||||
to: 'failed',
|
||||
expectedVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
errorCode: fact.errorCode,
|
||||
errorSummary: 'Legacy process failed to start',
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async exited(
|
||||
reference: LegacyShadowRunReference,
|
||||
fact: LegacyExecutionExitedFact,
|
||||
): Promise<void> {
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
let { run, attempt } = await this.load(transaction, reference);
|
||||
if (isTerminalRunStatus(run.status)) return;
|
||||
({ run, attempt } = await this.ensureSpawned(transaction, run, attempt, {
|
||||
atMs: fact.atMs,
|
||||
}));
|
||||
({ run, attempt } = await this.ensureRunning(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
fact.atMs,
|
||||
));
|
||||
|
||||
const succeeded = fact.exitCode === 0;
|
||||
const attemptTarget: RunAttemptStatus = succeeded
|
||||
? 'succeeded'
|
||||
: 'failed';
|
||||
const runTarget: RunStatus = succeeded ? 'succeeded' : 'failed';
|
||||
const errorCode =
|
||||
fact.exitCode === null
|
||||
? fact.signal
|
||||
? 'LEGACY_PROCESS_SIGNALLED'
|
||||
: 'LEGACY_EXIT_UNKNOWN'
|
||||
: succeeded
|
||||
? undefined
|
||||
: 'LEGACY_EXIT_NON_ZERO';
|
||||
const errorSummary = errorCode
|
||||
? 'Legacy process exited without success'
|
||||
: undefined;
|
||||
|
||||
if (!isTerminalRunAttemptStatus(attempt.status)) {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: attemptTarget,
|
||||
expectedRunVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
...(fact.exitCode === null ? {} : { exitCode: fact.exitCode }),
|
||||
...(errorCode === undefined ? {} : { errorCode }),
|
||||
...(errorSummary === undefined ? {} : { errorSummary }),
|
||||
},
|
||||
fact.signal === undefined ? {} : { legacy_signal: fact.signal },
|
||||
));
|
||||
}
|
||||
|
||||
if (!isTerminalRunStatus(run.status)) {
|
||||
await this.transitionRunStatus(
|
||||
transaction,
|
||||
run,
|
||||
{
|
||||
to: runTarget,
|
||||
expectedVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
...(errorCode === undefined ? {} : { errorCode }),
|
||||
...(errorSummary === undefined ? {} : { errorSummary }),
|
||||
},
|
||||
{
|
||||
legacy_exit_code: fact.exitCode,
|
||||
...(fact.signal === undefined
|
||||
? {}
|
||||
: { legacy_signal: fact.signal }),
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async cancelled(
|
||||
reference: LegacyShadowRunReference,
|
||||
fact: LegacyExecutionCancelledFact,
|
||||
): Promise<void> {
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
let { run, attempt } = await this.load(transaction, reference);
|
||||
if (isTerminalRunStatus(run.status)) return;
|
||||
|
||||
if (!isTerminalRunAttemptStatus(attempt.status)) {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: 'cancelled',
|
||||
expectedRunVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
errorCode: 'LEGACY_EXECUTION_CANCELLED',
|
||||
errorSummary: 'Legacy execution was cancelled',
|
||||
},
|
||||
{ legacy_cancel_reason: fact.reason },
|
||||
));
|
||||
}
|
||||
if (!isTerminalRunStatus(run.status)) {
|
||||
await this.transitionRunStatus(
|
||||
transaction,
|
||||
run,
|
||||
{
|
||||
to: 'cancelled',
|
||||
expectedVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
errorCode: 'LEGACY_EXECUTION_CANCELLED',
|
||||
errorSummary: 'Legacy execution was cancelled',
|
||||
},
|
||||
{ legacy_cancel_reason: fact.reason },
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async load(
|
||||
transaction: RunRepositoryTransaction,
|
||||
reference: LegacyShadowRunReference,
|
||||
): Promise<{ run: RunRecord; attempt: RunAttemptRecord }> {
|
||||
const run = await transaction.findRunById(reference.runId);
|
||||
if (!run) throw new RunNotFoundError(reference.runId);
|
||||
const attempt = await transaction.findAttemptById(reference.attemptId);
|
||||
if (!attempt) throw new RunAttemptNotFoundError(reference.attemptId);
|
||||
if (attempt.runId !== run.id) {
|
||||
throw new RunAttemptConcurrentWriteError(
|
||||
attempt.id,
|
||||
attempt.status,
|
||||
attempt.callbackSequence,
|
||||
);
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async ensureSpawned(
|
||||
transaction: RunRepositoryTransaction,
|
||||
currentRun: RunRecord,
|
||||
currentAttempt: RunAttemptRecord,
|
||||
fact: LegacyExecutionSpawnedFact,
|
||||
): Promise<{ run: RunRecord; attempt: RunAttemptRecord }> {
|
||||
let run = currentRun;
|
||||
let attempt = currentAttempt;
|
||||
if (isTerminalRunStatus(run.status)) return { run, attempt };
|
||||
|
||||
if (run.status === 'created') {
|
||||
run = await this.transitionRunStatus(transaction, run, {
|
||||
to: 'queued',
|
||||
expectedVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
});
|
||||
}
|
||||
if (run.status === 'queued') {
|
||||
run = await this.transitionRunStatus(transaction, run, {
|
||||
to: 'dispatching',
|
||||
expectedVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
});
|
||||
}
|
||||
if (attempt.status === 'claimed') {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: 'starting',
|
||||
expectedRunVersion: run.version,
|
||||
atMs: fact.atMs,
|
||||
...(fact.pid === undefined ? {} : { pid: fact.pid }),
|
||||
...(fact.executorHandle === undefined
|
||||
? {}
|
||||
: { executorHandle: fact.executorHandle }),
|
||||
...(fact.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: fact.logArtifactId }),
|
||||
},
|
||||
));
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async ensureRunning(
|
||||
transaction: RunRepositoryTransaction,
|
||||
currentRun: RunRecord,
|
||||
currentAttempt: RunAttemptRecord,
|
||||
atMs: number,
|
||||
): Promise<{ run: RunRecord; attempt: RunAttemptRecord }> {
|
||||
let run = currentRun;
|
||||
let attempt = currentAttempt;
|
||||
if (isTerminalRunStatus(run.status)) return { run, attempt };
|
||||
|
||||
if (attempt.status === 'starting') {
|
||||
({ run, attempt } = await this.transitionAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
{
|
||||
to: 'running',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
},
|
||||
));
|
||||
}
|
||||
if (run.status === 'dispatching') {
|
||||
run = await this.transitionRunStatus(transaction, run, {
|
||||
to: 'running',
|
||||
expectedVersion: run.version,
|
||||
atMs,
|
||||
});
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async transitionRunStatus(
|
||||
transaction: RunRepositoryTransaction,
|
||||
current: RunRecord,
|
||||
command: RunTransitionCommand,
|
||||
extraPayload: Readonly<Record<string, unknown>> = {},
|
||||
): Promise<RunRecord> {
|
||||
const decision = transitionRun(current, command);
|
||||
await this.persistRunDecision(
|
||||
transaction,
|
||||
current,
|
||||
decision,
|
||||
command.atMs,
|
||||
extraPayload,
|
||||
);
|
||||
return decision.run;
|
||||
}
|
||||
|
||||
private async transitionAttempt(
|
||||
transaction: RunRepositoryTransaction,
|
||||
currentRun: RunRecord,
|
||||
currentAttempt: RunAttemptRecord,
|
||||
command: RunAttemptTransitionCommand,
|
||||
extraPayload: Readonly<Record<string, unknown>> = {},
|
||||
): Promise<{ run: RunRecord; attempt: RunAttemptRecord }> {
|
||||
const decision = transitionRunAttempt(currentRun, currentAttempt, command);
|
||||
await this.persistAttemptDecision(
|
||||
transaction,
|
||||
currentRun,
|
||||
currentAttempt,
|
||||
decision,
|
||||
command.atMs,
|
||||
extraPayload,
|
||||
);
|
||||
return { run: decision.run, attempt: decision.attempt };
|
||||
}
|
||||
|
||||
private async persistRunDecision(
|
||||
transaction: RunRepositoryTransaction,
|
||||
current: RunRecord,
|
||||
decision: RunTransitionDecision,
|
||||
atMs: number,
|
||||
extraPayload: Readonly<Record<string, unknown>> = {},
|
||||
): Promise<void> {
|
||||
const updated = await transaction.compareAndSetRun(
|
||||
decision.run,
|
||||
current.version,
|
||||
);
|
||||
if (!updated) {
|
||||
const latest = await transaction.findRunById(current.id);
|
||||
throw new RunVersionConflictError(
|
||||
current.id,
|
||||
current.version,
|
||||
latest?.version ?? current.version,
|
||||
);
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(decision.run, decision.event, atMs, extraPayload),
|
||||
);
|
||||
}
|
||||
|
||||
private async persistAttemptDecision(
|
||||
transaction: RunRepositoryTransaction,
|
||||
currentRun: RunRecord,
|
||||
currentAttempt: RunAttemptRecord,
|
||||
decision: RunAttemptTransitionDecision,
|
||||
atMs: number,
|
||||
extraPayload: Readonly<Record<string, unknown>> = {},
|
||||
): Promise<void> {
|
||||
const runUpdated = await transaction.compareAndSetRun(
|
||||
decision.run,
|
||||
currentRun.version,
|
||||
);
|
||||
if (!runUpdated) {
|
||||
const latest = await transaction.findRunById(currentRun.id);
|
||||
throw new RunVersionConflictError(
|
||||
currentRun.id,
|
||||
currentRun.version,
|
||||
latest?.version ?? currentRun.version,
|
||||
);
|
||||
}
|
||||
const attemptUpdated = await transaction.compareAndSetAttempt(
|
||||
decision.attempt,
|
||||
{
|
||||
status: currentAttempt.status,
|
||||
callbackSequence: currentAttempt.callbackSequence,
|
||||
},
|
||||
);
|
||||
if (!attemptUpdated) {
|
||||
throw new RunAttemptConcurrentWriteError(
|
||||
currentAttempt.id,
|
||||
currentAttempt.status,
|
||||
currentAttempt.callbackSequence,
|
||||
);
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(decision.run, decision.event, atMs, extraPayload),
|
||||
);
|
||||
}
|
||||
|
||||
private event(
|
||||
run: RunRecord,
|
||||
draft: RunDomainEventDraft,
|
||||
createdAtMs: number,
|
||||
extraPayload: Readonly<Record<string, unknown>> = {},
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id: this.createId(),
|
||||
runId: run.id,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey: `shadow:${draft.sequence}:${draft.type}`,
|
||||
actorType: 'compatibility',
|
||||
payload: {
|
||||
...draft.payload,
|
||||
...extraPayload,
|
||||
shadow: true,
|
||||
},
|
||||
createdAtMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
import {
|
||||
assertArtifactReadProjectId,
|
||||
normalizeArtifactReadSubject,
|
||||
normalizeLocalArtifactReadMetadata,
|
||||
normalizeLocalArtifactReadRange,
|
||||
type ArtifactReadSubject,
|
||||
type LocalArtifactReadMetadata,
|
||||
type LocalArtifactReadRange,
|
||||
} from '../domain/artifactRead';
|
||||
import { assertCompletionReceiptId } from '../domain/completionReceipt';
|
||||
import type { LocalArtifactTruncationFact } from '../domain/localArtifactTruncation';
|
||||
import { assertLocalExecutionArtifactId } from '../domain/localExecutionArtifact';
|
||||
import type {
|
||||
ArtifactReadAuthorizationEffect,
|
||||
ArtifactReadAuthorizer,
|
||||
} from '../ports/artifactReadAuthorizer';
|
||||
import type { LocalArtifactByteRangeReader } from '../ports/localArtifactByteRangeReader';
|
||||
import type { LocalArtifactReadMetadataRepository } from '../ports/localArtifactReadMetadataRepository';
|
||||
import type { LocalArtifactTruncationFactStore } from '../ports/localArtifactTruncationFactStore';
|
||||
|
||||
export type LocalArtifactTruncationState = boolean | 'unknown';
|
||||
|
||||
export interface LocalArtifactTruncationView {
|
||||
truncated: LocalArtifactTruncationState;
|
||||
maximumBytes?: number;
|
||||
observedAtMs?: number;
|
||||
}
|
||||
|
||||
export interface LocalArtifactReadRequest {
|
||||
subject: ArtifactReadSubject;
|
||||
projectId: string;
|
||||
runId: string;
|
||||
logArtifactId: string;
|
||||
range: LocalArtifactReadRange;
|
||||
}
|
||||
|
||||
interface LocalArtifactReadIdentity {
|
||||
projectId: string;
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
logArtifactId: string;
|
||||
}
|
||||
|
||||
export type LocalArtifactReadResult =
|
||||
| { status: 'not_found' }
|
||||
| {
|
||||
status: 'forbidden';
|
||||
effect: Exclude<ArtifactReadAuthorizationEffect, 'allow'>;
|
||||
}
|
||||
| (LocalArtifactReadIdentity & {
|
||||
status: 'retained';
|
||||
retention: NonNullable<LocalArtifactReadMetadata['retention']>;
|
||||
truncation: { truncated: 'unknown' };
|
||||
})
|
||||
| (LocalArtifactReadIdentity & {
|
||||
status: 'missing';
|
||||
truncation: Readonly<LocalArtifactTruncationView>;
|
||||
})
|
||||
| (LocalArtifactReadIdentity & {
|
||||
status: 'available';
|
||||
content: Buffer;
|
||||
start: number;
|
||||
endExclusive: number;
|
||||
totalBytes: number;
|
||||
nextOffset?: number;
|
||||
truncation: Readonly<LocalArtifactTruncationView>;
|
||||
});
|
||||
|
||||
export class LocalArtifactReadEvidenceConflictError extends Error {
|
||||
constructor() {
|
||||
super('Local Artifact read evidence conflicts with database identity');
|
||||
this.name = 'LocalArtifactReadEvidenceConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
function identity(
|
||||
metadata: Readonly<LocalArtifactReadMetadata>,
|
||||
): LocalArtifactReadIdentity {
|
||||
return {
|
||||
projectId: metadata.projectId,
|
||||
runId: metadata.runId,
|
||||
attemptId: metadata.attemptId,
|
||||
logArtifactId: metadata.logArtifactId,
|
||||
};
|
||||
}
|
||||
|
||||
function sameIdentity(
|
||||
left: Readonly<LocalArtifactReadMetadata>,
|
||||
right: Readonly<LocalArtifactReadMetadata>,
|
||||
): boolean {
|
||||
return (
|
||||
left.projectId === right.projectId &&
|
||||
left.runId === right.runId &&
|
||||
left.attemptId === right.attemptId &&
|
||||
left.logArtifactId === right.logArtifactId
|
||||
);
|
||||
}
|
||||
|
||||
function truncationView(
|
||||
metadata: Readonly<LocalArtifactReadMetadata>,
|
||||
fact: Readonly<LocalArtifactTruncationFact> | null,
|
||||
): Readonly<LocalArtifactTruncationView> {
|
||||
if (!fact) return Object.freeze({ truncated: 'unknown' });
|
||||
if (
|
||||
fact.runId !== metadata.runId ||
|
||||
fact.attemptId !== metadata.attemptId ||
|
||||
fact.logArtifactId !== metadata.logArtifactId
|
||||
) {
|
||||
throw new LocalArtifactReadEvidenceConflictError();
|
||||
}
|
||||
return Object.freeze({
|
||||
truncated: fact.quotaReached,
|
||||
maximumBytes: fact.maximumBytes,
|
||||
observedAtMs: fact.observedAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
export class LocalArtifactReadService {
|
||||
constructor(
|
||||
private readonly metadata: LocalArtifactReadMetadataRepository,
|
||||
private readonly authorizer: ArtifactReadAuthorizer,
|
||||
private readonly bytes: LocalArtifactByteRangeReader,
|
||||
private readonly truncationFacts: LocalArtifactTruncationFactStore,
|
||||
) {}
|
||||
|
||||
async read(
|
||||
request: LocalArtifactReadRequest,
|
||||
): Promise<LocalArtifactReadResult> {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError('Local Artifact read request must be an object');
|
||||
}
|
||||
const subject = normalizeArtifactReadSubject(request.subject);
|
||||
assertArtifactReadProjectId(request.projectId);
|
||||
assertCompletionReceiptId(request.runId, 'runId');
|
||||
assertLocalExecutionArtifactId(request.logArtifactId);
|
||||
const range = normalizeLocalArtifactReadRange(request.range);
|
||||
const lookup = Object.freeze({
|
||||
projectId: request.projectId,
|
||||
runId: request.runId,
|
||||
logArtifactId: request.logArtifactId,
|
||||
});
|
||||
const initial = await this.metadata.find(lookup);
|
||||
if (!initial) return Object.freeze({ status: 'not_found' });
|
||||
const artifact = normalizeLocalArtifactReadMetadata(initial);
|
||||
const effect = await this.authorizer.authorize(
|
||||
Object.freeze({
|
||||
action: 'artifact.read',
|
||||
subject,
|
||||
projectId: artifact.projectId,
|
||||
runId: artifact.runId,
|
||||
logArtifactId: artifact.logArtifactId,
|
||||
}),
|
||||
);
|
||||
if (effect !== 'allow') {
|
||||
if (effect !== 'deny' && effect !== 'require_approval') {
|
||||
throw new TypeError('Artifact read authorization effect is invalid');
|
||||
}
|
||||
return Object.freeze({ status: 'forbidden', effect });
|
||||
}
|
||||
if (artifact.retention) {
|
||||
return Object.freeze({
|
||||
status: 'retained',
|
||||
...identity(artifact),
|
||||
retention: artifact.retention,
|
||||
truncation: Object.freeze({ truncated: 'unknown' as const }),
|
||||
});
|
||||
}
|
||||
|
||||
const content = await this.bytes.read(artifact.logArtifactId, range);
|
||||
if (content.status === 'missing') {
|
||||
const refreshedValue = await this.metadata.find(lookup);
|
||||
if (!refreshedValue) throw new LocalArtifactReadEvidenceConflictError();
|
||||
const refreshed = normalizeLocalArtifactReadMetadata(refreshedValue);
|
||||
if (!sameIdentity(artifact, refreshed)) {
|
||||
throw new LocalArtifactReadEvidenceConflictError();
|
||||
}
|
||||
if (refreshed.retention) {
|
||||
return Object.freeze({
|
||||
status: 'retained',
|
||||
...identity(refreshed),
|
||||
retention: refreshed.retention,
|
||||
truncation: Object.freeze({ truncated: 'unknown' as const }),
|
||||
});
|
||||
}
|
||||
const fact = await this.truncationFacts.read(artifact.logArtifactId);
|
||||
return Object.freeze({
|
||||
status: 'missing',
|
||||
...identity(artifact),
|
||||
truncation: truncationView(artifact, fact),
|
||||
});
|
||||
}
|
||||
|
||||
const fact = await this.truncationFacts.read(artifact.logArtifactId);
|
||||
return Object.freeze({
|
||||
status: 'available',
|
||||
...identity(artifact),
|
||||
content: content.content,
|
||||
start: content.start,
|
||||
endExclusive: content.endExclusive,
|
||||
totalBytes: content.totalBytes,
|
||||
...(content.nextOffset === undefined
|
||||
? {}
|
||||
: { nextOffset: content.nextOffset }),
|
||||
truncation: truncationView(artifact, fact),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
import type { LocalArtifactRetentionCursor } from '../domain/localArtifactRetention';
|
||||
import type { LocalArtifactRetentionCheckpointStore } from '../ports/localArtifactRetentionCheckpointStore';
|
||||
import type {
|
||||
LocalArtifactRetentionService,
|
||||
LocalArtifactRetentionSweepResult,
|
||||
} from './localArtifactRetentionService';
|
||||
|
||||
export const MIN_LOCAL_ARTIFACT_RETENTION_INTERVAL_MS = 1_000;
|
||||
export const MAX_LOCAL_ARTIFACT_RETENTION_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_LOCAL_ARTIFACT_RETENTION_INITIAL_DELAY_MS =
|
||||
24 * 60 * 60 * 1_000;
|
||||
export const MAX_LOCAL_ARTIFACT_RETENTION_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface LocalArtifactRetentionLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface LocalArtifactRetentionCycleSummary {
|
||||
pressure: boolean;
|
||||
observedAtMs: number;
|
||||
retentionMs: number;
|
||||
availableBytes: string;
|
||||
totalBytes: string;
|
||||
candidatesScanned: number;
|
||||
deletionsAttempted: number;
|
||||
recordsWritten: number;
|
||||
failedCandidates: number;
|
||||
bytesReclaimed: number;
|
||||
sweepStatus: LocalArtifactRetentionSweepResult['status'];
|
||||
cursorAction: 'unchanged' | 'advanced' | 'cleared' | 'fenced';
|
||||
}
|
||||
|
||||
export interface LocalArtifactRetentionLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
scheduler?: LocalArtifactRetentionLifecycleScheduler;
|
||||
onCycle?: (summary: Readonly<LocalArtifactRetentionCycleSummary>) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type LocalArtifactRetentionStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: LocalArtifactRetentionLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: LocalArtifactRetentionCursor | undefined,
|
||||
right: LocalArtifactRetentionCursor | undefined,
|
||||
): boolean {
|
||||
return (
|
||||
left === right ||
|
||||
(left !== undefined &&
|
||||
right !== undefined &&
|
||||
left.finishedAtMs === right.finishedAtMs &&
|
||||
left.attemptId === right.attemptId)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit one-page cadence with a durable CAS cursor. Idle complete cycles do
|
||||
* not write a checkpoint, keeping edge flash write amplification bounded.
|
||||
*/
|
||||
export class LocalArtifactRetentionLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly scheduler: LocalArtifactRetentionLifecycleScheduler;
|
||||
private readonly onCycle?: LocalArtifactRetentionLifecycleOptions['onCycle'];
|
||||
private readonly onError?: LocalArtifactRetentionLifecycleOptions['onError'];
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
|
||||
constructor(
|
||||
private readonly service: Pick<LocalArtifactRetentionService, 'sweep'>,
|
||||
private readonly checkpoints: LocalArtifactRetentionCheckpointStore,
|
||||
options: LocalArtifactRetentionLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_LOCAL_ARTIFACT_RETENTION_INTERVAL_MS,
|
||||
MAX_LOCAL_ARTIFACT_RETENTION_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_LOCAL_ARTIFACT_RETENTION_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_LOCAL_ARTIFACT_RETENTION_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<LocalArtifactRetentionStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<LocalArtifactRetentionStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.runCycle()
|
||||
.then((summary) => this.notifyCycle(summary))
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private async runCycle(): Promise<LocalArtifactRetentionCycleSummary> {
|
||||
const checkpoint = await this.checkpoints.load();
|
||||
const sweep = await this.service.sweep(checkpoint.cursor);
|
||||
const nextCursor =
|
||||
sweep.status === 'complete' ? undefined : sweep.nextCursor;
|
||||
if (sweep.status !== 'complete' && !nextCursor) {
|
||||
throw new TypeError(
|
||||
'Incomplete Local Artifact retention sweep requires a resume cursor',
|
||||
);
|
||||
}
|
||||
let cursorAction: LocalArtifactRetentionCycleSummary['cursorAction'] =
|
||||
'unchanged';
|
||||
if (!sameCursor(checkpoint.cursor, nextCursor)) {
|
||||
const updated = await this.checkpoints.compareAndSet({
|
||||
expectedVersion: checkpoint.version,
|
||||
...(nextCursor ? { cursor: nextCursor } : {}),
|
||||
updatedAtMs: sweep.observedAtMs,
|
||||
});
|
||||
cursorAction = updated ? (nextCursor ? 'advanced' : 'cleared') : 'fenced';
|
||||
}
|
||||
return Object.freeze({
|
||||
pressure: sweep.pressure,
|
||||
observedAtMs: sweep.observedAtMs,
|
||||
retentionMs: sweep.retentionMs,
|
||||
availableBytes: sweep.availableBytes.toString(10),
|
||||
totalBytes: sweep.totalBytes.toString(10),
|
||||
candidatesScanned: sweep.candidatesScanned,
|
||||
deletionsAttempted: sweep.deletionsAttempted,
|
||||
recordsWritten: sweep.recordsWritten,
|
||||
failedCandidates: sweep.failedCandidates,
|
||||
bytesReclaimed: sweep.bytesReclaimed,
|
||||
sweepStatus: sweep.status,
|
||||
cursorAction,
|
||||
});
|
||||
}
|
||||
|
||||
private notifyCycle(summary: LocalArtifactRetentionCycleSummary): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must never create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
import type {
|
||||
LocalArtifactRetentionCandidate,
|
||||
LocalArtifactRetentionCursor,
|
||||
} from '../domain/localArtifactRetention';
|
||||
import {
|
||||
normalizeLocalArtifactRetentionCandidate,
|
||||
normalizeLocalArtifactRetentionCursor,
|
||||
} from '../domain/localArtifactRetention';
|
||||
import type { LocalArtifactCapacitySource } from '../ports/localArtifactCapacityProbe';
|
||||
import type { LocalArtifactFileRetirementStore } from '../ports/localArtifactFileRetirementStore';
|
||||
import type {
|
||||
LocalArtifactRetentionPage,
|
||||
LocalArtifactRetentionRepository,
|
||||
} from '../ports/localArtifactRetentionRepository';
|
||||
import { MAX_LOCAL_ARTIFACT_RETENTION_PAGE_SIZE } from '../ports/localArtifactRetentionRepository';
|
||||
|
||||
export const MIN_LOCAL_ARTIFACT_RETENTION_MS = 60_000;
|
||||
export const MAX_LOCAL_ARTIFACT_RETENTION_MS = 365 * 24 * 60 * 60_000;
|
||||
|
||||
export interface LocalArtifactRetentionServiceOptions {
|
||||
normalRetentionMs: number;
|
||||
pressureRetentionMs: number;
|
||||
minimumFreeBytes: number;
|
||||
pageSize?: number;
|
||||
maximumDeletions?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
export interface LocalArtifactRetentionEntry {
|
||||
attemptId: string;
|
||||
logArtifactId: string;
|
||||
outcome: 'deleted' | 'already_absent' | 'file_failed' | 'record_failed';
|
||||
bytesReclaimed: number;
|
||||
}
|
||||
|
||||
export interface LocalArtifactRetentionSweepResult {
|
||||
status: 'complete' | 'page_complete' | 'deletion_budget_exhausted';
|
||||
pressure: boolean;
|
||||
observedAtMs: number;
|
||||
retentionMs: number;
|
||||
availableBytes: bigint;
|
||||
totalBytes: bigint;
|
||||
candidatesScanned: number;
|
||||
deletionsAttempted: number;
|
||||
recordsWritten: number;
|
||||
failedCandidates: number;
|
||||
bytesReclaimed: number;
|
||||
entries: readonly LocalArtifactRetentionEntry[];
|
||||
nextCursor?: LocalArtifactRetentionCursor;
|
||||
}
|
||||
|
||||
export class InvalidLocalArtifactRetentionPageError extends Error {
|
||||
constructor(message: string) {
|
||||
super(`Local Artifact retention page is invalid: ${message}`);
|
||||
this.name = 'InvalidLocalArtifactRetentionPageError';
|
||||
}
|
||||
}
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalArtifactRetentionService {
|
||||
private readonly normalRetentionMs: number;
|
||||
private readonly pressureRetentionMs: number;
|
||||
private readonly minimumFreeBytes: number;
|
||||
private readonly pageSize: number;
|
||||
private readonly maximumDeletions: number;
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly repository: LocalArtifactRetentionRepository,
|
||||
private readonly files: LocalArtifactFileRetirementStore,
|
||||
private readonly capacity: LocalArtifactCapacitySource,
|
||||
options: LocalArtifactRetentionServiceOptions,
|
||||
) {
|
||||
this.normalRetentionMs = options.normalRetentionMs;
|
||||
this.pressureRetentionMs = options.pressureRetentionMs;
|
||||
this.minimumFreeBytes = options.minimumFreeBytes;
|
||||
this.pageSize = options.pageSize ?? 16;
|
||||
this.maximumDeletions = options.maximumDeletions ?? 8;
|
||||
this.clock = options.clock ?? Date;
|
||||
assertIntegerBetween(
|
||||
'normalRetentionMs',
|
||||
this.normalRetentionMs,
|
||||
MIN_LOCAL_ARTIFACT_RETENTION_MS,
|
||||
MAX_LOCAL_ARTIFACT_RETENTION_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'pressureRetentionMs',
|
||||
this.pressureRetentionMs,
|
||||
MIN_LOCAL_ARTIFACT_RETENTION_MS,
|
||||
this.normalRetentionMs,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'minimumFreeBytes',
|
||||
this.minimumFreeBytes,
|
||||
0,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'pageSize',
|
||||
this.pageSize,
|
||||
1,
|
||||
MAX_LOCAL_ARTIFACT_RETENTION_PAGE_SIZE,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'maximumDeletions',
|
||||
this.maximumDeletions,
|
||||
1,
|
||||
this.pageSize,
|
||||
);
|
||||
}
|
||||
|
||||
async sweep(
|
||||
cursor?: LocalArtifactRetentionCursor,
|
||||
): Promise<LocalArtifactRetentionSweepResult> {
|
||||
const normalizedCursor = cursor
|
||||
? normalizeLocalArtifactRetentionCursor(cursor)
|
||||
: undefined;
|
||||
const observedAtMs = this.now();
|
||||
const capacity = await this.capacity.inspect();
|
||||
if (
|
||||
typeof capacity?.availableBytes !== 'bigint' ||
|
||||
typeof capacity.totalBytes !== 'bigint' ||
|
||||
capacity.availableBytes < BigInt(0) ||
|
||||
capacity.totalBytes < BigInt(1) ||
|
||||
capacity.availableBytes > capacity.totalBytes
|
||||
) {
|
||||
throw new TypeError('Local Artifact capacity snapshot is invalid');
|
||||
}
|
||||
const pressure = capacity.availableBytes < BigInt(this.minimumFreeBytes);
|
||||
const retentionMs = pressure
|
||||
? this.pressureRetentionMs
|
||||
: this.normalRetentionMs;
|
||||
const cutoffMs = Math.max(0, observedAtMs - retentionMs);
|
||||
const page = await this.repository.list({
|
||||
cutoffMs,
|
||||
...(normalizedCursor ? { cursor: normalizedCursor } : {}),
|
||||
limit: this.pageSize,
|
||||
});
|
||||
this.assertPage(page, normalizedCursor);
|
||||
|
||||
const entries: LocalArtifactRetentionEntry[] = [];
|
||||
let candidatesScanned = 0;
|
||||
let deletionsAttempted = 0;
|
||||
let recordsWritten = 0;
|
||||
let failedCandidates = 0;
|
||||
let bytesReclaimed = 0;
|
||||
let lastProcessed = normalizedCursor;
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
if (deletionsAttempted >= this.maximumDeletions) {
|
||||
return this.result({
|
||||
status: 'deletion_budget_exhausted',
|
||||
pressure,
|
||||
observedAtMs,
|
||||
retentionMs,
|
||||
availableBytes: capacity.availableBytes,
|
||||
totalBytes: capacity.totalBytes,
|
||||
candidatesScanned,
|
||||
deletionsAttempted,
|
||||
recordsWritten,
|
||||
failedCandidates,
|
||||
bytesReclaimed,
|
||||
entries,
|
||||
nextCursor: lastProcessed,
|
||||
});
|
||||
}
|
||||
candidatesScanned += 1;
|
||||
deletionsAttempted += 1;
|
||||
lastProcessed = this.cursor(candidate);
|
||||
let retired;
|
||||
try {
|
||||
retired = await this.files.retire(candidate.logArtifactId);
|
||||
} catch {
|
||||
failedCandidates += 1;
|
||||
entries.push({
|
||||
attemptId: candidate.attemptId,
|
||||
logArtifactId: candidate.logArtifactId,
|
||||
outcome: 'file_failed',
|
||||
bytesReclaimed: 0,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await this.repository.record({
|
||||
...candidate,
|
||||
eligibleAtMs: candidate.finishedAtMs + retentionMs,
|
||||
disposition: retired.disposition,
|
||||
bytesReclaimed: retired.bytesReclaimed,
|
||||
recordedAtMs: observedAtMs,
|
||||
});
|
||||
recordsWritten += 1;
|
||||
bytesReclaimed += retired.bytesReclaimed;
|
||||
entries.push({
|
||||
attemptId: candidate.attemptId,
|
||||
logArtifactId: candidate.logArtifactId,
|
||||
outcome: retired.disposition,
|
||||
bytesReclaimed: retired.bytesReclaimed,
|
||||
});
|
||||
} catch {
|
||||
failedCandidates += 1;
|
||||
entries.push({
|
||||
attemptId: candidate.attemptId,
|
||||
logArtifactId: candidate.logArtifactId,
|
||||
outcome: 'record_failed',
|
||||
bytesReclaimed: retired.bytesReclaimed,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return this.result({
|
||||
status: page.truncated ? 'page_complete' : 'complete',
|
||||
pressure,
|
||||
observedAtMs,
|
||||
retentionMs,
|
||||
availableBytes: capacity.availableBytes,
|
||||
totalBytes: capacity.totalBytes,
|
||||
candidatesScanned,
|
||||
deletionsAttempted,
|
||||
recordsWritten,
|
||||
failedCandidates,
|
||||
bytesReclaimed,
|
||||
entries,
|
||||
nextCursor: page.nextCursor,
|
||||
});
|
||||
}
|
||||
|
||||
private assertPage(
|
||||
page: LocalArtifactRetentionPage,
|
||||
cursor: Readonly<LocalArtifactRetentionCursor> | undefined,
|
||||
): void {
|
||||
if (
|
||||
!page ||
|
||||
!Array.isArray(page.candidates) ||
|
||||
page.candidates.length > this.pageSize ||
|
||||
typeof page.truncated !== 'boolean'
|
||||
) {
|
||||
throw new InvalidLocalArtifactRetentionPageError(
|
||||
'candidate count exceeds pageSize',
|
||||
);
|
||||
}
|
||||
let previous = cursor;
|
||||
for (const candidate of page.candidates) {
|
||||
normalizeLocalArtifactRetentionCandidate(candidate);
|
||||
if (
|
||||
previous &&
|
||||
(candidate.finishedAtMs < previous.finishedAtMs ||
|
||||
(candidate.finishedAtMs === previous.finishedAtMs &&
|
||||
candidate.attemptId <= previous.attemptId))
|
||||
) {
|
||||
throw new InvalidLocalArtifactRetentionPageError(
|
||||
'candidate cursor did not advance',
|
||||
);
|
||||
}
|
||||
previous = candidate;
|
||||
}
|
||||
const last = page.candidates[page.candidates.length - 1];
|
||||
if (
|
||||
page.truncated !== (page.nextCursor !== undefined) ||
|
||||
(page.nextCursor &&
|
||||
(!last ||
|
||||
page.nextCursor.finishedAtMs !== last.finishedAtMs ||
|
||||
page.nextCursor.attemptId !== last.attemptId))
|
||||
) {
|
||||
throw new InvalidLocalArtifactRetentionPageError(
|
||||
'resume cursor is inconsistent',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private cursor(
|
||||
candidate: LocalArtifactRetentionCandidate,
|
||||
): LocalArtifactRetentionCursor {
|
||||
return Object.freeze({
|
||||
finishedAtMs: candidate.finishedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
});
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const value = this.clock.now();
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError('Local Artifact retention clock is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private result(
|
||||
value: Omit<LocalArtifactRetentionSweepResult, 'entries'> & {
|
||||
entries: LocalArtifactRetentionEntry[];
|
||||
},
|
||||
): LocalArtifactRetentionSweepResult {
|
||||
const { nextCursor, ...rest } = value;
|
||||
return Object.freeze({
|
||||
...rest,
|
||||
entries: Object.freeze([...value.entries]),
|
||||
...(nextCursor ? { nextCursor: Object.freeze({ ...nextCursor }) } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
import { EXECUTOR_TYPES, type ExecutorType } from '../domain/execution';
|
||||
import {
|
||||
assertRunDispatchCandidate,
|
||||
assertRunDispatchCandidatePageSize,
|
||||
type RunDispatchCandidate,
|
||||
type RunDispatchCandidateCursor,
|
||||
} from '../domain/runDispatchCandidate';
|
||||
import { assertRunDispatchLeaseVersion } from '../domain/runDispatchLease';
|
||||
import { executionSpecForRunDispatchCandidate } from '../domain/runDispatchPlan';
|
||||
import type { RunDispatchCandidateSource } from '../ports/runDispatchCandidateSource';
|
||||
import type { LocalRunDispatchPlanSource } from '../ports/localRunDispatchPlanSource';
|
||||
import type {
|
||||
ActivePrimaryRun,
|
||||
PrimaryClaimedRunStartCommand,
|
||||
} from './primaryRunOrchestrator';
|
||||
import {
|
||||
PrimaryClaimedRunRejectedError,
|
||||
PrimaryRunLaunchError,
|
||||
} from './primaryRunOrchestrator';
|
||||
|
||||
const DEFAULT_LOCAL_DISPATCH_PAGE_SIZE = 8;
|
||||
const DEFAULT_LOCAL_DISPATCH_MAX_PAGES = 1;
|
||||
const MAX_LOCAL_DISPATCH_PAGES = 16;
|
||||
|
||||
export interface LocalClaimedRunActivator {
|
||||
activateClaimed(
|
||||
command: PrimaryClaimedRunStartCommand,
|
||||
): Promise<ActivePrimaryRun>;
|
||||
}
|
||||
|
||||
export interface LocalRunDispatcherOptions {
|
||||
executorType: ExecutorType;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
clock?: { now(): number };
|
||||
onDisposeError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export interface LocalRunDispatcherStats {
|
||||
pages: number;
|
||||
candidatesScanned: number;
|
||||
executorMismatches: number;
|
||||
plansUnavailable: number;
|
||||
activationRaces: number;
|
||||
}
|
||||
|
||||
export type LocalRunDispatcherIdleReason =
|
||||
| 'no_candidates'
|
||||
| 'no_matching_executor'
|
||||
| 'plans_unavailable'
|
||||
| 'activation_raced'
|
||||
| 'scan_budget_exhausted';
|
||||
|
||||
export type LocalRunDispatcherResult =
|
||||
| {
|
||||
status: 'activated';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
completion: ActivePrimaryRun['completion'];
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
}
|
||||
| {
|
||||
status: 'activation_failed';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
}
|
||||
| {
|
||||
status: 'idle';
|
||||
reason: LocalRunDispatcherIdleReason;
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
};
|
||||
|
||||
function cursorOf(candidate: RunDispatchCandidate): RunDispatchCandidateCursor {
|
||||
return {
|
||||
priority: candidate.priority,
|
||||
queuedAtMs: candidate.queuedAtMs,
|
||||
attemptCreatedAtMs: candidate.attemptCreatedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
};
|
||||
}
|
||||
|
||||
function cursorAdvances(
|
||||
previous: RunDispatchCandidateCursor,
|
||||
next: RunDispatchCandidateCursor,
|
||||
): boolean {
|
||||
return (
|
||||
next.priority < previous.priority ||
|
||||
(next.priority === previous.priority &&
|
||||
(next.queuedAtMs > previous.queuedAtMs ||
|
||||
(next.queuedAtMs === previous.queuedAtMs &&
|
||||
(next.attemptCreatedAtMs > previous.attemptCreatedAtMs ||
|
||||
(next.attemptCreatedAtMs === previous.attemptCreatedAtMs &&
|
||||
next.attemptId > previous.attemptId)))))
|
||||
);
|
||||
}
|
||||
|
||||
function emptyStats(): LocalRunDispatcherStats {
|
||||
return {
|
||||
pages: 0,
|
||||
candidatesScanned: 0,
|
||||
executorMismatches: 0,
|
||||
plansUnavailable: 0,
|
||||
activationRaces: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/** One bounded local dispatch cycle. It owns neither a timer nor task storage. */
|
||||
export class LocalRunDispatcher {
|
||||
private readonly executorType: ExecutorType;
|
||||
private readonly pageSize: number;
|
||||
private readonly maxPages: number;
|
||||
private readonly clock: { now(): number };
|
||||
private readonly onDisposeError?: (error: unknown) => void;
|
||||
|
||||
constructor(
|
||||
private readonly candidates: RunDispatchCandidateSource,
|
||||
private readonly plans: LocalRunDispatchPlanSource,
|
||||
private readonly activator: LocalClaimedRunActivator,
|
||||
options: LocalRunDispatcherOptions,
|
||||
) {
|
||||
this.executorType = options.executorType;
|
||||
this.pageSize = options.pageSize ?? DEFAULT_LOCAL_DISPATCH_PAGE_SIZE;
|
||||
this.maxPages = options.maxPages ?? DEFAULT_LOCAL_DISPATCH_MAX_PAGES;
|
||||
this.clock = options.clock ?? Date;
|
||||
this.onDisposeError = options.onDisposeError;
|
||||
if (!EXECUTOR_TYPES.includes(this.executorType)) {
|
||||
throw new TypeError('Local Run Dispatcher executorType is invalid');
|
||||
}
|
||||
assertRunDispatchCandidatePageSize(this.pageSize);
|
||||
if (
|
||||
!Number.isSafeInteger(this.maxPages) ||
|
||||
this.maxPages < 1 ||
|
||||
this.maxPages > MAX_LOCAL_DISPATCH_PAGES
|
||||
) {
|
||||
throw new RangeError(
|
||||
`maxPages must be between 1 and ${MAX_LOCAL_DISPATCH_PAGES}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchOnce(): Promise<LocalRunDispatcherResult> {
|
||||
const observedAtMs = this.clock.now();
|
||||
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
|
||||
const stats = emptyStats();
|
||||
const seen = new Set<string>();
|
||||
let after: RunDispatchCandidateCursor | undefined;
|
||||
|
||||
for (let pageIndex = 0; pageIndex < this.maxPages; pageIndex += 1) {
|
||||
const page = await this.candidates.listCandidates({
|
||||
observedAtMs,
|
||||
...(after === undefined ? {} : { after }),
|
||||
limit: this.pageSize,
|
||||
});
|
||||
if (page.length > this.pageSize) {
|
||||
throw new RangeError('Local Run candidate source exceeded page size');
|
||||
}
|
||||
stats.pages += 1;
|
||||
let previous = after;
|
||||
for (const candidate of page) {
|
||||
assertRunDispatchCandidate(candidate);
|
||||
const cursor = cursorOf(candidate);
|
||||
if (
|
||||
seen.has(candidate.attemptId) ||
|
||||
(previous !== undefined && !cursorAdvances(previous, cursor))
|
||||
) {
|
||||
throw new Error('Local Run candidate page is not strictly ordered');
|
||||
}
|
||||
seen.add(candidate.attemptId);
|
||||
previous = cursor;
|
||||
stats.candidatesScanned += 1;
|
||||
if (candidate.executorType !== this.executorType) {
|
||||
stats.executorMismatches += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const plan = await this.plans.prepare(Object.freeze({ ...candidate }));
|
||||
if (!plan) {
|
||||
stats.plansUnavailable += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const spec = executionSpecForRunDispatchCandidate(
|
||||
candidate,
|
||||
plan.executionSpec,
|
||||
);
|
||||
const active = await this.activator.activateClaimed({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
...(spec.timeoutMs === undefined
|
||||
? {}
|
||||
: { timeoutMs: spec.timeoutMs }),
|
||||
createSpec: () => spec,
|
||||
context: plan.context,
|
||||
...(plan.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: plan.logArtifactId }),
|
||||
});
|
||||
this.disposeAfterCompletion(active, plan.dispose);
|
||||
return {
|
||||
status: 'activated',
|
||||
runId: active.run.id,
|
||||
attemptId: active.attempt.id,
|
||||
completion: active.completion,
|
||||
stats,
|
||||
truncated: page.length === this.pageSize,
|
||||
};
|
||||
} catch (error) {
|
||||
await this.dispose(plan.dispose);
|
||||
if (error instanceof PrimaryClaimedRunRejectedError) {
|
||||
if (
|
||||
error.reason === 'aggregate_mismatch' ||
|
||||
error.reason === 'executor_mismatch'
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
stats.activationRaces += 1;
|
||||
continue;
|
||||
}
|
||||
if (error instanceof PrimaryRunLaunchError) {
|
||||
return {
|
||||
status: 'activation_failed',
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
stats,
|
||||
truncated: page.length === this.pageSize,
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (page.length < this.pageSize) {
|
||||
return this.idle(stats, false);
|
||||
}
|
||||
after = cursorOf(page[page.length - 1]);
|
||||
}
|
||||
return {
|
||||
status: 'idle',
|
||||
reason: 'scan_budget_exhausted',
|
||||
stats,
|
||||
truncated: true,
|
||||
};
|
||||
}
|
||||
|
||||
private idle(
|
||||
stats: LocalRunDispatcherStats,
|
||||
truncated: boolean,
|
||||
): LocalRunDispatcherResult {
|
||||
const eligible = stats.candidatesScanned - stats.executorMismatches;
|
||||
const reason: LocalRunDispatcherIdleReason =
|
||||
stats.candidatesScanned === 0
|
||||
? 'no_candidates'
|
||||
: eligible === 0
|
||||
? 'no_matching_executor'
|
||||
: stats.plansUnavailable === eligible
|
||||
? 'plans_unavailable'
|
||||
: 'activation_raced';
|
||||
return { status: 'idle', reason, stats, truncated };
|
||||
}
|
||||
|
||||
private disposeAfterCompletion(
|
||||
active: ActivePrimaryRun,
|
||||
dispose: (() => void | Promise<void>) | undefined,
|
||||
): void {
|
||||
if (!dispose) return;
|
||||
void active.completion.then(
|
||||
() => this.dispose(dispose),
|
||||
() => this.dispose(dispose),
|
||||
);
|
||||
}
|
||||
|
||||
private async dispose(
|
||||
dispose: (() => void | Promise<void>) | undefined,
|
||||
): Promise<void> {
|
||||
if (!dispose) return;
|
||||
try {
|
||||
await dispose();
|
||||
} catch (error) {
|
||||
try {
|
||||
this.onDisposeError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must not change activation ownership.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import type { PinnedTaskExecutionRevision } from '../domain/taskExecutionRevision';
|
||||
import type { LocalExecutionContextRecipe } from '../domain/localExecutionContextRecipe';
|
||||
import type { LocalExecutionContextRecipeRepository } from '../ports/localExecutionContextRecipeRepository';
|
||||
import type { TaskExecutionRevisionRepository } from '../ports/taskExecutionRevisionRepository';
|
||||
|
||||
export interface PublishLocalTaskExecutionRevisionCommand {
|
||||
revision: PinnedTaskExecutionRevision;
|
||||
contextRecipe: LocalExecutionContextRecipe;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
export interface PublishLocalTaskExecutionRevisionResult {
|
||||
contextRecipe: 'inserted' | 'idempotent';
|
||||
revision: 'inserted' | 'idempotent';
|
||||
}
|
||||
|
||||
/** Publishes the dependency first so a revision never points at a missing recipe. */
|
||||
export class LocalTaskExecutionRevisionPublisher {
|
||||
constructor(
|
||||
private readonly recipes: LocalExecutionContextRecipeRepository,
|
||||
private readonly revisions: TaskExecutionRevisionRepository,
|
||||
) {}
|
||||
|
||||
async publish(
|
||||
command: PublishLocalTaskExecutionRevisionCommand,
|
||||
): Promise<PublishLocalTaskExecutionRevisionResult> {
|
||||
if (command.revision.contextRef !== command.contextRecipe.contextRef) {
|
||||
throw new TypeError('Task revision contextRef does not match its recipe');
|
||||
}
|
||||
const contextRecipe = await this.recipes.insert(
|
||||
command.contextRecipe,
|
||||
command.createdAtMs,
|
||||
);
|
||||
const revision = await this.revisions.insert(
|
||||
command.revision,
|
||||
command.createdAtMs,
|
||||
);
|
||||
return { contextRecipe, revision };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
import type {
|
||||
ExecutionOutcome,
|
||||
ExecutionOutputSink,
|
||||
} from '../domain/execution';
|
||||
import type { Executor } from '../ports/executor';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import { buildLegacyCronExecutionSpec } from '../adapters/legacy/legacyCronExecutionSpec';
|
||||
import { createLegacyTaskRevision } from '../compatibility/legacyTaskRevision';
|
||||
import { createLegacyLogOutputRef } from '../compatibility/legacyLogOutputRef';
|
||||
import type {
|
||||
ManualPrimaryCompletion,
|
||||
ManualPrimaryExecutionRouter,
|
||||
ManualPrimaryStartInput,
|
||||
ManualPrimaryStartedExecution,
|
||||
ManualPrimaryStopResult,
|
||||
} from '../compatibility/manualPrimaryExecutionBridge';
|
||||
import type { RuntimeRolloutPolicy } from '../domain/runtimeRollout';
|
||||
import {
|
||||
PrimaryRunOrchestrator,
|
||||
type ActivePrimaryRun,
|
||||
type PrimaryRunClock,
|
||||
type PrimaryRunOrchestratorOptions,
|
||||
} from './primaryRunOrchestrator';
|
||||
|
||||
export interface PreparedManualPrimaryLog {
|
||||
logPath: string;
|
||||
output: ExecutionOutputSink;
|
||||
completionCommitted?(attemptId: string): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface ManualPrimaryLogFiles {
|
||||
prepare(input: ManualPrimaryStartInput): Promise<PreparedManualPrimaryLog>;
|
||||
}
|
||||
|
||||
export interface ManualPrimaryRuntimeOptions {
|
||||
clock?: PrimaryRunClock;
|
||||
orchestrator?: PrimaryRunOrchestratorOptions;
|
||||
}
|
||||
|
||||
interface ActiveManualExecution {
|
||||
cronId: number;
|
||||
attemptId: string;
|
||||
execution: ActivePrimaryRun;
|
||||
}
|
||||
|
||||
interface PendingManualExecution {
|
||||
cronId: number;
|
||||
controller: AbortController;
|
||||
}
|
||||
|
||||
export class ManualPrimaryOwnershipError extends Error {
|
||||
constructor() {
|
||||
super('Manual execution is not Runtime-owned');
|
||||
this.name = 'ManualPrimaryOwnershipError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process-local manual Primary owner. The active maps are bounded by actual
|
||||
* concurrent executions; durable restart/cross-worker ownership remains the
|
||||
* startup Reconciler and future supervisor's responsibility.
|
||||
*/
|
||||
export class ManualPrimaryRuntime implements ManualPrimaryExecutionRouter {
|
||||
private readonly orchestrator: PrimaryRunOrchestrator;
|
||||
private readonly clock: PrimaryRunClock;
|
||||
private readonly pending = new Map<symbol, PendingManualExecution>();
|
||||
private readonly active = new Map<string, ActiveManualExecution>();
|
||||
|
||||
constructor(
|
||||
repository: RunRepository,
|
||||
executor: Executor,
|
||||
private readonly rollout: RuntimeRolloutPolicy,
|
||||
private readonly logs: ManualPrimaryLogFiles,
|
||||
options: ManualPrimaryRuntimeOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.orchestrator = new PrimaryRunOrchestrator(repository, executor, {
|
||||
...options.orchestrator,
|
||||
clock: this.clock,
|
||||
});
|
||||
}
|
||||
|
||||
ownsNewRuns(): boolean {
|
||||
return this.rollout.decide('manual').owner === 'runtime';
|
||||
}
|
||||
|
||||
async start(
|
||||
input: ManualPrimaryStartInput,
|
||||
): Promise<ManualPrimaryStartedExecution> {
|
||||
if (!this.ownsNewRuns()) throw new ManualPrimaryOwnershipError();
|
||||
const pendingId = Symbol('manual-primary-pending');
|
||||
const controller = new AbortController();
|
||||
this.pending.set(pendingId, { cronId: input.cron.id, controller });
|
||||
|
||||
let prepared: PreparedManualPrimaryLog | undefined;
|
||||
try {
|
||||
prepared = await this.logs.prepare(input);
|
||||
const taskRevision = createLegacyTaskRevision({
|
||||
command: input.cron.command,
|
||||
...(input.cron.schedule === undefined
|
||||
? {}
|
||||
: { schedule: input.cron.schedule }),
|
||||
extraSchedules: input.cron.extraSchedules,
|
||||
...(input.cron.taskBefore === undefined
|
||||
? {}
|
||||
: { taskBefore: input.cron.taskBefore }),
|
||||
...(input.cron.taskAfter === undefined
|
||||
? {}
|
||||
: { taskAfter: input.cron.taskAfter }),
|
||||
...(input.cron.workDirectory === undefined
|
||||
? {}
|
||||
: { workDirectory: input.cron.workDirectory }),
|
||||
...(input.cron.logName === undefined
|
||||
? {}
|
||||
: { logName: input.cron.logName }),
|
||||
});
|
||||
const outputRef = createLegacyLogOutputRef(prepared.logPath);
|
||||
const active = await this.orchestrator.start({
|
||||
definition: {
|
||||
projectId: 'default',
|
||||
taskId: 'legacy-cron:' + input.cron.id,
|
||||
taskRevision,
|
||||
...(input.cron.name === undefined
|
||||
? {}
|
||||
: { taskName: input.cron.name }),
|
||||
legacyCronId: input.cron.id,
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
triggeredBy: 'legacy:manual-api',
|
||||
outputRef,
|
||||
acceptedAtMs: input.acceptedAtMs,
|
||||
actor: { type: 'compatibility', id: 'legacy:manual-api' },
|
||||
},
|
||||
createSpec: (reference) =>
|
||||
buildLegacyCronExecutionSpec({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
projectId: reference.run.projectId,
|
||||
taskRevision: reference.run.taskRevision,
|
||||
cron: {
|
||||
id: input.cron.id,
|
||||
command: input.cron.command,
|
||||
...(input.cron.taskBefore === undefined
|
||||
? {}
|
||||
: { taskBefore: input.cron.taskBefore }),
|
||||
...(input.cron.taskAfter === undefined
|
||||
? {}
|
||||
: { taskAfter: input.cron.taskAfter }),
|
||||
...(input.cron.workDirectory === undefined
|
||||
? {}
|
||||
: { workDirectory: input.cron.workDirectory }),
|
||||
...(input.cron.logName === undefined
|
||||
? {}
|
||||
: { logName: input.cron.logName }),
|
||||
},
|
||||
realTime: true,
|
||||
realLogPath: prepared!.logPath,
|
||||
noDelay: true,
|
||||
}),
|
||||
context: {
|
||||
environment: {},
|
||||
signal: controller.signal,
|
||||
output: prepared.output,
|
||||
},
|
||||
});
|
||||
|
||||
const entry: ActiveManualExecution = {
|
||||
cronId: input.cron.id,
|
||||
attemptId: active.attempt.id,
|
||||
execution: active,
|
||||
};
|
||||
this.active.set(active.run.id, entry);
|
||||
this.pending.delete(pendingId);
|
||||
const completion = this.completion(active, prepared).finally(() => {
|
||||
if (this.active.get(active.run.id) === entry) {
|
||||
this.active.delete(active.run.id);
|
||||
}
|
||||
});
|
||||
void completion.catch(() => undefined);
|
||||
return {
|
||||
runId: active.run.id,
|
||||
attemptId: active.attempt.id,
|
||||
...(active.handle.pid === undefined ? {} : { pid: active.handle.pid }),
|
||||
logPath: prepared.logPath,
|
||||
completion,
|
||||
};
|
||||
} catch (error) {
|
||||
if (prepared) await prepared.close().catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
this.pending.delete(pendingId);
|
||||
}
|
||||
}
|
||||
|
||||
async stopCron(
|
||||
cronId: number,
|
||||
requestedAtMs: number,
|
||||
): Promise<ManualPrimaryStopResult> {
|
||||
let matched = 0;
|
||||
for (const pending of this.pending.values()) {
|
||||
if (pending.cronId !== cronId) continue;
|
||||
matched += 1;
|
||||
pending.controller.abort();
|
||||
}
|
||||
const executions = [...this.active.values()].filter(
|
||||
(entry) => entry.cronId === cronId,
|
||||
);
|
||||
matched += executions.length;
|
||||
const failed = await this.stopActive(executions, requestedAtMs);
|
||||
return { matched, failed };
|
||||
}
|
||||
|
||||
async stopAttempt(
|
||||
attemptId: string,
|
||||
requestedAtMs: number,
|
||||
): Promise<ManualPrimaryStopResult> {
|
||||
const execution = [...this.active.values()].find(
|
||||
(entry) => entry.attemptId === attemptId,
|
||||
);
|
||||
if (!execution) return { matched: 0, failed: 0 };
|
||||
const failed = await this.stopActive([execution], requestedAtMs);
|
||||
return { matched: 1, failed };
|
||||
}
|
||||
|
||||
private async stopActive(
|
||||
executions: readonly ActiveManualExecution[],
|
||||
requestedAtMs: number,
|
||||
): Promise<number> {
|
||||
const results = await Promise.allSettled(
|
||||
executions.map((entry) =>
|
||||
entry.execution.cancel({ kind: 'user', requestedAtMs }),
|
||||
),
|
||||
);
|
||||
return results.filter((result) => result.status === 'rejected').length;
|
||||
}
|
||||
|
||||
private async completion(
|
||||
active: ActivePrimaryRun,
|
||||
prepared: PreparedManualPrimaryLog,
|
||||
): Promise<ManualPrimaryCompletion> {
|
||||
try {
|
||||
const completed = await active.completion;
|
||||
await prepared
|
||||
.completionCommitted?.(completed.attempt.id)
|
||||
.catch(() => undefined);
|
||||
const result = completed.result;
|
||||
return {
|
||||
runId: completed.run.id,
|
||||
attemptId: completed.attempt.id,
|
||||
outcome: result.outcome as ExecutionOutcome,
|
||||
...(result.exitCode === undefined ? {} : { exitCode: result.exitCode }),
|
||||
};
|
||||
} finally {
|
||||
await prepared.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import type { ManualPrimaryExecutionRouter } from '../compatibility/manualPrimaryExecutionBridge';
|
||||
import type { RuntimeRolloutPolicy } from '../domain/runtimeRollout';
|
||||
import type {
|
||||
RuntimeRolloutLoadAudit,
|
||||
RuntimeRolloutLoadResult,
|
||||
} from '../ports/runtimeRolloutLoader';
|
||||
import type { PrimaryCancellationStopResult } from './primaryCancellationLifecycle';
|
||||
import type { PrimaryCompletionReceiptStopResult } from './primaryCompletionReceiptLifecycle';
|
||||
import type { PrimaryRunStartupSummary } from './primaryRunStartupSupervisor';
|
||||
import type { PrimaryTimeoutStopResult } from './primaryTimeoutLifecycle';
|
||||
|
||||
export type ManualPrimaryActivationState =
|
||||
| 'not_activated'
|
||||
| 'selected'
|
||||
| 'reconciled'
|
||||
| 'activated'
|
||||
| 'failed'
|
||||
| 'stopped';
|
||||
|
||||
export interface ManualPrimaryActivationAudit extends RuntimeRolloutLoadAudit {
|
||||
activation: ManualPrimaryActivationState;
|
||||
recovery?: {
|
||||
scanned: number;
|
||||
verifiedRunning: number;
|
||||
recoveredRunning: number;
|
||||
completedFromReceipt: number;
|
||||
quarantinedReceipts: number;
|
||||
publishGraceWaits: number;
|
||||
markedLost: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ManualPrimaryActivationStack {
|
||||
router: ManualPrimaryExecutionRouter;
|
||||
reconcile(): Promise<PrimaryRunStartupSummary>;
|
||||
startCompletion(): boolean;
|
||||
stopCompletion(): Promise<PrimaryCompletionReceiptStopResult>;
|
||||
startTimeout(): boolean;
|
||||
stopTimeout(): Promise<PrimaryTimeoutStopResult>;
|
||||
startCancellation(): boolean;
|
||||
stopCancellation(): Promise<PrimaryCancellationStopResult>;
|
||||
}
|
||||
|
||||
export interface ManualPrimaryRuntimeActivationOptions {
|
||||
load(): Promise<RuntimeRolloutLoadResult>;
|
||||
create(policy: RuntimeRolloutPolicy): ManualPrimaryActivationStack;
|
||||
install(router: ManualPrimaryExecutionRouter): () => void;
|
||||
audit(record: ManualPrimaryActivationAudit): void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ManualPrimaryRuntimeActivationResult {
|
||||
load: RuntimeRolloutLoadResult;
|
||||
active: boolean;
|
||||
recovery?: PrimaryRunStartupSummary;
|
||||
stop(): Promise<PrimaryCancellationStopResult>;
|
||||
}
|
||||
|
||||
const NOOP_STOP = async (): Promise<PrimaryCancellationStopResult> => 'drained';
|
||||
|
||||
async function stopLifecycles(
|
||||
stack: ManualPrimaryActivationStack,
|
||||
started: { completion: boolean; timeout: boolean; cancellation: boolean },
|
||||
): Promise<PrimaryCancellationStopResult> {
|
||||
let result: PrimaryCancellationStopResult = 'drained';
|
||||
let firstError: unknown;
|
||||
if (started.timeout) {
|
||||
try {
|
||||
if ((await stack.stopTimeout()) === 'timed_out') result = 'timed_out';
|
||||
} catch (error) {
|
||||
firstError = error;
|
||||
}
|
||||
}
|
||||
if (started.cancellation) {
|
||||
try {
|
||||
if ((await stack.stopCancellation()) === 'timed_out') {
|
||||
result = 'timed_out';
|
||||
}
|
||||
} catch (error) {
|
||||
firstError ??= error;
|
||||
}
|
||||
}
|
||||
if (started.completion) {
|
||||
try {
|
||||
if ((await stack.stopCompletion()) === 'timed_out') {
|
||||
result = 'timed_out';
|
||||
}
|
||||
} catch (error) {
|
||||
firstError ??= error;
|
||||
}
|
||||
}
|
||||
if (firstError !== undefined) throw firstError;
|
||||
return result;
|
||||
}
|
||||
|
||||
function assertSafeRecovery(summary: PrimaryRunStartupSummary): void {
|
||||
if (
|
||||
summary.remaining ||
|
||||
summary.stopReason !== 'complete' ||
|
||||
summary.skipped > 0 ||
|
||||
summary.ambiguous > 0 ||
|
||||
summary.failed > 0
|
||||
) {
|
||||
throw new Error('Primary startup reconciliation did not converge safely');
|
||||
}
|
||||
}
|
||||
|
||||
function recoveryAudit(summary: PrimaryRunStartupSummary) {
|
||||
return {
|
||||
scanned: summary.scanned,
|
||||
verifiedRunning: summary.verifiedRunning,
|
||||
recoveredRunning: summary.recoveredRunning,
|
||||
completedFromReceipt: summary.completedFromReceipt,
|
||||
quarantinedReceipts: summary.quarantinedReceipts,
|
||||
publishGraceWaits: summary.publishGraceWaits,
|
||||
markedLost: summary.markedLost,
|
||||
};
|
||||
}
|
||||
|
||||
export async function activateManualPrimaryRuntime(
|
||||
options: ManualPrimaryRuntimeActivationOptions,
|
||||
): Promise<ManualPrimaryRuntimeActivationResult> {
|
||||
const load = await options.load();
|
||||
const shouldActivate =
|
||||
load.status === 'accepted' && load.policy.modeFor('manual') === 'primary';
|
||||
if (!shouldActivate) {
|
||||
await options.audit({ ...load.audit, activation: 'not_activated' });
|
||||
return { load, active: false, stop: NOOP_STOP };
|
||||
}
|
||||
|
||||
let stack: ManualPrimaryActivationStack | undefined;
|
||||
let dispose: (() => void) | undefined;
|
||||
let completionStarted = false;
|
||||
let timeoutStarted = false;
|
||||
let cancellationStarted = false;
|
||||
try {
|
||||
await options.audit({ ...load.audit, activation: 'selected' });
|
||||
stack = options.create(load.policy);
|
||||
const recovery = await stack.reconcile();
|
||||
assertSafeRecovery(recovery);
|
||||
await options.audit({
|
||||
...load.audit,
|
||||
activation: 'reconciled',
|
||||
recovery: recoveryAudit(recovery),
|
||||
});
|
||||
completionStarted = stack.startCompletion();
|
||||
if (!completionStarted) {
|
||||
throw new Error('Primary completion lifecycle did not start');
|
||||
}
|
||||
timeoutStarted = stack.startTimeout();
|
||||
if (!timeoutStarted) {
|
||||
throw new Error('Primary timeout lifecycle did not start');
|
||||
}
|
||||
cancellationStarted = stack.startCancellation();
|
||||
if (!cancellationStarted) {
|
||||
throw new Error('Primary cancellation lifecycle did not start');
|
||||
}
|
||||
dispose = options.install(stack.router);
|
||||
await options.audit({
|
||||
...load.audit,
|
||||
activation: 'activated',
|
||||
recovery: recoveryAudit(recovery),
|
||||
});
|
||||
|
||||
let stopped = false;
|
||||
return {
|
||||
load,
|
||||
active: true,
|
||||
recovery,
|
||||
async stop() {
|
||||
if (stopped) return 'drained';
|
||||
stopped = true;
|
||||
dispose?.();
|
||||
const result = await stopLifecycles(stack!, {
|
||||
completion: completionStarted,
|
||||
timeout: timeoutStarted,
|
||||
cancellation: cancellationStarted,
|
||||
});
|
||||
try {
|
||||
await options.audit({
|
||||
...load.audit,
|
||||
activation: 'stopped',
|
||||
recovery: recoveryAudit(recovery),
|
||||
});
|
||||
} catch {
|
||||
// Cleanup must not be reversed by a diagnostic failure.
|
||||
}
|
||||
return result;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
dispose?.();
|
||||
if (stack && (completionStarted || timeoutStarted || cancellationStarted)) {
|
||||
try {
|
||||
await stopLifecycles(stack, {
|
||||
completion: completionStarted,
|
||||
timeout: timeoutStarted,
|
||||
cancellation: cancellationStarted,
|
||||
});
|
||||
} catch {
|
||||
// Preserve the activation error after best-effort cleanup.
|
||||
}
|
||||
}
|
||||
try {
|
||||
await options.audit({ ...load.audit, activation: 'failed' });
|
||||
} catch {
|
||||
// Preserve the activation error while ownership remains uninstalled.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import { normalizeExecutionContext } from '../domain/executionContext';
|
||||
import type { RunDispatchCandidate } from '../domain/runDispatchCandidate';
|
||||
import {
|
||||
executionSpecFromPinnedTaskRevision,
|
||||
type PinnedTaskExecutionRevision,
|
||||
} from '../domain/taskExecutionRevision';
|
||||
import type { LocalExecutionContextMaterializer } from '../ports/localExecutionContextMaterializer';
|
||||
import type {
|
||||
LocalRunDispatchPlan,
|
||||
LocalRunDispatchPlanSource,
|
||||
} from '../ports/localRunDispatchPlanSource';
|
||||
import type { TaskExecutionRevisionSource } from '../ports/taskExecutionRevisionSource';
|
||||
|
||||
/** Composes immutable Task facts with fresh Attempt-scoped local capabilities. */
|
||||
export class PinnedTaskLocalRunDispatchPlanSource
|
||||
implements LocalRunDispatchPlanSource
|
||||
{
|
||||
constructor(
|
||||
private readonly revisions: TaskExecutionRevisionSource,
|
||||
private readonly contexts: LocalExecutionContextMaterializer,
|
||||
) {}
|
||||
|
||||
async prepare(
|
||||
candidate: Readonly<RunDispatchCandidate>,
|
||||
): Promise<LocalRunDispatchPlan | null> {
|
||||
const revision = await this.revisions.resolve(
|
||||
Object.freeze({
|
||||
projectId: candidate.projectId,
|
||||
taskId: candidate.taskId,
|
||||
taskRevision: candidate.taskRevision,
|
||||
}),
|
||||
);
|
||||
if (!revision) return null;
|
||||
const executionSpec = executionSpecFromPinnedTaskRevision(
|
||||
candidate,
|
||||
this.knownRevision(revision),
|
||||
);
|
||||
const context = await this.contexts.prepare(
|
||||
Object.freeze({
|
||||
candidate: Object.freeze({ ...candidate }),
|
||||
contextRef: revision.contextRef,
|
||||
}),
|
||||
);
|
||||
if (!context) return null;
|
||||
let normalizedContext;
|
||||
try {
|
||||
normalizedContext = normalizeExecutionContext(context.context);
|
||||
} catch (error) {
|
||||
try {
|
||||
await context.dispose?.();
|
||||
} catch {
|
||||
// Cleanup failure must not replace the validation failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
executionSpec,
|
||||
context: normalizedContext,
|
||||
...(context.logArtifactId === undefined
|
||||
? {}
|
||||
: { logArtifactId: context.logArtifactId }),
|
||||
...(context.dispose === undefined ? {} : { dispose: context.dispose }),
|
||||
};
|
||||
}
|
||||
|
||||
private knownRevision(
|
||||
revision: PinnedTaskExecutionRevision,
|
||||
): PinnedTaskExecutionRevision {
|
||||
return {
|
||||
projectId: revision.projectId,
|
||||
taskId: revision.taskId,
|
||||
taskRevision: revision.taskRevision,
|
||||
executorType: revision.executorType,
|
||||
execution: revision.execution,
|
||||
contextRef: revision.contextRef,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type { CancellationDispatchResult } from '../domain/cancellationDispatch';
|
||||
import { RUN_CANCELLATION_REASONS } from '../domain/run';
|
||||
import type { CancellationDispatchRepository } from '../ports/cancellationDispatchRepository';
|
||||
import type { PersistedExecutionController } from '../ports/persistedExecutionController';
|
||||
import type {
|
||||
PrimaryCancellationAttemptReference,
|
||||
PrimaryCancellationCursor,
|
||||
PrimaryCancellationSource,
|
||||
} from '../ports/primaryCancellationSource';
|
||||
|
||||
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
||||
const DEFAULT_RETRY_BASE_MS = 1_000;
|
||||
const DEFAULT_RETRY_MAX_MS = 60_000;
|
||||
|
||||
export interface PrimaryCancellationDispatchSummary {
|
||||
scanned: number;
|
||||
claimed: number;
|
||||
terminationRequested: number;
|
||||
alreadyExited: number;
|
||||
pending: number;
|
||||
ambiguous: number;
|
||||
blocked: number;
|
||||
deferred: number;
|
||||
alreadyResolved: number;
|
||||
notEligible: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
unsafeAttemptOverflow: boolean;
|
||||
nextCursor?: PrimaryCancellationCursor;
|
||||
}
|
||||
|
||||
export interface PrimaryCancellationDispatcherOptions {
|
||||
owner: string;
|
||||
leaseDurationMs?: number;
|
||||
retryBaseMs?: number;
|
||||
retryMaxMs?: number;
|
||||
clock?: () => number;
|
||||
createId?: () => string;
|
||||
}
|
||||
|
||||
function assertPositiveInteger(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
/** One bounded pass. The caller owns scheduling and pagination. */
|
||||
export class PrimaryCancellationDispatcher {
|
||||
private readonly controllers = new Map<
|
||||
PersistedExecutionController['executorType'],
|
||||
PersistedExecutionController
|
||||
>();
|
||||
private readonly owner: string;
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly retryBaseMs: number;
|
||||
private readonly retryMaxMs: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly createId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly source: PrimaryCancellationSource,
|
||||
private readonly dispatches: CancellationDispatchRepository,
|
||||
controllers: readonly PersistedExecutionController[],
|
||||
options: PrimaryCancellationDispatcherOptions,
|
||||
) {
|
||||
if (!options.owner || options.owner.length > 128) {
|
||||
throw new RangeError('owner must be between 1 and 128 characters');
|
||||
}
|
||||
this.owner = options.owner;
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
||||
this.retryBaseMs = options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS;
|
||||
this.retryMaxMs = options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS;
|
||||
this.clock = options.clock ?? Date.now;
|
||||
this.createId = options.createId ?? uuidV7;
|
||||
assertPositiveInteger('leaseDurationMs', this.leaseDurationMs);
|
||||
assertPositiveInteger('retryBaseMs', this.retryBaseMs);
|
||||
assertPositiveInteger('retryMaxMs', this.retryMaxMs);
|
||||
if (this.retryMaxMs < this.retryBaseMs) {
|
||||
throw new RangeError(
|
||||
'retryMaxMs must be greater than or equal to retryBaseMs',
|
||||
);
|
||||
}
|
||||
|
||||
for (const controller of controllers) {
|
||||
if (this.controllers.has(controller.executorType)) {
|
||||
throw new Error(
|
||||
`Duplicate persisted Executor controller: ${controller.executorType}`,
|
||||
);
|
||||
}
|
||||
this.controllers.set(controller.executorType, controller);
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchBatch(
|
||||
options: { cursor?: PrimaryCancellationCursor; limit?: number } = {},
|
||||
): Promise<PrimaryCancellationDispatchSummary> {
|
||||
const page = await this.source.listCandidates(options);
|
||||
const summary: PrimaryCancellationDispatchSummary = {
|
||||
scanned: page.candidates.length,
|
||||
claimed: 0,
|
||||
terminationRequested: 0,
|
||||
alreadyExited: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
blocked: 0,
|
||||
deferred: 0,
|
||||
alreadyResolved: 0,
|
||||
notEligible: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
unsafeAttemptOverflow: page.unsafeAttemptOverflow,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
if (page.unsafeAttemptOverflow) return summary;
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
if (!RUN_CANCELLATION_REASONS.includes(candidate.reason)) {
|
||||
summary.pending += 1;
|
||||
continue;
|
||||
}
|
||||
if (candidate.attempts.length === 0) {
|
||||
summary.pending += 1;
|
||||
continue;
|
||||
}
|
||||
if (candidate.attempts.length > 1) {
|
||||
summary.ambiguous += 1;
|
||||
summary.pending += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const attempt = candidate.attempts[0];
|
||||
const claimedAtMs = this.now();
|
||||
let claim;
|
||||
try {
|
||||
claim = await this.dispatches.claim({
|
||||
runId: candidate.runId,
|
||||
attemptId: attempt.attemptId,
|
||||
requestedAtMs: candidate.requestedAtMs,
|
||||
owner: this.owner,
|
||||
leaseToken: this.createId(),
|
||||
nowMs: claimedAtMs,
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
});
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
summary.pending += 1;
|
||||
continue;
|
||||
}
|
||||
if (claim.status === 'not_eligible') {
|
||||
summary.notEligible += 1;
|
||||
continue;
|
||||
}
|
||||
if (claim.status === 'leased' || claim.status === 'not_due') {
|
||||
summary.deferred += 1;
|
||||
summary.pending += 1;
|
||||
continue;
|
||||
}
|
||||
if (claim.status === 'dispatched') {
|
||||
summary.alreadyResolved += 1;
|
||||
continue;
|
||||
}
|
||||
if (claim.status === 'blocked') {
|
||||
summary.alreadyResolved += 1;
|
||||
summary.blocked += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
summary.claimed += 1;
|
||||
await this.dispatchClaimed(
|
||||
candidate.reason,
|
||||
candidate.requestedAtMs,
|
||||
attempt,
|
||||
claim.dispatch,
|
||||
summary,
|
||||
);
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
private async dispatchClaimed(
|
||||
reason: (typeof RUN_CANCELLATION_REASONS)[number],
|
||||
requestedAtMs: number,
|
||||
attempt: PrimaryCancellationAttemptReference,
|
||||
dispatch: Extract<
|
||||
Awaited<ReturnType<CancellationDispatchRepository['claim']>>,
|
||||
{ status: 'claimed' }
|
||||
>['dispatch'],
|
||||
summary: PrimaryCancellationDispatchSummary,
|
||||
): Promise<void> {
|
||||
const controller = this.controllers.get(attempt.executorType);
|
||||
if (!controller) {
|
||||
await this.record(attempt, dispatch, 'controller_missing', summary);
|
||||
return;
|
||||
}
|
||||
if (!attempt.executorHandle) {
|
||||
await this.record(attempt, dispatch, 'handle_missing', summary);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await controller.stop({
|
||||
durableHandle: attempt.executorHandle,
|
||||
...(attempt.pid === undefined ? {} : { expectedPid: attempt.pid }),
|
||||
reason: {
|
||||
kind: reason,
|
||||
requestedAtMs,
|
||||
},
|
||||
});
|
||||
await this.record(attempt, dispatch, result.status, summary);
|
||||
if (result.status === 'termination_requested') {
|
||||
summary.terminationRequested += 1;
|
||||
} else if (result.status === 'already_exited') {
|
||||
summary.alreadyExited += 1;
|
||||
} else {
|
||||
summary.blocked += 1;
|
||||
}
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
await this.record(attempt, dispatch, 'dispatch_error', summary);
|
||||
}
|
||||
}
|
||||
|
||||
private async record(
|
||||
attempt: PrimaryCancellationAttemptReference,
|
||||
dispatch: Extract<
|
||||
Awaited<ReturnType<CancellationDispatchRepository['claim']>>,
|
||||
{ status: 'claimed' }
|
||||
>['dispatch'],
|
||||
result: CancellationDispatchResult,
|
||||
summary: PrimaryCancellationDispatchSummary,
|
||||
): Promise<void> {
|
||||
const atMs = this.now();
|
||||
const retryable = [
|
||||
'controller_missing',
|
||||
'handle_missing',
|
||||
'dispatch_error',
|
||||
].includes(result);
|
||||
try {
|
||||
await this.dispatches.recordResult({
|
||||
runId: dispatch.runId,
|
||||
attemptId: attempt.attemptId,
|
||||
owner: this.owner,
|
||||
leaseToken: dispatch.leaseToken!,
|
||||
expectedVersion: dispatch.version,
|
||||
result,
|
||||
atMs,
|
||||
...(retryable
|
||||
? { nextAttemptAtMs: this.nextRetryAt(atMs, dispatch.dispatchCount) }
|
||||
: {}),
|
||||
eventId: this.createId(),
|
||||
});
|
||||
if (retryable) summary.pending += 1;
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
summary.pending += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private nextRetryAt(atMs: number, dispatchCount: number): number {
|
||||
const exponent = Math.max(0, Math.min(dispatchCount - 1, 30));
|
||||
const delay = Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** exponent);
|
||||
return Math.min(Number.MAX_SAFE_INTEGER, atMs + delay);
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const nowMs = this.clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('clock must return a non-negative safe integer');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
import type {
|
||||
PrimaryCancellationCycleOptions,
|
||||
PrimaryCancellationCycleSummary,
|
||||
PrimaryCancellationSupervisor,
|
||||
} from './primaryCancellationSupervisor';
|
||||
|
||||
export const MIN_CANCELLATION_CYCLE_INTERVAL_MS = 250;
|
||||
export const MAX_CANCELLATION_CYCLE_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_CANCELLATION_INITIAL_DELAY_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_CANCELLATION_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface CancellationLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface PrimaryCancellationLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
cycle?: PrimaryCancellationCycleOptions;
|
||||
scheduler?: CancellationLifecycleScheduler;
|
||||
onCycle?: (summary: PrimaryCancellationCycleSummary) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type PrimaryCancellationStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: CancellationLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit lifecycle wrapper for a bounded supervisor cycle. It is inert until
|
||||
* start() is called and schedules the next cycle only after the current one
|
||||
* settles, so slow edge devices cannot accumulate overlapping scans.
|
||||
*/
|
||||
export class PrimaryCancellationLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly cycleOptions: PrimaryCancellationCycleOptions;
|
||||
private readonly scheduler: CancellationLifecycleScheduler;
|
||||
private readonly onCycle?: (summary: PrimaryCancellationCycleSummary) => void;
|
||||
private readonly onError?: (error: unknown) => void;
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
|
||||
constructor(
|
||||
private readonly supervisor: Pick<
|
||||
PrimaryCancellationSupervisor,
|
||||
'runCycle'
|
||||
>,
|
||||
options: PrimaryCancellationLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.cycleOptions = {
|
||||
...(options.cycle?.cursor === undefined
|
||||
? {}
|
||||
: { cursor: { ...options.cycle.cursor } }),
|
||||
...(options.cycle?.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.cycle.pageSize }),
|
||||
...(options.cycle?.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.cycle.maxPages }),
|
||||
};
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_CANCELLATION_CYCLE_INTERVAL_MS,
|
||||
MAX_CANCELLATION_CYCLE_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_CANCELLATION_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_CANCELLATION_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<PrimaryCancellationStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<PrimaryCancellationStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.supervisor
|
||||
.runCycle(this.cycleOptions)
|
||||
.then((summary) => this.notifyCycle(summary))
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private notifyCycle(summary: PrimaryCancellationCycleSummary): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must never create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { MAX_PRIMARY_CANCELLATION_BATCH_SIZE } from '../ports/primaryCancellationSource';
|
||||
import type { PrimaryCancellationCursor } from '../ports/primaryCancellationSource';
|
||||
import type {
|
||||
PrimaryCancellationDispatcher,
|
||||
PrimaryCancellationDispatchSummary,
|
||||
} from './primaryCancellationDispatcher';
|
||||
|
||||
export const MAX_PRIMARY_CANCELLATION_PAGES_PER_CYCLE = 64;
|
||||
|
||||
export type PrimaryCancellationCycleStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'unsafe_attempt_overflow'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface PrimaryCancellationCycleSummary
|
||||
extends Omit<
|
||||
PrimaryCancellationDispatchSummary,
|
||||
'truncated' | 'unsafeAttemptOverflow' | 'nextCursor'
|
||||
> {
|
||||
pages: number;
|
||||
stopReason: PrimaryCancellationCycleStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: PrimaryCancellationCursor;
|
||||
}
|
||||
|
||||
export interface PrimaryCancellationCycleOptions {
|
||||
cursor?: PrimaryCancellationCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: PrimaryCancellationCursor | undefined,
|
||||
right: PrimaryCancellationCursor,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.requestedAtMs === right.requestedAtMs &&
|
||||
left.runId === right.runId
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a bounded recovery cycle. It deliberately owns no timer or process hook;
|
||||
* edge and cluster deployments choose their own cadence and lifecycle.
|
||||
*/
|
||||
export class PrimaryCancellationSupervisor {
|
||||
constructor(
|
||||
private readonly dispatcher: Pick<
|
||||
PrimaryCancellationDispatcher,
|
||||
'dispatchBatch'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async runCycle(
|
||||
options: PrimaryCancellationCycleOptions = {},
|
||||
): Promise<PrimaryCancellationCycleSummary> {
|
||||
const pageSize = options.pageSize ?? 32;
|
||||
const maxPages = options.maxPages ?? 4;
|
||||
if (
|
||||
!Number.isSafeInteger(pageSize) ||
|
||||
pageSize < 1 ||
|
||||
pageSize > MAX_PRIMARY_CANCELLATION_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'pageSize must be between 1 and MAX_PRIMARY_CANCELLATION_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxPages) ||
|
||||
maxPages < 1 ||
|
||||
maxPages > MAX_PRIMARY_CANCELLATION_PAGES_PER_CYCLE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_PRIMARY_CANCELLATION_PAGES_PER_CYCLE',
|
||||
);
|
||||
}
|
||||
|
||||
const total: PrimaryCancellationCycleSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
claimed: 0,
|
||||
terminationRequested: 0,
|
||||
alreadyExited: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
blocked: 0,
|
||||
deferred: 0,
|
||||
alreadyResolved: 0,
|
||||
notEligible: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
|
||||
for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
|
||||
const page = await this.dispatcher.dispatchBatch({
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.claimed += page.claimed;
|
||||
total.terminationRequested += page.terminationRequested;
|
||||
total.alreadyExited += page.alreadyExited;
|
||||
total.pending += page.pending;
|
||||
total.ambiguous += page.ambiguous;
|
||||
total.blocked += page.blocked;
|
||||
total.deferred += page.deferred;
|
||||
total.alreadyResolved += page.alreadyResolved;
|
||||
total.notEligible += page.notEligible;
|
||||
total.failed += page.failed;
|
||||
|
||||
if (page.unsafeAttemptOverflow) {
|
||||
total.stopReason = 'unsafe_attempt_overflow';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
if (!page.truncated) return total;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
if (pageNumber === maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return total;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import type { CompletionReceipt } from '../domain/completionReceipt';
|
||||
import { InvalidCompletionReceiptError } from '../domain/completionReceipt';
|
||||
import type { CompletionReceiptStore } from '../ports/completionReceiptStore';
|
||||
import type { CompletionReceiptJournal } from '../ports/completionReceiptJournal';
|
||||
import type {
|
||||
PrimaryRunCompletionResult,
|
||||
PrimaryRunCompletionService,
|
||||
} from './primaryRunCompletionService';
|
||||
import {
|
||||
PrimaryCompletionNotFoundError,
|
||||
PrimaryCompletionSequenceError,
|
||||
PrimaryCompletionStateError,
|
||||
PrimaryCompletionUnauthorizedError,
|
||||
} from './primaryRunCompletionService';
|
||||
|
||||
export interface PrimaryCompletionReceiptConsumeResult {
|
||||
status: 'missing' | 'quarantined' | PrimaryRunCompletionResult['status'];
|
||||
cleaned: boolean;
|
||||
quarantineRef?: string;
|
||||
completion?: PrimaryRunCompletionResult;
|
||||
}
|
||||
|
||||
export interface PrimaryCompletionReceiptConsumerOptions {
|
||||
journal?: Pick<CompletionReceiptJournal, 'markQuarantined' | 'resolve'>;
|
||||
quarantineRetentionMs?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
function mustQuarantine(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof InvalidCompletionReceiptError ||
|
||||
error instanceof PrimaryCompletionNotFoundError ||
|
||||
error instanceof PrimaryCompletionUnauthorizedError ||
|
||||
error instanceof PrimaryCompletionSequenceError ||
|
||||
error instanceof PrimaryCompletionStateError
|
||||
);
|
||||
}
|
||||
|
||||
function receiptResult(receipt: CompletionReceipt) {
|
||||
return {
|
||||
outcome:
|
||||
receipt.exitCode === 0 ? ('succeeded' as const) : ('failed' as const),
|
||||
startedAtMs: receipt.startedAtMs,
|
||||
finishedAtMs: receipt.finishedAtMs,
|
||||
exitCode: receipt.exitCode,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads only a database-discovered Attempt receipt. Cleanup happens after the
|
||||
* terminal transaction; failures leave the immutable receipt replayable.
|
||||
*/
|
||||
export class PrimaryCompletionReceiptConsumer {
|
||||
private readonly journal?: Pick<
|
||||
CompletionReceiptJournal,
|
||||
'markQuarantined' | 'resolve'
|
||||
>;
|
||||
private readonly quarantineRetentionMs: number;
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly store: CompletionReceiptStore,
|
||||
private readonly completions: Pick<PrimaryRunCompletionService, 'complete'>,
|
||||
options: PrimaryCompletionReceiptConsumerOptions = {},
|
||||
) {
|
||||
this.journal = options.journal;
|
||||
this.quarantineRetentionMs = options.quarantineRetentionMs ?? 60 * 60_000;
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
if (
|
||||
!Number.isSafeInteger(this.quarantineRetentionMs) ||
|
||||
this.quarantineRetentionMs < 1 ||
|
||||
this.quarantineRetentionMs > 30 * 24 * 60 * 60_000
|
||||
) {
|
||||
throw new RangeError(
|
||||
'quarantineRetentionMs must be between 1 and 30 days',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async consume(
|
||||
attemptId: string,
|
||||
): Promise<PrimaryCompletionReceiptConsumeResult> {
|
||||
let completion: PrimaryRunCompletionResult;
|
||||
try {
|
||||
const receipt = await this.store.read(attemptId);
|
||||
if (!receipt) return { status: 'missing', cleaned: false };
|
||||
completion = await this.completions.complete({
|
||||
runId: receipt.runId,
|
||||
attemptId: receipt.attemptId,
|
||||
callbackSequence: receipt.callbackSequence,
|
||||
result: receiptResult(receipt),
|
||||
source: { kind: 'receipt', token: receipt.token },
|
||||
});
|
||||
} catch (error) {
|
||||
if (!mustQuarantine(error)) throw error;
|
||||
const quarantineRef = this.store.quarantineReference(attemptId);
|
||||
if (this.journal) {
|
||||
const updatedAtMs = this.clock.now();
|
||||
const purgeAfterMs = updatedAtMs + this.quarantineRetentionMs;
|
||||
if (
|
||||
!Number.isSafeInteger(updatedAtMs) ||
|
||||
updatedAtMs < 0 ||
|
||||
!Number.isSafeInteger(purgeAfterMs)
|
||||
) {
|
||||
throw new RangeError('Completion receipt quarantine time is invalid');
|
||||
}
|
||||
await this.journal.markQuarantined({
|
||||
attemptId,
|
||||
quarantineRef,
|
||||
updatedAtMs,
|
||||
purgeAfterMs,
|
||||
});
|
||||
}
|
||||
const quarantined = await this.store.quarantine(attemptId);
|
||||
if (!quarantined) return { status: 'missing', cleaned: false };
|
||||
return {
|
||||
status: 'quarantined',
|
||||
cleaned: true,
|
||||
quarantineRef: quarantined,
|
||||
};
|
||||
}
|
||||
const cleaned = await this.store.remove(attemptId);
|
||||
if (cleaned) await this.journal?.resolve(attemptId);
|
||||
return { status: completion.status, cleaned, completion };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
import type { CompletionReceiptJournal } from '../ports/completionReceiptJournal';
|
||||
import type { CompletionReceiptStore } from '../ports/completionReceiptStore';
|
||||
import type { PrimaryRunRecoveryCursor } from '../ports/primaryRunRecoverySource';
|
||||
import { isTerminalRunAttemptStatus } from '../domain/runStateMachine';
|
||||
import type { PrimaryCompletionReceiptConsumer } from './primaryCompletionReceiptConsumer';
|
||||
import type { PrimaryCompletionReceiptScanSummary } from './primaryCompletionReceiptScanner';
|
||||
|
||||
export interface PrimaryCompletionReceiptJournalScannerOptions {
|
||||
terminalMissingRetentionMs?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
/**
|
||||
* Database-indexed receipt retention. The supervisor cursor is only a transport
|
||||
* shape here: createdAtMs carries journal.updatedAtMs and runId carries Attempt
|
||||
* id. No directory enumeration is performed.
|
||||
*/
|
||||
export class PrimaryCompletionReceiptJournalScanner {
|
||||
private readonly terminalMissingRetentionMs: number;
|
||||
private readonly clock: { now(): number };
|
||||
|
||||
constructor(
|
||||
private readonly journal: CompletionReceiptJournal,
|
||||
private readonly store: CompletionReceiptStore,
|
||||
private readonly consumer: Pick<
|
||||
PrimaryCompletionReceiptConsumer,
|
||||
'consume'
|
||||
>,
|
||||
options: PrimaryCompletionReceiptJournalScannerOptions = {},
|
||||
) {
|
||||
this.terminalMissingRetentionMs =
|
||||
options.terminalMissingRetentionMs ?? 60_000;
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
if (
|
||||
!Number.isSafeInteger(this.terminalMissingRetentionMs) ||
|
||||
this.terminalMissingRetentionMs < 0 ||
|
||||
this.terminalMissingRetentionMs > 24 * 60 * 60_000
|
||||
) {
|
||||
throw new RangeError(
|
||||
'terminalMissingRetentionMs must be between 0 and 24 hours',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async scanBatch(
|
||||
options: { cursor?: PrimaryRunRecoveryCursor; limit?: number } = {},
|
||||
): Promise<PrimaryCompletionReceiptScanSummary> {
|
||||
const observedAtMs = this.clock.now();
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new RangeError('Completion receipt observation time is invalid');
|
||||
}
|
||||
const page = await this.journal.listCandidates({
|
||||
observedAtMs,
|
||||
...(options.limit === undefined ? {} : { limit: options.limit }),
|
||||
...(options.cursor === undefined
|
||||
? {}
|
||||
: {
|
||||
cursor: {
|
||||
updatedAtMs: options.cursor.createdAtMs,
|
||||
attemptId: options.cursor.runId,
|
||||
},
|
||||
}),
|
||||
});
|
||||
const summary: PrimaryCompletionReceiptScanSummary = {
|
||||
scanned: page.candidates.length,
|
||||
applied: 0,
|
||||
alreadyTerminal: 0,
|
||||
quarantined: 0,
|
||||
purgedQuarantines: 0,
|
||||
expiredMissing: 0,
|
||||
missing: 0,
|
||||
cleanupPending: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
unsafeAttemptOverflow: false,
|
||||
...(page.nextCursor === undefined
|
||||
? {}
|
||||
: {
|
||||
nextCursor: {
|
||||
createdAtMs: page.nextCursor.updatedAtMs,
|
||||
runId: page.nextCursor.attemptId,
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
if (candidate.executorType !== 'local_process') {
|
||||
summary.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
if (candidate.state === 'quarantined') {
|
||||
await this.store.quarantine(candidate.attemptId);
|
||||
await this.store.purgeQuarantine(candidate.attemptId);
|
||||
await this.journal.resolve(candidate.attemptId);
|
||||
summary.purgedQuarantines += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const result = await this.consumer.consume(candidate.attemptId);
|
||||
if (result.status === 'quarantined') {
|
||||
summary.quarantined += 1;
|
||||
continue;
|
||||
}
|
||||
if (result.status === 'missing') {
|
||||
if (
|
||||
isTerminalRunAttemptStatus(candidate.attemptStatus) &&
|
||||
candidate.finishedAtMs !== undefined &&
|
||||
candidate.finishedAtMs + this.terminalMissingRetentionMs <=
|
||||
observedAtMs
|
||||
) {
|
||||
await this.journal.resolve(candidate.attemptId);
|
||||
summary.expiredMissing += 1;
|
||||
} else {
|
||||
summary.missing += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (result.status === 'applied') summary.applied += 1;
|
||||
else summary.alreadyTerminal += 1;
|
||||
if (!result.cleaned) summary.cleanupPending += 1;
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
import type {
|
||||
PrimaryCompletionReceiptSupervisor,
|
||||
PrimaryCompletionReceiptSupervisorOptions,
|
||||
PrimaryCompletionReceiptSupervisorSummary,
|
||||
} from './primaryCompletionReceiptSupervisor';
|
||||
import type { PrimaryRunRecoveryCursor } from '../ports/primaryRunRecoverySource';
|
||||
|
||||
export const MIN_COMPLETION_RECEIPT_INTERVAL_MS = 250;
|
||||
export const MAX_COMPLETION_RECEIPT_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_COMPLETION_RECEIPT_INITIAL_DELAY_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_COMPLETION_RECEIPT_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface CompletionReceiptLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface PrimaryCompletionReceiptLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
cycle?: PrimaryCompletionReceiptSupervisorOptions;
|
||||
scheduler?: CompletionReceiptLifecycleScheduler;
|
||||
onCycle?: (summary: PrimaryCompletionReceiptSupervisorSummary) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type PrimaryCompletionReceiptStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: CompletionReceiptLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Explicit, non-overlapping receipt polling for small edge deployments. */
|
||||
export class PrimaryCompletionReceiptLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly cycleOptions: Omit<
|
||||
PrimaryCompletionReceiptSupervisorOptions,
|
||||
'cursor'
|
||||
>;
|
||||
private readonly scheduler: CompletionReceiptLifecycleScheduler;
|
||||
private readonly onCycle?: (
|
||||
summary: PrimaryCompletionReceiptSupervisorSummary,
|
||||
) => void;
|
||||
private readonly onError?: (error: unknown) => void;
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
private resumeCursor?: PrimaryRunRecoveryCursor;
|
||||
|
||||
constructor(
|
||||
private readonly supervisor: Pick<
|
||||
PrimaryCompletionReceiptSupervisor,
|
||||
'run'
|
||||
>,
|
||||
options: PrimaryCompletionReceiptLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.cycleOptions = {
|
||||
...(options.cycle?.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.cycle.pageSize }),
|
||||
...(options.cycle?.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.cycle.maxPages }),
|
||||
};
|
||||
this.resumeCursor =
|
||||
options.cycle?.cursor === undefined
|
||||
? undefined
|
||||
: { ...options.cycle.cursor };
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_COMPLETION_RECEIPT_INTERVAL_MS,
|
||||
MAX_COMPLETION_RECEIPT_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_COMPLETION_RECEIPT_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_COMPLETION_RECEIPT_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<PrimaryCompletionReceiptStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<PrimaryCompletionReceiptStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.supervisor
|
||||
.run({
|
||||
...this.cycleOptions,
|
||||
...(this.resumeCursor === undefined
|
||||
? {}
|
||||
: { cursor: { ...this.resumeCursor } }),
|
||||
})
|
||||
.then((summary) => {
|
||||
this.resumeCursor =
|
||||
summary.remaining && summary.nextCursor
|
||||
? { ...summary.nextCursor }
|
||||
: undefined;
|
||||
this.notifyCycle(summary);
|
||||
})
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private notifyCycle(
|
||||
summary: PrimaryCompletionReceiptSupervisorSummary,
|
||||
): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must never create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import type {
|
||||
PrimaryRunRecoveryCursor,
|
||||
PrimaryRunRecoverySource,
|
||||
} from '../ports/primaryRunRecoverySource';
|
||||
import type { PrimaryCompletionReceiptConsumer } from './primaryCompletionReceiptConsumer';
|
||||
|
||||
export interface PrimaryCompletionReceiptScanSummary {
|
||||
scanned: number;
|
||||
applied: number;
|
||||
alreadyTerminal: number;
|
||||
quarantined: number;
|
||||
purgedQuarantines: number;
|
||||
expiredMissing: number;
|
||||
missing: number;
|
||||
cleanupPending: number;
|
||||
skipped: number;
|
||||
ambiguous: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
unsafeAttemptOverflow: boolean;
|
||||
nextCursor?: PrimaryRunRecoveryCursor;
|
||||
}
|
||||
|
||||
/**
|
||||
* One bounded database-driven receipt pass. The database is the index: this
|
||||
* scanner never watches or enumerates the receipt directory.
|
||||
*/
|
||||
export class PrimaryCompletionReceiptScanner {
|
||||
constructor(
|
||||
private readonly source: PrimaryRunRecoverySource,
|
||||
private readonly consumer: Pick<
|
||||
PrimaryCompletionReceiptConsumer,
|
||||
'consume'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async scanBatch(
|
||||
options: {
|
||||
cursor?: PrimaryRunRecoveryCursor;
|
||||
limit?: number;
|
||||
} = {},
|
||||
): Promise<PrimaryCompletionReceiptScanSummary> {
|
||||
const page = await this.source.listCandidates(options);
|
||||
const summary: PrimaryCompletionReceiptScanSummary = {
|
||||
scanned: page.candidates.length,
|
||||
applied: 0,
|
||||
alreadyTerminal: 0,
|
||||
quarantined: 0,
|
||||
purgedQuarantines: 0,
|
||||
expiredMissing: 0,
|
||||
missing: 0,
|
||||
cleanupPending: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
unsafeAttemptOverflow: page.unsafeAttemptOverflow,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
if (page.unsafeAttemptOverflow) return summary;
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
if (candidate.attempts.length !== 1) {
|
||||
summary.ambiguous += 1;
|
||||
continue;
|
||||
}
|
||||
const attempt = candidate.attempts[0];
|
||||
if (attempt.executorType !== 'local_process') {
|
||||
summary.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const result = await this.consumer.consume(attempt.attemptId);
|
||||
if (result.status === 'missing') {
|
||||
summary.missing += 1;
|
||||
continue;
|
||||
}
|
||||
if (result.status === 'quarantined') {
|
||||
summary.quarantined += 1;
|
||||
continue;
|
||||
}
|
||||
if (result.status === 'applied') summary.applied += 1;
|
||||
else summary.alreadyTerminal += 1;
|
||||
if (!result.cleaned) summary.cleanupPending += 1;
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import {
|
||||
MAX_PRIMARY_RECOVERY_BATCH_SIZE,
|
||||
type PrimaryRunRecoveryCursor,
|
||||
} from '../ports/primaryRunRecoverySource';
|
||||
import type {
|
||||
PrimaryCompletionReceiptScanner,
|
||||
PrimaryCompletionReceiptScanSummary,
|
||||
} from './primaryCompletionReceiptScanner';
|
||||
|
||||
export const MAX_PRIMARY_COMPLETION_RECEIPT_PAGES = 64;
|
||||
|
||||
export type PrimaryCompletionReceiptStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'unsafe_attempt_overflow'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface PrimaryCompletionReceiptSupervisorOptions {
|
||||
cursor?: PrimaryRunRecoveryCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
export interface PrimaryCompletionReceiptSupervisorSummary
|
||||
extends Omit<
|
||||
PrimaryCompletionReceiptScanSummary,
|
||||
'truncated' | 'unsafeAttemptOverflow' | 'nextCursor'
|
||||
> {
|
||||
pages: number;
|
||||
stopReason: PrimaryCompletionReceiptStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: PrimaryRunRecoveryCursor;
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: PrimaryRunRecoveryCursor | undefined,
|
||||
right: PrimaryRunRecoveryCursor,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.createdAtMs === right.createdAtMs &&
|
||||
left.runId === right.runId
|
||||
);
|
||||
}
|
||||
|
||||
export class PrimaryCompletionReceiptSupervisor {
|
||||
constructor(
|
||||
private readonly scanner: Pick<
|
||||
PrimaryCompletionReceiptScanner,
|
||||
'scanBatch'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async run(
|
||||
options: PrimaryCompletionReceiptSupervisorOptions = {},
|
||||
): Promise<PrimaryCompletionReceiptSupervisorSummary> {
|
||||
const pageSize = options.pageSize ?? 32;
|
||||
const maxPages = options.maxPages ?? 4;
|
||||
if (
|
||||
!Number.isSafeInteger(pageSize) ||
|
||||
pageSize < 1 ||
|
||||
pageSize > MAX_PRIMARY_RECOVERY_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'pageSize must be between 1 and MAX_PRIMARY_RECOVERY_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxPages) ||
|
||||
maxPages < 1 ||
|
||||
maxPages > MAX_PRIMARY_COMPLETION_RECEIPT_PAGES
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_PRIMARY_COMPLETION_RECEIPT_PAGES',
|
||||
);
|
||||
}
|
||||
|
||||
const total: PrimaryCompletionReceiptSupervisorSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
applied: 0,
|
||||
alreadyTerminal: 0,
|
||||
quarantined: 0,
|
||||
purgedQuarantines: 0,
|
||||
expiredMissing: 0,
|
||||
missing: 0,
|
||||
cleanupPending: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
|
||||
for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
|
||||
const page = await this.scanner.scanBatch({
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.applied += page.applied;
|
||||
total.alreadyTerminal += page.alreadyTerminal;
|
||||
total.quarantined += page.quarantined;
|
||||
total.purgedQuarantines += page.purgedQuarantines;
|
||||
total.expiredMissing += page.expiredMissing;
|
||||
total.missing += page.missing;
|
||||
total.cleanupPending += page.cleanupPending;
|
||||
total.skipped += page.skipped;
|
||||
total.ambiguous += page.ambiguous;
|
||||
total.failed += page.failed;
|
||||
|
||||
if (page.unsafeAttemptOverflow) {
|
||||
total.stopReason = 'unsafe_attempt_overflow';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
if (!page.truncated) return total;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
if (page.nextCursor) total.nextCursor = page.nextCursor;
|
||||
return total;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
if (pageNumber === maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return total;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
import { createHash, timingSafeEqual } from 'crypto';
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type { ExecutionOutcome, ExecutionResult } from '../domain/execution';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunStatus,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
isTerminalRunAttemptStatus,
|
||||
transitionRun,
|
||||
transitionRunAttempt,
|
||||
type RunDomainEventDraft,
|
||||
} from '../domain/runStateMachine';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
|
||||
interface TerminalMapping {
|
||||
attemptStatus: Exclude<
|
||||
RunAttemptRecord['status'],
|
||||
'claimed' | 'starting' | 'running'
|
||||
>;
|
||||
runStatus: Exclude<
|
||||
RunStatus,
|
||||
| 'created'
|
||||
| 'queued'
|
||||
| 'dispatching'
|
||||
| 'running'
|
||||
| 'waiting_approval'
|
||||
| 'retry_wait'
|
||||
>;
|
||||
errorCode?: string;
|
||||
errorSummary?: string;
|
||||
}
|
||||
|
||||
const TERMINAL_MAPPING: Readonly<Record<ExecutionOutcome, TerminalMapping>> = {
|
||||
succeeded: {
|
||||
attemptStatus: 'succeeded',
|
||||
runStatus: 'succeeded',
|
||||
},
|
||||
failed: {
|
||||
attemptStatus: 'failed',
|
||||
runStatus: 'failed',
|
||||
errorCode: 'EXECUTION_FAILED',
|
||||
errorSummary: 'Execution completed without success',
|
||||
},
|
||||
cancelled: {
|
||||
attemptStatus: 'cancelled',
|
||||
runStatus: 'cancelled',
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled',
|
||||
},
|
||||
timed_out: {
|
||||
attemptStatus: 'timed_out',
|
||||
runStatus: 'timed_out',
|
||||
errorCode: 'EXECUTION_TIMED_OUT',
|
||||
errorSummary: 'Execution exceeded its configured timeout',
|
||||
},
|
||||
lost: {
|
||||
attemptStatus: 'lost',
|
||||
runStatus: 'lost',
|
||||
errorCode: 'EXECUTION_LOST',
|
||||
errorSummary: 'Execution ownership was lost',
|
||||
},
|
||||
};
|
||||
|
||||
export const MAX_PRIMARY_COMPLETION_RETRIES = 4;
|
||||
const TOKEN_PATTERN = /^[A-Za-z0-9_-]{32,128}$/;
|
||||
|
||||
export type PrimaryCompletionSource =
|
||||
| { kind: 'executor'; executorType: string }
|
||||
| { kind: 'receipt'; token: string };
|
||||
|
||||
export interface PrimaryRunCompletionCommand {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
callbackSequence: number;
|
||||
result: ExecutionResult;
|
||||
source: PrimaryCompletionSource;
|
||||
}
|
||||
|
||||
export interface PrimaryRunCompletionResult {
|
||||
status: 'applied' | 'already_terminal';
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
result: ExecutionResult;
|
||||
}
|
||||
|
||||
export type PrimaryCompletionEventIdFactory = () => string;
|
||||
|
||||
export class PrimaryCompletionNotFoundError extends Error {
|
||||
constructor() {
|
||||
super('Primary completion target was not found');
|
||||
this.name = 'PrimaryCompletionNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryCompletionUnauthorizedError extends Error {
|
||||
constructor() {
|
||||
super('Primary completion source is not authorized');
|
||||
this.name = 'PrimaryCompletionUnauthorizedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryCompletionSequenceError extends Error {
|
||||
constructor() {
|
||||
super('Primary completion callback sequence is invalid');
|
||||
this.name = 'PrimaryCompletionSequenceError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryCompletionStateError extends Error {
|
||||
constructor() {
|
||||
super('Primary completion target state is inconsistent');
|
||||
this.name = 'PrimaryCompletionStateError';
|
||||
}
|
||||
}
|
||||
|
||||
class PrimaryCompletionConcurrentWriteError extends Error {}
|
||||
|
||||
export function hashPrimaryCompletionToken(token: string): string {
|
||||
if (!TOKEN_PATTERN.test(token)) {
|
||||
throw new TypeError('Primary completion token is invalid');
|
||||
}
|
||||
return createHash('sha256').update(token, 'utf8').digest('hex');
|
||||
}
|
||||
|
||||
function validateResult(result: ExecutionResult): void {
|
||||
if (!Object.hasOwn(TERMINAL_MAPPING, result.outcome)) {
|
||||
throw new TypeError('Primary completion outcome is invalid');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(result.startedAtMs) ||
|
||||
result.startedAtMs < 0 ||
|
||||
!Number.isSafeInteger(result.finishedAtMs) ||
|
||||
result.finishedAtMs < result.startedAtMs
|
||||
) {
|
||||
throw new TypeError('Primary completion timestamps are invalid');
|
||||
}
|
||||
if (
|
||||
result.exitCode !== undefined &&
|
||||
(!Number.isInteger(result.exitCode) ||
|
||||
result.exitCode < 0 ||
|
||||
result.exitCode > 255)
|
||||
) {
|
||||
throw new TypeError('Primary completion exitCode is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function mappingFor(run: RunRecord, result: ExecutionResult): TerminalMapping {
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
return run.cancelReason === 'timeout'
|
||||
? TERMINAL_MAPPING.timed_out
|
||||
: TERMINAL_MAPPING.cancelled;
|
||||
}
|
||||
return TERMINAL_MAPPING[result.outcome];
|
||||
}
|
||||
|
||||
function sameTerminalState(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
mapping: TerminalMapping,
|
||||
): boolean {
|
||||
return (
|
||||
attempt.status === mapping.attemptStatus && run.status === mapping.runStatus
|
||||
);
|
||||
}
|
||||
|
||||
function authorize(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
source: PrimaryCompletionSource,
|
||||
): void {
|
||||
if (run.executionOwner !== 'runtime') {
|
||||
throw new PrimaryCompletionUnauthorizedError();
|
||||
}
|
||||
if (source.kind === 'executor') {
|
||||
if (!source.executorType || attempt.executorType !== source.executorType) {
|
||||
throw new PrimaryCompletionUnauthorizedError();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let actualHash: string;
|
||||
try {
|
||||
actualHash = hashPrimaryCompletionToken(source.token);
|
||||
} catch {
|
||||
throw new PrimaryCompletionUnauthorizedError();
|
||||
}
|
||||
const expectedHash = attempt.callbackTokenHash;
|
||||
if (!expectedHash || !/^[a-f0-9]{64}$/.test(expectedHash)) {
|
||||
throw new PrimaryCompletionUnauthorizedError();
|
||||
}
|
||||
const expected = Buffer.from(expectedHash, 'hex');
|
||||
const actual = Buffer.from(actualHash, 'hex');
|
||||
if (!timingSafeEqual(expected, actual)) {
|
||||
throw new PrimaryCompletionUnauthorizedError();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The only terminal completion transaction for both live Executor callbacks
|
||||
* and durable receipt replay. Attempt, Run and both events commit atomically.
|
||||
*/
|
||||
export class PrimaryRunCompletionService {
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly createEventId: PrimaryCompletionEventIdFactory = uuidV7,
|
||||
) {}
|
||||
|
||||
async complete(
|
||||
command: PrimaryRunCompletionCommand,
|
||||
): Promise<PrimaryRunCompletionResult> {
|
||||
if (
|
||||
!Number.isSafeInteger(command.callbackSequence) ||
|
||||
command.callbackSequence < 1
|
||||
) {
|
||||
throw new PrimaryCompletionSequenceError();
|
||||
}
|
||||
validateResult(command.result);
|
||||
|
||||
for (let retry = 0; retry <= MAX_PRIMARY_COMPLETION_RETRIES; retry += 1) {
|
||||
try {
|
||||
return await this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(command.runId);
|
||||
const attempt = await transaction.findAttemptById(command.attemptId);
|
||||
if (!run || !attempt || attempt.runId !== run.id) {
|
||||
throw new PrimaryCompletionNotFoundError();
|
||||
}
|
||||
authorize(run, attempt, command.source);
|
||||
const mapping = mappingFor(run, command.result);
|
||||
|
||||
if (isTerminalRunAttemptStatus(attempt.status)) {
|
||||
if (attempt.callbackSequence !== command.callbackSequence) {
|
||||
throw new PrimaryCompletionSequenceError();
|
||||
}
|
||||
if (!sameTerminalState(run, attempt, mapping)) {
|
||||
throw new PrimaryCompletionStateError();
|
||||
}
|
||||
return {
|
||||
status: 'already_terminal',
|
||||
run,
|
||||
attempt,
|
||||
result: command.result,
|
||||
};
|
||||
}
|
||||
if (command.callbackSequence !== attempt.callbackSequence + 1) {
|
||||
throw new PrimaryCompletionSequenceError();
|
||||
}
|
||||
if (
|
||||
run.status === 'succeeded' ||
|
||||
run.status === 'failed' ||
|
||||
run.status === 'cancelled' ||
|
||||
run.status === 'timed_out'
|
||||
) {
|
||||
throw new PrimaryCompletionStateError();
|
||||
}
|
||||
|
||||
const atMs = Math.max(
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs ?? 0,
|
||||
command.result.finishedAtMs,
|
||||
);
|
||||
const attemptDecision = transitionRunAttempt(run, attempt, {
|
||||
to: mapping.attemptStatus,
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
callbackSequence: command.callbackSequence,
|
||||
...(command.result.exitCode === undefined
|
||||
? {}
|
||||
: { exitCode: command.result.exitCode }),
|
||||
...(mapping.errorCode === undefined
|
||||
? {}
|
||||
: { errorCode: mapping.errorCode }),
|
||||
...(mapping.errorSummary === undefined
|
||||
? {}
|
||||
: { errorSummary: mapping.errorSummary }),
|
||||
});
|
||||
const runDecision = transitionRun(attemptDecision.run, {
|
||||
to: mapping.runStatus,
|
||||
expectedVersion: attemptDecision.run.version,
|
||||
atMs,
|
||||
...(mapping.errorCode === undefined
|
||||
? {}
|
||||
: { errorCode: mapping.errorCode }),
|
||||
...(mapping.errorSummary === undefined
|
||||
? {}
|
||||
: { errorSummary: mapping.errorSummary }),
|
||||
});
|
||||
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
attemptDecision.run,
|
||||
run.version,
|
||||
))
|
||||
) {
|
||||
throw new PrimaryCompletionConcurrentWriteError();
|
||||
}
|
||||
if (
|
||||
!(await transaction.compareAndSetAttempt(attemptDecision.attempt, {
|
||||
status: attempt.status,
|
||||
callbackSequence: attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new PrimaryCompletionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(
|
||||
attemptDecision.run,
|
||||
attemptDecision.event,
|
||||
attempt.id,
|
||||
command.source,
|
||||
`primary-completion:${attempt.id}:${command.callbackSequence}:attempt`,
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
runDecision.run,
|
||||
attemptDecision.run.version,
|
||||
))
|
||||
) {
|
||||
throw new PrimaryCompletionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(
|
||||
runDecision.run,
|
||||
runDecision.event,
|
||||
attempt.id,
|
||||
command.source,
|
||||
`primary-completion:${attempt.id}:${command.callbackSequence}:run`,
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
status: 'applied',
|
||||
run: runDecision.run,
|
||||
attempt: attemptDecision.attempt,
|
||||
result: command.result,
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof PrimaryCompletionConcurrentWriteError) ||
|
||||
retry === MAX_PRIMARY_COMPLETION_RETRIES
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error('Primary completion retry budget was exhausted');
|
||||
}
|
||||
|
||||
private event(
|
||||
run: RunRecord,
|
||||
draft: RunDomainEventDraft,
|
||||
attemptId: string,
|
||||
source: PrimaryCompletionSource,
|
||||
dedupeKey: string,
|
||||
createdAtMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id: this.createEventId(),
|
||||
runId: run.id,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey,
|
||||
actorType: 'executor',
|
||||
actorId:
|
||||
source.kind === 'executor' ? source.executorType : 'completion-receipt',
|
||||
attemptId,
|
||||
payload: draft.payload,
|
||||
createdAtMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type {
|
||||
ExecutionOrigin,
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
reserveRunEvent,
|
||||
transitionRun,
|
||||
type RunDomainEventDraft,
|
||||
} from '../domain/runStateMachine';
|
||||
import { RunVersionConflictError } from '../domain/stateMachineErrors';
|
||||
import type { ExecutorType } from '../domain/execution';
|
||||
import {
|
||||
assertAdmittedRunRetryPolicy,
|
||||
type RunRetryPolicyDefinition,
|
||||
type RunRetryPolicyRecord,
|
||||
} from '../domain/runRetryPolicy';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import type { RunCommandActor } from './runCommandService';
|
||||
|
||||
export interface PrimaryRunDefinition {
|
||||
projectId: string;
|
||||
taskId: string;
|
||||
taskRevision: string;
|
||||
taskName?: string;
|
||||
taskSnapshotRef?: string;
|
||||
legacyCronId?: number;
|
||||
triggerId?: string;
|
||||
triggerType: string;
|
||||
executionOrigin: ExecutionOrigin;
|
||||
triggeredBy?: string;
|
||||
requestId?: string;
|
||||
scheduledForMs?: number;
|
||||
priority?: number;
|
||||
idempotencyKey?: string;
|
||||
inputRef?: string;
|
||||
outputRef?: string;
|
||||
acceptedAtMs: number;
|
||||
actor: RunCommandActor;
|
||||
retryPolicy?: RunRetryPolicyDefinition;
|
||||
}
|
||||
|
||||
export interface PrimaryRunReference {
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
}
|
||||
|
||||
export type PrimaryRunIdFactory = () => string;
|
||||
|
||||
/**
|
||||
* Creates the durable runtime-owned aggregate before an Executor can observe it.
|
||||
*/
|
||||
export class PrimaryRunCreator {
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly createId: PrimaryRunIdFactory = uuidV7,
|
||||
) {}
|
||||
|
||||
async create(
|
||||
definition: PrimaryRunDefinition,
|
||||
executorType: ExecutorType,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const initialRun: RunRecord = {
|
||||
id: this.createId(),
|
||||
projectId: definition.projectId,
|
||||
taskId: definition.taskId,
|
||||
taskRevision: definition.taskRevision,
|
||||
...(definition.taskName === undefined
|
||||
? {}
|
||||
: { taskName: definition.taskName }),
|
||||
...(definition.taskSnapshotRef === undefined
|
||||
? {}
|
||||
: { taskSnapshotRef: definition.taskSnapshotRef }),
|
||||
...(definition.legacyCronId === undefined
|
||||
? {}
|
||||
: { legacyCronId: definition.legacyCronId }),
|
||||
...(definition.triggerId === undefined
|
||||
? {}
|
||||
: { triggerId: definition.triggerId }),
|
||||
triggerType: definition.triggerType,
|
||||
executionOrigin: definition.executionOrigin,
|
||||
executionOwner: 'runtime',
|
||||
...(definition.triggeredBy === undefined
|
||||
? {}
|
||||
: { triggeredBy: definition.triggeredBy }),
|
||||
...(definition.requestId === undefined
|
||||
? {}
|
||||
: { requestId: definition.requestId }),
|
||||
...(definition.scheduledForMs === undefined
|
||||
? {}
|
||||
: { scheduledForMs: definition.scheduledForMs }),
|
||||
status: 'created',
|
||||
version: 0,
|
||||
eventSequence: 0,
|
||||
priority: definition.priority ?? 0,
|
||||
...(definition.idempotencyKey === undefined
|
||||
? {}
|
||||
: { idempotencyKey: definition.idempotencyKey }),
|
||||
...(definition.inputRef === undefined
|
||||
? {}
|
||||
: { inputRef: definition.inputRef }),
|
||||
...(definition.outputRef === undefined
|
||||
? {}
|
||||
: { outputRef: definition.outputRef }),
|
||||
createdAtMs: definition.acceptedAtMs,
|
||||
};
|
||||
const initialAttempt: RunAttemptRecord = {
|
||||
id: this.createId(),
|
||||
runId: initialRun.id,
|
||||
attempt: 1,
|
||||
status: 'claimed',
|
||||
executorType,
|
||||
callbackSequence: 0,
|
||||
createdAtMs: definition.acceptedAtMs,
|
||||
};
|
||||
let retryPolicy: RunRetryPolicyRecord | undefined;
|
||||
if (definition.retryPolicy !== undefined) {
|
||||
assertAdmittedRunRetryPolicy(definition.retryPolicy);
|
||||
retryPolicy = {
|
||||
runId: initialRun.id,
|
||||
...definition.retryPolicy,
|
||||
version: 0,
|
||||
createdAtMs: definition.acceptedAtMs,
|
||||
updatedAtMs: definition.acceptedAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
const run = await this.repository.transaction(async (transaction) => {
|
||||
await transaction.insertRun(initialRun);
|
||||
await transaction.insertAttempt(initialAttempt);
|
||||
if (retryPolicy !== undefined) {
|
||||
await transaction.insertRetryPolicy(retryPolicy);
|
||||
}
|
||||
|
||||
const created = reserveRunEvent(initialRun, 0);
|
||||
const createdUpdated = await transaction.compareAndSetRun(created.run, 0);
|
||||
if (!createdUpdated) {
|
||||
throw new RunVersionConflictError(initialRun.id, 0, initialRun.version);
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(
|
||||
created.run,
|
||||
{
|
||||
sequence: created.sequence,
|
||||
type: 'run.created',
|
||||
payload: {
|
||||
status: 'created',
|
||||
version: created.run.version,
|
||||
execution_owner: 'runtime',
|
||||
},
|
||||
},
|
||||
definition.actor,
|
||||
`primary-run-created:${initialRun.id}`,
|
||||
definition.acceptedAtMs,
|
||||
),
|
||||
);
|
||||
|
||||
const queued = transitionRun(created.run, {
|
||||
to: 'queued',
|
||||
expectedVersion: created.run.version,
|
||||
atMs: definition.acceptedAtMs,
|
||||
});
|
||||
const queuedUpdated = await transaction.compareAndSetRun(
|
||||
queued.run,
|
||||
created.run.version,
|
||||
);
|
||||
if (!queuedUpdated) {
|
||||
throw new RunVersionConflictError(
|
||||
initialRun.id,
|
||||
created.run.version,
|
||||
created.run.version,
|
||||
);
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
this.event(
|
||||
queued.run,
|
||||
queued.event,
|
||||
definition.actor,
|
||||
`primary-run-queued:${initialRun.id}`,
|
||||
definition.acceptedAtMs,
|
||||
),
|
||||
);
|
||||
return queued.run;
|
||||
});
|
||||
|
||||
return { run, attempt: initialAttempt };
|
||||
}
|
||||
|
||||
private event(
|
||||
run: RunRecord,
|
||||
draft: RunDomainEventDraft,
|
||||
actor: RunCommandActor,
|
||||
dedupeKey: string,
|
||||
createdAtMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id: this.createId(),
|
||||
runId: run.id,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey,
|
||||
actorType: actor.type,
|
||||
...(actor.id === undefined ? {} : { actorId: actor.id }),
|
||||
payload: draft.payload,
|
||||
createdAtMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,706 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import type {
|
||||
ExecutionContext,
|
||||
ExecutionHandle,
|
||||
ExecutionResult,
|
||||
ExecutionSpec,
|
||||
ExecutionStopReason,
|
||||
ExecutionStopResult,
|
||||
} from '../domain/execution';
|
||||
import type { RunAttemptRecord, RunRecord } from '../domain/run';
|
||||
import {
|
||||
assertAdmittedRunRetryPolicy,
|
||||
type RunRetryPolicyDefinition,
|
||||
} from '../domain/runRetryPolicy';
|
||||
import {
|
||||
MAX_LOG_ARTIFACT_ID_LENGTH,
|
||||
isTerminalRunAttemptStatus,
|
||||
isTerminalRunStatus,
|
||||
} from '../domain/runStateMachine';
|
||||
import type { Executor } from '../ports/executor';
|
||||
import type { CompletionReceiptJournal } from '../ports/completionReceiptJournal';
|
||||
import type { PrimaryRunIdempotencyLookup } from '../ports/primaryRunIdempotencyLookup';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import type { RunRetryPolicyAdmission } from '../ports/runRetryPolicyAdmission';
|
||||
import { DuplicateIdempotencyKeyError } from '../domain/repositoryErrors';
|
||||
import {
|
||||
PrimaryRunCreator,
|
||||
type PrimaryRunDefinition,
|
||||
type PrimaryRunIdFactory,
|
||||
type PrimaryRunReference,
|
||||
} from './primaryRunCreator';
|
||||
import { RunCommandService } from './runCommandService';
|
||||
import {
|
||||
hashPrimaryCompletionToken,
|
||||
PrimaryRunCompletionService,
|
||||
} from './primaryRunCompletionService';
|
||||
|
||||
export interface PrimaryRunClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface PrimaryRunOrchestratorOptions {
|
||||
clock?: PrimaryRunClock;
|
||||
createId?: PrimaryRunIdFactory;
|
||||
idempotencyLookup?: PrimaryRunIdempotencyLookup;
|
||||
createCallbackToken?: () => string;
|
||||
completionReceiptJournal?: Pick<CompletionReceiptJournal, 'register'>;
|
||||
retryPolicyAdmission?: RunRetryPolicyAdmission;
|
||||
}
|
||||
|
||||
export interface PrimaryRunStartCommand {
|
||||
definition: Omit<PrimaryRunDefinition, 'acceptedAtMs' | 'retryPolicy'> & {
|
||||
acceptedAtMs?: number;
|
||||
};
|
||||
timeoutMs?: number;
|
||||
createSpec(reference: PrimaryRunReference): ExecutionSpec;
|
||||
context: ExecutionContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trusted local-dispatch input for an aggregate that already owns a claimed
|
||||
* Attempt. Callers must materialize the spec from the persisted Task revision.
|
||||
*/
|
||||
export interface PrimaryClaimedRunStartCommand {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
timeoutMs?: number;
|
||||
createSpec(reference: PrimaryRunReference): ExecutionSpec;
|
||||
context: ExecutionContext;
|
||||
logArtifactId?: string;
|
||||
}
|
||||
|
||||
export interface PrimaryRunCompletion {
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
result: ExecutionResult;
|
||||
}
|
||||
|
||||
export interface ActivePrimaryRun extends PrimaryRunReference {
|
||||
handle: ExecutionHandle;
|
||||
completion: Promise<PrimaryRunCompletion>;
|
||||
cancel(reason: ExecutionStopReason): Promise<ExecutionStopResult>;
|
||||
}
|
||||
|
||||
export class PrimaryRunLaunchError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly reference: PrimaryRunReference,
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(message, options);
|
||||
this.name = 'PrimaryRunLaunchError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunNotActiveError extends Error {
|
||||
constructor(readonly runId: string) {
|
||||
super(`Primary Run is not active: ${runId}`);
|
||||
this.name = 'PrimaryRunNotActiveError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunDuplicateRequestError extends Error {
|
||||
constructor(
|
||||
readonly projectId: string,
|
||||
readonly idempotencyKey: string,
|
||||
readonly existingRunId: string,
|
||||
) {
|
||||
super('A Primary Run already exists for this idempotent request');
|
||||
this.name = 'PrimaryRunDuplicateRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunIdempotencyUnavailableError extends Error {
|
||||
constructor() {
|
||||
super('Primary Run idempotency lookup is not configured');
|
||||
this.name = 'PrimaryRunIdempotencyUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunRetryPolicyAuthorityError extends Error {
|
||||
readonly code = 'PRIMARY_RUN_RETRY_POLICY_AUTHORITY_REQUIRED';
|
||||
|
||||
constructor() {
|
||||
super('Run requests cannot self-assert automatic retry safety');
|
||||
this.name = 'PrimaryRunRetryPolicyAuthorityError';
|
||||
}
|
||||
}
|
||||
|
||||
export type PrimaryClaimedRunRejectionReason =
|
||||
| 'run_not_found'
|
||||
| 'attempt_not_found'
|
||||
| 'aggregate_mismatch'
|
||||
| 'not_latest_attempt'
|
||||
| 'not_queued'
|
||||
| 'not_claimed'
|
||||
| 'stale_execution_authority'
|
||||
| 'executor_mismatch'
|
||||
| 'cancellation_requested'
|
||||
| 'already_active';
|
||||
|
||||
export class PrimaryClaimedRunRejectedError extends Error {
|
||||
readonly code = 'PRIMARY_CLAIMED_RUN_REJECTED';
|
||||
|
||||
constructor(readonly reason: PrimaryClaimedRunRejectionReason) {
|
||||
super(`Claimed Primary Run activation was rejected: ${reason}`);
|
||||
this.name = 'PrimaryClaimedRunRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
interface ActiveExecution {
|
||||
handle: ExecutionHandle;
|
||||
completion: Promise<PrimaryRunCompletion>;
|
||||
}
|
||||
|
||||
const ALREADY_EXITED_STOP_RESULT: ExecutionStopResult = {
|
||||
status: 'already_exited',
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
};
|
||||
|
||||
/**
|
||||
* Serializes durable Run transitions around a single Executor side effect.
|
||||
* It deliberately owns no scheduler or HTTP routing policy.
|
||||
*/
|
||||
export class PrimaryRunOrchestrator {
|
||||
private readonly clock: PrimaryRunClock;
|
||||
private readonly creator: PrimaryRunCreator;
|
||||
private readonly commands: RunCommandService;
|
||||
private readonly completions: PrimaryRunCompletionService;
|
||||
private readonly createCallbackToken: () => string;
|
||||
private readonly idempotencyLookup?: PrimaryRunIdempotencyLookup;
|
||||
private readonly completionReceiptJournal?: Pick<
|
||||
CompletionReceiptJournal,
|
||||
'register'
|
||||
>;
|
||||
private readonly retryPolicyAdmission?: RunRetryPolicyAdmission;
|
||||
private readonly active = new Map<string, ActiveExecution>();
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly executor: Executor,
|
||||
options: PrimaryRunOrchestratorOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.creator = new PrimaryRunCreator(repository, options.createId);
|
||||
this.commands = new RunCommandService(repository, options.createId);
|
||||
this.completions = new PrimaryRunCompletionService(
|
||||
repository,
|
||||
options.createId,
|
||||
);
|
||||
this.createCallbackToken =
|
||||
options.createCallbackToken ??
|
||||
(() => randomBytes(32).toString('base64url'));
|
||||
this.idempotencyLookup = options.idempotencyLookup;
|
||||
this.completionReceiptJournal = options.completionReceiptJournal;
|
||||
this.retryPolicyAdmission = options.retryPolicyAdmission;
|
||||
}
|
||||
|
||||
async start(command: PrimaryRunStartCommand): Promise<ActivePrimaryRun> {
|
||||
this.assertTimeout(command.timeoutMs);
|
||||
if (
|
||||
Object.prototype.hasOwnProperty.call(command.definition, 'retryPolicy')
|
||||
) {
|
||||
throw new PrimaryRunRetryPolicyAuthorityError();
|
||||
}
|
||||
const retryPolicy = await this.admitRetryPolicy(command.definition);
|
||||
const acceptedAtMs = command.definition.acceptedAtMs ?? this.clock.now();
|
||||
const reference = await this.createPrimaryRun(
|
||||
{
|
||||
...command.definition,
|
||||
acceptedAtMs,
|
||||
...(retryPolicy === undefined ? {} : { retryPolicy }),
|
||||
},
|
||||
this.executor.type,
|
||||
);
|
||||
return this.activateReference(reference, command);
|
||||
}
|
||||
|
||||
async activateClaimed(
|
||||
command: PrimaryClaimedRunStartCommand,
|
||||
): Promise<ActivePrimaryRun> {
|
||||
this.assertTimeout(command.timeoutMs);
|
||||
this.assertLogArtifactId(command.logArtifactId);
|
||||
const reference = await this.loadClaimedReference(
|
||||
command.runId,
|
||||
command.attemptId,
|
||||
);
|
||||
return this.activateReference(reference, command);
|
||||
}
|
||||
|
||||
private async activateReference(
|
||||
initialReference: PrimaryRunReference,
|
||||
command: Pick<
|
||||
PrimaryClaimedRunStartCommand,
|
||||
'timeoutMs' | 'createSpec' | 'context' | 'logArtifactId'
|
||||
>,
|
||||
): Promise<ActivePrimaryRun> {
|
||||
const callbackToken = this.createCallbackToken();
|
||||
const callbackTokenHash = hashPrimaryCompletionToken(callbackToken);
|
||||
const callbackSequence = initialReference.attempt.callbackSequence + 1;
|
||||
let reference = initialReference;
|
||||
|
||||
reference = await this.prepareForSpawn(
|
||||
reference,
|
||||
command.timeoutMs,
|
||||
callbackTokenHash,
|
||||
command.logArtifactId,
|
||||
);
|
||||
|
||||
let spec: ExecutionSpec;
|
||||
let handle: ExecutionHandle;
|
||||
try {
|
||||
await this.completionReceiptJournal?.register({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
registeredAtMs: reference.attempt.createdAtMs,
|
||||
});
|
||||
spec = command.createSpec(reference);
|
||||
this.assertSpecMatches(reference, spec, command.timeoutMs);
|
||||
handle = await this.executor.start(spec, {
|
||||
...command.context,
|
||||
completionCallback: {
|
||||
token: callbackToken,
|
||||
callbackSequence,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
reference = await this.recordStartFailure(reference);
|
||||
throw new PrimaryRunLaunchError(
|
||||
'Primary Run could not start its Executor',
|
||||
reference,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
this.assertHandleMatches(reference, handle);
|
||||
reference = await this.recordRunning(reference, handle);
|
||||
} catch (error) {
|
||||
void handle.completion.catch(() => undefined);
|
||||
await this.compensateActivationFailure(reference, handle);
|
||||
const latest = await this.loadReference(reference);
|
||||
throw new PrimaryRunLaunchError(
|
||||
'Primary Run could not persist Executor ownership',
|
||||
latest,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
const completion = handle.completion.then(
|
||||
(result) =>
|
||||
this.completions.complete({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
callbackSequence,
|
||||
result,
|
||||
source: { kind: 'executor', executorType: this.executor.type },
|
||||
}),
|
||||
() =>
|
||||
this.completions.complete({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
callbackSequence,
|
||||
result: {
|
||||
outcome: 'lost',
|
||||
startedAtMs: handle.startedAtMs,
|
||||
finishedAtMs: this.atOrAfter(handle.startedAtMs),
|
||||
errorCode: 'EXECUTOR_COMPLETION_REJECTED',
|
||||
errorSummary: 'Executor completion channel rejected',
|
||||
},
|
||||
source: { kind: 'executor', executorType: this.executor.type },
|
||||
}),
|
||||
);
|
||||
this.active.set(reference.run.id, { handle, completion });
|
||||
void completion.then(
|
||||
() => this.deleteActive(reference.run.id, handle),
|
||||
() => this.deleteActive(reference.run.id, handle),
|
||||
);
|
||||
|
||||
return {
|
||||
...reference,
|
||||
handle,
|
||||
completion,
|
||||
cancel: (reason) => this.cancel(reference.run.id, reason),
|
||||
};
|
||||
}
|
||||
|
||||
private async loadClaimedReference(
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const [run, attempt, latestAttempt] = await Promise.all([
|
||||
this.repository.findRunById(runId),
|
||||
this.repository.findAttemptById(attemptId),
|
||||
this.repository.findLatestAttemptByRunId(runId),
|
||||
]);
|
||||
if (!run) throw new PrimaryClaimedRunRejectedError('run_not_found');
|
||||
if (!attempt) {
|
||||
throw new PrimaryClaimedRunRejectedError('attempt_not_found');
|
||||
}
|
||||
if (
|
||||
attempt.runId !== run.id ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
latestAttempt?.runId !== run.id
|
||||
) {
|
||||
throw new PrimaryClaimedRunRejectedError('aggregate_mismatch');
|
||||
}
|
||||
if (latestAttempt.id !== attempt.id) {
|
||||
throw new PrimaryClaimedRunRejectedError('not_latest_attempt');
|
||||
}
|
||||
if (run.status !== 'queued') {
|
||||
throw new PrimaryClaimedRunRejectedError('not_queued');
|
||||
}
|
||||
if (attempt.status !== 'claimed') {
|
||||
throw new PrimaryClaimedRunRejectedError('not_claimed');
|
||||
}
|
||||
if (
|
||||
attempt.callbackSequence !== 0 ||
|
||||
attempt.callbackTokenHash !== undefined ||
|
||||
attempt.workerId !== undefined ||
|
||||
attempt.executorHandle !== undefined ||
|
||||
attempt.pid !== undefined ||
|
||||
attempt.logArtifactId !== undefined ||
|
||||
attempt.leaseToken !== undefined ||
|
||||
attempt.leaseExpiresAtMs !== undefined ||
|
||||
attempt.startedAtMs !== undefined ||
|
||||
attempt.finishedAtMs !== undefined ||
|
||||
attempt.exitCode !== undefined ||
|
||||
attempt.errorCode !== undefined ||
|
||||
attempt.errorSummary !== undefined
|
||||
) {
|
||||
throw new PrimaryClaimedRunRejectedError('stale_execution_authority');
|
||||
}
|
||||
if (attempt.executorType !== this.executor.type) {
|
||||
throw new PrimaryClaimedRunRejectedError('executor_mismatch');
|
||||
}
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
throw new PrimaryClaimedRunRejectedError('cancellation_requested');
|
||||
}
|
||||
if (this.active.has(run.id)) {
|
||||
throw new PrimaryClaimedRunRejectedError('already_active');
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async admitRetryPolicy(
|
||||
definition: PrimaryRunStartCommand['definition'],
|
||||
): Promise<RunRetryPolicyDefinition | undefined> {
|
||||
if (!this.retryPolicyAdmission) return undefined;
|
||||
const admitted = await this.retryPolicyAdmission.resolve(
|
||||
Object.freeze({
|
||||
projectId: definition.projectId,
|
||||
taskId: definition.taskId,
|
||||
taskRevision: definition.taskRevision,
|
||||
triggerType: definition.triggerType,
|
||||
executionOrigin: definition.executionOrigin,
|
||||
}),
|
||||
);
|
||||
if (admitted === undefined) return undefined;
|
||||
const policy: RunRetryPolicyDefinition = {
|
||||
maxAttempts: admitted.maxAttempts,
|
||||
retryOnLost: admitted.retryOnLost,
|
||||
safety: admitted.safety,
|
||||
backoffBaseMs: admitted.backoffBaseMs,
|
||||
backoffMaxMs: admitted.backoffMaxMs,
|
||||
};
|
||||
assertAdmittedRunRetryPolicy(policy);
|
||||
return policy;
|
||||
}
|
||||
|
||||
async cancel(
|
||||
runId: string,
|
||||
reason: ExecutionStopReason,
|
||||
): Promise<ExecutionStopResult> {
|
||||
const active = this.active.get(runId);
|
||||
if (!active) throw new PrimaryRunNotActiveError(runId);
|
||||
const request = await this.commands.requestCancellation({
|
||||
runId,
|
||||
attemptId: active.handle.attemptId,
|
||||
atMs: this.atOrAfter(reason.requestedAtMs, active.handle.startedAtMs),
|
||||
reason: reason.kind,
|
||||
actor: this.cancellationActor(reason),
|
||||
});
|
||||
if (request.status === 'already_terminal') {
|
||||
return ALREADY_EXITED_STOP_RESULT;
|
||||
}
|
||||
return this.executor.stop(active.handle, reason);
|
||||
}
|
||||
|
||||
isActive(runId: string): boolean {
|
||||
return this.active.has(runId);
|
||||
}
|
||||
|
||||
private async createPrimaryRun(
|
||||
definition: PrimaryRunDefinition,
|
||||
executorType: Executor['type'],
|
||||
): Promise<PrimaryRunReference> {
|
||||
const key = definition.idempotencyKey;
|
||||
if (key === undefined) {
|
||||
return this.creator.create(definition, executorType);
|
||||
}
|
||||
if (!this.idempotencyLookup) {
|
||||
throw new PrimaryRunIdempotencyUnavailableError();
|
||||
}
|
||||
|
||||
const existingRunId = await this.idempotencyLookup.findRunId(
|
||||
definition.projectId,
|
||||
key,
|
||||
);
|
||||
if (existingRunId) {
|
||||
throw new PrimaryRunDuplicateRequestError(
|
||||
definition.projectId,
|
||||
key,
|
||||
existingRunId,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return await this.creator.create(definition, executorType);
|
||||
} catch (error) {
|
||||
if (!(error instanceof DuplicateIdempotencyKeyError)) throw error;
|
||||
const racedRunId = await this.idempotencyLookup.findRunId(
|
||||
definition.projectId,
|
||||
key,
|
||||
);
|
||||
if (!racedRunId) throw error;
|
||||
throw new PrimaryRunDuplicateRequestError(
|
||||
definition.projectId,
|
||||
key,
|
||||
racedRunId,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async prepareForSpawn(
|
||||
reference: PrimaryRunReference,
|
||||
timeoutMs: number | undefined,
|
||||
callbackTokenHash: string,
|
||||
logArtifactId?: string,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const startingAtMs = this.atOrAfter(
|
||||
reference.run.createdAtMs,
|
||||
reference.attempt.createdAtMs,
|
||||
);
|
||||
const deadlineAtMs =
|
||||
timeoutMs === undefined ? undefined : startingAtMs + timeoutMs;
|
||||
if (deadlineAtMs !== undefined && !Number.isSafeInteger(deadlineAtMs)) {
|
||||
throw new RangeError('Primary Run deadline exceeds the supported range');
|
||||
}
|
||||
const dispatching = await this.commands.transitionRun({
|
||||
runId: reference.run.id,
|
||||
to: 'dispatching',
|
||||
expectedVersion: reference.run.version,
|
||||
atMs: startingAtMs,
|
||||
actor: { type: 'scheduler' },
|
||||
});
|
||||
const starting = await this.commands.transitionRunAttempt({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
to: 'starting',
|
||||
expectedRunVersion: dispatching.run.version,
|
||||
atMs: startingAtMs,
|
||||
...(deadlineAtMs === undefined ? {} : { deadlineAtMs }),
|
||||
callbackTokenHash,
|
||||
...(logArtifactId === undefined ? {} : { logArtifactId }),
|
||||
actor: { type: 'worker', id: this.executor.type },
|
||||
});
|
||||
return { run: starting.run, attempt: starting.attempt };
|
||||
}
|
||||
|
||||
private assertLogArtifactId(value: string | undefined): void {
|
||||
if (value === undefined) return;
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > MAX_LOG_ARTIFACT_ID_LENGTH ||
|
||||
!/^[A-Za-z0-9._:-]+$/.test(value)
|
||||
) {
|
||||
throw new TypeError('Primary Run logArtifactId is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
private async recordRunning(
|
||||
reference: PrimaryRunReference,
|
||||
handle: ExecutionHandle,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const runningAttempt = await this.commands.transitionRunAttempt({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
to: 'running',
|
||||
expectedRunVersion: reference.run.version,
|
||||
atMs: this.atOrAfter(
|
||||
reference.run.createdAtMs,
|
||||
reference.attempt.createdAtMs,
|
||||
handle.startedAtMs,
|
||||
),
|
||||
executorHandle: handle.durableHandle ?? handle.id,
|
||||
...(handle.pid === undefined ? {} : { pid: handle.pid }),
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
const runningRun = await this.commands.transitionRun({
|
||||
runId: reference.run.id,
|
||||
to: 'running',
|
||||
expectedVersion: runningAttempt.run.version,
|
||||
atMs: this.atOrAfter(
|
||||
runningAttempt.run.createdAtMs,
|
||||
runningAttempt.attempt.startedAtMs,
|
||||
),
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
return { run: runningRun.run, attempt: runningAttempt.attempt };
|
||||
}
|
||||
|
||||
private async recordStartFailure(
|
||||
reference: PrimaryRunReference,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const atMs = this.atOrAfter(
|
||||
reference.run.createdAtMs,
|
||||
reference.attempt.createdAtMs,
|
||||
);
|
||||
const failedAttempt = await this.commands.transitionRunAttempt({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
to: 'failed',
|
||||
expectedRunVersion: reference.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTOR_START_FAILED',
|
||||
errorSummary: 'Executor failed before ownership was established',
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
const failedRun = await this.commands.transitionRun({
|
||||
runId: reference.run.id,
|
||||
to: 'failed',
|
||||
expectedVersion: failedAttempt.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTOR_START_FAILED',
|
||||
errorSummary: 'Executor failed before ownership was established',
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
return { run: failedRun.run, attempt: failedAttempt.attempt };
|
||||
}
|
||||
|
||||
private async compensateActivationFailure(
|
||||
reference: PrimaryRunReference,
|
||||
handle: ExecutionHandle,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.executor.stop(handle, {
|
||||
kind: 'reconcile',
|
||||
requestedAtMs: this.atOrAfter(reference.run.createdAtMs),
|
||||
});
|
||||
} catch {
|
||||
// The durable state below remains lost when process ownership is unknown.
|
||||
}
|
||||
|
||||
try {
|
||||
let latest = await this.loadReference(reference);
|
||||
const atMs = this.atOrAfter(
|
||||
latest.run.createdAtMs,
|
||||
latest.run.startedAtMs,
|
||||
latest.attempt.createdAtMs,
|
||||
latest.attempt.startedAtMs,
|
||||
);
|
||||
if (!isTerminalRunAttemptStatus(latest.attempt.status)) {
|
||||
const attempt = await this.commands.transitionRunAttempt({
|
||||
runId: latest.run.id,
|
||||
attemptId: latest.attempt.id,
|
||||
to: 'lost',
|
||||
expectedRunVersion: latest.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTION_ACTIVATION_PERSISTENCE_FAILED',
|
||||
errorSummary: 'Executor ownership could not be persisted',
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
latest = { run: attempt.run, attempt: attempt.attempt };
|
||||
}
|
||||
if (!isTerminalRunStatus(latest.run.status)) {
|
||||
await this.commands.transitionRun({
|
||||
runId: latest.run.id,
|
||||
to: 'lost',
|
||||
expectedVersion: latest.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTION_ACTIVATION_PERSISTENCE_FAILED',
|
||||
errorSummary: 'Executor ownership could not be persisted',
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Original persistence failure is reported to the caller.
|
||||
}
|
||||
}
|
||||
|
||||
private async loadReference(
|
||||
fallback: PrimaryRunReference,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const [run, attempt] = await Promise.all([
|
||||
this.repository.findRunById(fallback.run.id),
|
||||
this.repository.findAttemptById(fallback.attempt.id),
|
||||
]);
|
||||
return {
|
||||
run: run ?? fallback.run,
|
||||
attempt: attempt ?? fallback.attempt,
|
||||
};
|
||||
}
|
||||
|
||||
private assertSpecMatches(
|
||||
reference: PrimaryRunReference,
|
||||
spec: ExecutionSpec,
|
||||
timeoutMs?: number,
|
||||
): void {
|
||||
if (
|
||||
spec.runId !== reference.run.id ||
|
||||
spec.attemptId !== reference.attempt.id ||
|
||||
spec.projectId !== reference.run.projectId ||
|
||||
spec.taskId !== reference.run.taskId ||
|
||||
spec.taskRevision !== reference.run.taskRevision ||
|
||||
spec.timeoutMs !== timeoutMs
|
||||
) {
|
||||
throw new Error('ExecutionSpec does not match its persisted Primary Run');
|
||||
}
|
||||
}
|
||||
|
||||
private assertTimeout(timeoutMs: number | undefined): void {
|
||||
if (
|
||||
timeoutMs !== undefined &&
|
||||
(!Number.isSafeInteger(timeoutMs) || timeoutMs < 1)
|
||||
) {
|
||||
throw new RangeError('Primary Run timeoutMs must be a positive integer');
|
||||
}
|
||||
}
|
||||
|
||||
private assertHandleMatches(
|
||||
reference: PrimaryRunReference,
|
||||
handle: ExecutionHandle,
|
||||
): void {
|
||||
if (
|
||||
handle.runId !== reference.run.id ||
|
||||
handle.attemptId !== reference.attempt.id ||
|
||||
handle.executorType !== this.executor.type
|
||||
) {
|
||||
throw new Error(
|
||||
'Executor handle does not match its persisted Primary Run',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private atOrAfter(...timestamps: Array<number | undefined>): number {
|
||||
return Math.max(
|
||||
this.clock.now(),
|
||||
...timestamps.filter((value): value is number => value !== undefined),
|
||||
);
|
||||
}
|
||||
|
||||
private cancellationActor(reason: ExecutionStopReason): {
|
||||
type: 'user' | 'reconciler' | 'system';
|
||||
} {
|
||||
if (reason.kind === 'user') return { type: 'user' };
|
||||
if (reason.kind === 'reconcile') return { type: 'reconciler' };
|
||||
return { type: 'system' };
|
||||
}
|
||||
|
||||
private deleteActive(runId: string, handle: ExecutionHandle): void {
|
||||
if (this.active.get(runId)?.handle === handle) this.active.delete(runId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '../domain/run';
|
||||
import {
|
||||
isTerminalRunAttemptStatus,
|
||||
reserveRunEvent,
|
||||
} from '../domain/runStateMachine';
|
||||
import { RunVersionConflictError } from '../domain/stateMachineErrors';
|
||||
import type { PersistedExecutionInspector } from '../ports/persistedExecutionInspector';
|
||||
import type { CompletionReceiptJournal } from '../ports/completionReceiptJournal';
|
||||
import type {
|
||||
PrimaryRunRecoveryCandidate,
|
||||
PrimaryRunRecoveryCursor,
|
||||
PrimaryRunRecoverySource,
|
||||
} from '../ports/primaryRunRecoverySource';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import type { PrimaryCompletionReceiptConsumer } from './primaryCompletionReceiptConsumer';
|
||||
import { RunCommandService } from './runCommandService';
|
||||
|
||||
export interface PrimaryRunStartupReconcilerClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface PrimaryRunStartupReconcilerOptions {
|
||||
clock?: PrimaryRunStartupReconcilerClock;
|
||||
createEventId?: () => string;
|
||||
completionReceipts?: Pick<PrimaryCompletionReceiptConsumer, 'consume'>;
|
||||
completionReceiptJournal?: Pick<CompletionReceiptJournal, 'register'>;
|
||||
receiptPublishGraceMs?: number;
|
||||
wait?: (delayMs: number) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface PrimaryRunStartupReconcileSummary {
|
||||
scanned: number;
|
||||
verifiedRunning: number;
|
||||
recoveredRunning: number;
|
||||
completedFromReceipt: number;
|
||||
quarantinedReceipts: number;
|
||||
publishGraceWaits: number;
|
||||
markedLost: number;
|
||||
skipped: number;
|
||||
ambiguous: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
unsafeAttemptOverflow: boolean;
|
||||
nextCursor?: PrimaryRunRecoveryCursor;
|
||||
}
|
||||
|
||||
type RecoveryLossReason =
|
||||
| 'attempt_missing'
|
||||
| 'attempt_incomplete'
|
||||
| 'handle_missing'
|
||||
| 'handle_invalid'
|
||||
| 'identity_mismatch'
|
||||
| 'identity_pid_mismatch'
|
||||
| 'identity_unsupported'
|
||||
| 'process_exited_unobserved';
|
||||
|
||||
const LOSS_METADATA: Readonly<
|
||||
Record<RecoveryLossReason, { errorCode: string; errorSummary: string }>
|
||||
> = {
|
||||
attempt_missing: {
|
||||
errorCode: 'RECOVERY_ATTEMPT_MISSING',
|
||||
errorSummary: 'Active Run has no recoverable Attempt',
|
||||
},
|
||||
attempt_incomplete: {
|
||||
errorCode: 'RECOVERY_ATTEMPT_INCOMPLETE',
|
||||
errorSummary: 'Attempt did not persist executable ownership',
|
||||
},
|
||||
handle_missing: {
|
||||
errorCode: 'RECOVERY_HANDLE_MISSING',
|
||||
errorSummary: 'Attempt has no durable Executor handle',
|
||||
},
|
||||
handle_invalid: {
|
||||
errorCode: 'RECOVERY_HANDLE_INVALID',
|
||||
errorSummary: 'Attempt durable Executor handle is invalid',
|
||||
},
|
||||
identity_mismatch: {
|
||||
errorCode: 'RECOVERY_IDENTITY_MISMATCH',
|
||||
errorSummary: 'Operating system process identity does not match',
|
||||
},
|
||||
identity_pid_mismatch: {
|
||||
errorCode: 'RECOVERY_IDENTITY_PID_MISMATCH',
|
||||
errorSummary: 'Persisted PID does not match the durable handle',
|
||||
},
|
||||
identity_unsupported: {
|
||||
errorCode: 'RECOVERY_IDENTITY_UNSUPPORTED',
|
||||
errorSummary: 'Process identity cannot be verified on this platform',
|
||||
},
|
||||
process_exited_unobserved: {
|
||||
errorCode: 'RECOVERY_PROCESS_EXITED_UNOBSERVED',
|
||||
errorSummary: 'Process exited without a durable completion result',
|
||||
},
|
||||
};
|
||||
|
||||
/** One bounded startup pass. The caller owns scheduling and pagination. */
|
||||
export class PrimaryRunStartupReconciler {
|
||||
private readonly clock: PrimaryRunStartupReconcilerClock;
|
||||
private readonly createEventId: () => string;
|
||||
private readonly commands: RunCommandService;
|
||||
private readonly completionReceipts?: Pick<
|
||||
PrimaryCompletionReceiptConsumer,
|
||||
'consume'
|
||||
>;
|
||||
private readonly receiptPublishGraceMs: number;
|
||||
private readonly wait: (delayMs: number) => Promise<void>;
|
||||
private readonly completionReceiptJournal?: Pick<
|
||||
CompletionReceiptJournal,
|
||||
'register'
|
||||
>;
|
||||
private readonly inspectors = new Map<
|
||||
PersistedExecutionInspector['executorType'],
|
||||
PersistedExecutionInspector
|
||||
>();
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly source: PrimaryRunRecoverySource,
|
||||
inspectors: readonly PersistedExecutionInspector[],
|
||||
options: PrimaryRunStartupReconcilerOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.createEventId = options.createEventId ?? uuidV7;
|
||||
this.commands = new RunCommandService(repository, this.createEventId);
|
||||
this.completionReceipts = options.completionReceipts;
|
||||
this.completionReceiptJournal = options.completionReceiptJournal;
|
||||
this.receiptPublishGraceMs = options.receiptPublishGraceMs ?? 0;
|
||||
if (
|
||||
!Number.isSafeInteger(this.receiptPublishGraceMs) ||
|
||||
this.receiptPublishGraceMs < 0 ||
|
||||
this.receiptPublishGraceMs > 5_000
|
||||
) {
|
||||
throw new RangeError('receiptPublishGraceMs must be between 0 and 5000');
|
||||
}
|
||||
this.wait =
|
||||
options.wait ??
|
||||
((delayMs) =>
|
||||
new Promise((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
}));
|
||||
for (const inspector of inspectors) {
|
||||
if (this.inspectors.has(inspector.executorType)) {
|
||||
throw new Error(
|
||||
`Duplicate persisted Executor inspector: ${inspector.executorType}`,
|
||||
);
|
||||
}
|
||||
this.inspectors.set(inspector.executorType, inspector);
|
||||
}
|
||||
}
|
||||
|
||||
async reconcileBatch(
|
||||
options: {
|
||||
cursor?: PrimaryRunRecoveryCursor;
|
||||
limit?: number;
|
||||
} = {},
|
||||
): Promise<PrimaryRunStartupReconcileSummary> {
|
||||
const page = await this.source.listCandidates(options);
|
||||
const summary: PrimaryRunStartupReconcileSummary = {
|
||||
scanned: page.candidates.length,
|
||||
verifiedRunning: 0,
|
||||
recoveredRunning: 0,
|
||||
completedFromReceipt: 0,
|
||||
quarantinedReceipts: 0,
|
||||
publishGraceWaits: 0,
|
||||
markedLost: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
unsafeAttemptOverflow: page.unsafeAttemptOverflow,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
if (page.unsafeAttemptOverflow) return summary;
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
try {
|
||||
await this.reconcileCandidate(candidate, summary);
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
private async reconcileCandidate(
|
||||
candidate: PrimaryRunRecoveryCandidate,
|
||||
summary: PrimaryRunStartupReconcileSummary,
|
||||
): Promise<void> {
|
||||
const run = await this.repository.findRunById(candidate.runId);
|
||||
if (
|
||||
!run ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
!['dispatching', 'running'].includes(run.status)
|
||||
) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
}
|
||||
if (candidate.attempts.length > 1) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
}
|
||||
if (candidate.attempts.length === 0) {
|
||||
await this.markLost(run, undefined, 'attempt_missing');
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const attemptReference = candidate.attempts[0];
|
||||
const attempt = await this.repository.findAttemptById(
|
||||
attemptReference.attemptId,
|
||||
);
|
||||
if (
|
||||
!attempt ||
|
||||
attempt.runId !== run.id ||
|
||||
!['claimed', 'starting', 'running'].includes(attempt.status)
|
||||
) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
attempt.executorType === 'local_process' &&
|
||||
this.completionReceiptJournal
|
||||
) {
|
||||
await this.completionReceiptJournal.register({
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
registeredAtMs: attempt.createdAtMs,
|
||||
});
|
||||
}
|
||||
if (await this.consumeCompletionReceipt(attempt, summary)) return;
|
||||
const inspector = this.inspectors.get(attemptReference.executorType);
|
||||
if (!inspector || attempt.executorType !== inspector.executorType) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
}
|
||||
if (attempt.status !== 'running') {
|
||||
await this.markLost(run, attempt, 'attempt_incomplete');
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
}
|
||||
if (!attempt.executorHandle) {
|
||||
await this.markLost(run, attempt, 'handle_missing');
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const inspection = await inspector.inspect(attempt.executorHandle);
|
||||
if (
|
||||
inspection.status !== 'running' ||
|
||||
(inspection.identityPid !== undefined &&
|
||||
inspection.identityPid !== attempt.pid)
|
||||
) {
|
||||
if (await this.consumeCompletionReceipt(attempt, summary)) return;
|
||||
if (
|
||||
inspection.status === 'exited' &&
|
||||
(inspection.identityPid === undefined ||
|
||||
inspection.identityPid === attempt.pid) &&
|
||||
this.receiptPublishGraceMs > 0
|
||||
) {
|
||||
summary.publishGraceWaits += 1;
|
||||
await this.wait(this.receiptPublishGraceMs);
|
||||
if (await this.consumeCompletionReceipt(attempt, summary)) return;
|
||||
}
|
||||
}
|
||||
if (
|
||||
inspection.identityPid !== undefined &&
|
||||
inspection.identityPid !== attempt.pid
|
||||
) {
|
||||
await this.markLost(run, attempt, 'identity_pid_mismatch');
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
}
|
||||
if (inspection.status === 'running') {
|
||||
if (run.status === 'dispatching') {
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
to: 'running',
|
||||
expectedVersion: run.version,
|
||||
atMs: this.atOrAfter(
|
||||
run.createdAtMs,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs,
|
||||
),
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
summary.recoveredRunning += 1;
|
||||
} else {
|
||||
await this.appendVerifiedRunningEvent(run, attempt);
|
||||
summary.verifiedRunning += 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const reason: RecoveryLossReason =
|
||||
inspection.status === 'invalid'
|
||||
? 'handle_invalid'
|
||||
: inspection.status === 'identity_mismatch'
|
||||
? 'identity_mismatch'
|
||||
: inspection.status === 'unsupported'
|
||||
? 'identity_unsupported'
|
||||
: 'process_exited_unobserved';
|
||||
await this.markLost(run, attempt, reason);
|
||||
summary.markedLost += 1;
|
||||
}
|
||||
|
||||
private async consumeCompletionReceipt(
|
||||
attempt: RunAttemptRecord,
|
||||
summary: PrimaryRunStartupReconcileSummary,
|
||||
): Promise<boolean> {
|
||||
if (!this.completionReceipts || attempt.executorType !== 'local_process') {
|
||||
return false;
|
||||
}
|
||||
const result = await this.completionReceipts.consume(attempt.id);
|
||||
if (result.status === 'missing') return false;
|
||||
if (result.status === 'quarantined') {
|
||||
summary.quarantinedReceipts += 1;
|
||||
return false;
|
||||
}
|
||||
summary.completedFromReceipt += 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
private async markLost(
|
||||
initialRun: RunRecord,
|
||||
initialAttempt: RunAttemptRecord | undefined,
|
||||
reason: RecoveryLossReason,
|
||||
): Promise<void> {
|
||||
const metadata = LOSS_METADATA[reason];
|
||||
let run = initialRun;
|
||||
const atMs = this.atOrAfter(
|
||||
run.createdAtMs,
|
||||
run.startedAtMs,
|
||||
initialAttempt?.createdAtMs,
|
||||
initialAttempt?.startedAtMs,
|
||||
);
|
||||
if (initialAttempt && !isTerminalRunAttemptStatus(initialAttempt.status)) {
|
||||
const attempt = await this.commands.transitionRunAttempt({
|
||||
runId: run.id,
|
||||
attemptId: initialAttempt.id,
|
||||
to: 'lost',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
errorCode: metadata.errorCode,
|
||||
errorSummary: metadata.errorSummary,
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
run = attempt.run;
|
||||
}
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
to: 'lost',
|
||||
expectedVersion: run.version,
|
||||
atMs,
|
||||
errorCode: metadata.errorCode,
|
||||
errorSummary: metadata.errorSummary,
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
}
|
||||
|
||||
private async appendVerifiedRunningEvent(
|
||||
expectedRun: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
): Promise<void> {
|
||||
const atMs = this.atOrAfter(
|
||||
expectedRun.createdAtMs,
|
||||
expectedRun.startedAtMs,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs,
|
||||
);
|
||||
await this.repository.transaction(async (transaction) => {
|
||||
const current = await transaction.findRunById(expectedRun.id);
|
||||
if (!current) throw new Error('Primary Run disappeared during recovery');
|
||||
if (current.version !== expectedRun.version) {
|
||||
throw new RunVersionConflictError(
|
||||
current.id,
|
||||
expectedRun.version,
|
||||
current.version,
|
||||
);
|
||||
}
|
||||
const reserved = reserveRunEvent(current, current.version);
|
||||
const updated = await transaction.compareAndSetRun(
|
||||
reserved.run,
|
||||
current.version,
|
||||
);
|
||||
if (!updated) {
|
||||
throw new RunVersionConflictError(
|
||||
current.id,
|
||||
current.version,
|
||||
current.version,
|
||||
);
|
||||
}
|
||||
const event: RunEventRecord = {
|
||||
id: this.createEventId(),
|
||||
runId: current.id,
|
||||
attemptId: attempt.id,
|
||||
sequence: reserved.sequence,
|
||||
type: 'run.reconciled',
|
||||
dedupeKey: `primary-running-reconciled:${attempt.id}:${current.version}`,
|
||||
actorType: 'reconciler',
|
||||
payload: {
|
||||
status: 'running',
|
||||
executor_type: attempt.executorType,
|
||||
evidence: 'durable_handle',
|
||||
version: reserved.run.version,
|
||||
},
|
||||
createdAtMs: atMs,
|
||||
};
|
||||
await transaction.appendEvent(event);
|
||||
});
|
||||
}
|
||||
|
||||
private atOrAfter(...timestamps: Array<number | undefined>): number {
|
||||
return Math.max(
|
||||
this.clock.now(),
|
||||
...timestamps.filter((value): value is number => value !== undefined),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import { MAX_PRIMARY_RECOVERY_BATCH_SIZE } from '../ports/primaryRunRecoverySource';
|
||||
import type { PrimaryRunRecoveryCursor } from '../ports/primaryRunRecoverySource';
|
||||
import type {
|
||||
PrimaryRunStartupReconcileSummary,
|
||||
PrimaryRunStartupReconciler,
|
||||
} from './primaryRunStartupReconciler';
|
||||
|
||||
export const MAX_PRIMARY_RECOVERY_PAGES_PER_STARTUP = 64;
|
||||
|
||||
export type PrimaryRunStartupStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'unsafe_attempt_overflow'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface PrimaryRunStartupSummary
|
||||
extends Omit<
|
||||
PrimaryRunStartupReconcileSummary,
|
||||
'truncated' | 'unsafeAttemptOverflow' | 'nextCursor'
|
||||
> {
|
||||
pages: number;
|
||||
stopReason: PrimaryRunStartupStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: PrimaryRunRecoveryCursor;
|
||||
}
|
||||
|
||||
export interface PrimaryRunStartupOptions {
|
||||
cursor?: PrimaryRunRecoveryCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: PrimaryRunRecoveryCursor | undefined,
|
||||
right: PrimaryRunRecoveryCursor,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.createdAtMs === right.createdAtMs &&
|
||||
left.runId === right.runId
|
||||
);
|
||||
}
|
||||
|
||||
/** Runs a complete but bounded startup reconciliation before Primary activates. */
|
||||
export class PrimaryRunStartupSupervisor {
|
||||
constructor(
|
||||
private readonly reconciler: Pick<
|
||||
PrimaryRunStartupReconciler,
|
||||
'reconcileBatch'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async run(
|
||||
options: PrimaryRunStartupOptions = {},
|
||||
): Promise<PrimaryRunStartupSummary> {
|
||||
const pageSize = options.pageSize ?? 32;
|
||||
const maxPages = options.maxPages ?? 4;
|
||||
if (
|
||||
!Number.isSafeInteger(pageSize) ||
|
||||
pageSize < 1 ||
|
||||
pageSize > MAX_PRIMARY_RECOVERY_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'pageSize must be between 1 and MAX_PRIMARY_RECOVERY_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxPages) ||
|
||||
maxPages < 1 ||
|
||||
maxPages > MAX_PRIMARY_RECOVERY_PAGES_PER_STARTUP
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_PRIMARY_RECOVERY_PAGES_PER_STARTUP',
|
||||
);
|
||||
}
|
||||
|
||||
const total: PrimaryRunStartupSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
verifiedRunning: 0,
|
||||
recoveredRunning: 0,
|
||||
completedFromReceipt: 0,
|
||||
quarantinedReceipts: 0,
|
||||
publishGraceWaits: 0,
|
||||
markedLost: 0,
|
||||
skipped: 0,
|
||||
ambiguous: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
|
||||
for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
|
||||
const page = await this.reconciler.reconcileBatch({
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.verifiedRunning += page.verifiedRunning;
|
||||
total.recoveredRunning += page.recoveredRunning;
|
||||
total.completedFromReceipt += page.completedFromReceipt;
|
||||
total.quarantinedReceipts += page.quarantinedReceipts;
|
||||
total.publishGraceWaits += page.publishGraceWaits;
|
||||
total.markedLost += page.markedLost;
|
||||
total.skipped += page.skipped;
|
||||
total.ambiguous += page.ambiguous;
|
||||
total.failed += page.failed;
|
||||
|
||||
if (page.unsafeAttemptOverflow) {
|
||||
total.stopReason = 'unsafe_attempt_overflow';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
if (!page.truncated) return total;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
if (pageNumber === maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return total;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import type {
|
||||
PrimaryTimeoutSupervisor,
|
||||
PrimaryTimeoutSupervisorOptions,
|
||||
PrimaryTimeoutSupervisorSummary,
|
||||
} from './primaryTimeoutSupervisor';
|
||||
|
||||
export const MIN_TIMEOUT_CYCLE_INTERVAL_MS = 250;
|
||||
export const MAX_TIMEOUT_CYCLE_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_TIMEOUT_INITIAL_DELAY_MS = 24 * 60 * 60 * 1_000;
|
||||
export const MAX_TIMEOUT_STOP_TIMEOUT_MS = 60_000;
|
||||
|
||||
interface ScheduledTimer {
|
||||
unref?: () => void;
|
||||
}
|
||||
|
||||
export interface TimeoutLifecycleScheduler {
|
||||
setTimeout(callback: () => void, delayMs: number): ScheduledTimer;
|
||||
clearTimeout(timer: ScheduledTimer): void;
|
||||
}
|
||||
|
||||
export interface PrimaryTimeoutLifecycleOptions {
|
||||
intervalMs: number;
|
||||
initialDelayMs?: number;
|
||||
stopTimeoutMs?: number;
|
||||
cycle?: Omit<PrimaryTimeoutSupervisorOptions, 'nowMs'>;
|
||||
scheduler?: TimeoutLifecycleScheduler;
|
||||
onCycle?: (summary: PrimaryTimeoutSupervisorSummary) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export type PrimaryTimeoutStopResult = 'drained' | 'timed_out';
|
||||
|
||||
const defaultScheduler: TimeoutLifecycleScheduler = {
|
||||
setTimeout(callback, delayMs) {
|
||||
return setTimeout(callback, delayMs);
|
||||
},
|
||||
clearTimeout(timer) {
|
||||
clearTimeout(timer as ReturnType<typeof setTimeout>);
|
||||
},
|
||||
};
|
||||
|
||||
function assertIntegerBetween(
|
||||
name: string,
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): void {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit, non-overlapping timeout-intent lifecycle. It owns no process
|
||||
* signal path: each cycle only asks the supervisor to persist timeout intent.
|
||||
*/
|
||||
export class PrimaryTimeoutLifecycle {
|
||||
private readonly intervalMs: number;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly stopTimeoutMs: number;
|
||||
private readonly cycleOptions: Omit<PrimaryTimeoutSupervisorOptions, 'nowMs'>;
|
||||
private readonly scheduler: TimeoutLifecycleScheduler;
|
||||
private readonly onCycle?: (summary: PrimaryTimeoutSupervisorSummary) => void;
|
||||
private readonly onError?: (error: unknown) => void;
|
||||
private started = false;
|
||||
private timer?: ScheduledTimer;
|
||||
private inFlight?: Promise<void>;
|
||||
|
||||
constructor(
|
||||
private readonly supervisor: Pick<PrimaryTimeoutSupervisor, 'run'>,
|
||||
options: PrimaryTimeoutLifecycleOptions,
|
||||
) {
|
||||
this.intervalMs = options.intervalMs;
|
||||
this.initialDelayMs = options.initialDelayMs ?? 0;
|
||||
this.stopTimeoutMs = options.stopTimeoutMs ?? 5_000;
|
||||
this.cycleOptions = {
|
||||
...(options.cycle?.cursor === undefined
|
||||
? {}
|
||||
: { cursor: { ...options.cycle.cursor } }),
|
||||
...(options.cycle?.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.cycle.pageSize }),
|
||||
...(options.cycle?.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.cycle.maxPages }),
|
||||
};
|
||||
this.scheduler = options.scheduler ?? defaultScheduler;
|
||||
this.onCycle = options.onCycle;
|
||||
this.onError = options.onError;
|
||||
assertIntegerBetween(
|
||||
'intervalMs',
|
||||
this.intervalMs,
|
||||
MIN_TIMEOUT_CYCLE_INTERVAL_MS,
|
||||
MAX_TIMEOUT_CYCLE_INTERVAL_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'initialDelayMs',
|
||||
this.initialDelayMs,
|
||||
0,
|
||||
MAX_TIMEOUT_INITIAL_DELAY_MS,
|
||||
);
|
||||
assertIntegerBetween(
|
||||
'stopTimeoutMs',
|
||||
this.stopTimeoutMs,
|
||||
1,
|
||||
MAX_TIMEOUT_STOP_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
start(): boolean {
|
||||
if (this.started || this.inFlight) return false;
|
||||
this.started = true;
|
||||
this.schedule(this.initialDelayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<PrimaryTimeoutStopResult> {
|
||||
this.started = false;
|
||||
if (this.timer) {
|
||||
this.scheduler.clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
const inFlight = this.inFlight;
|
||||
if (!inFlight) return 'drained';
|
||||
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const result = await Promise.race<PrimaryTimeoutStopResult>([
|
||||
inFlight.then(() => 'drained' as const),
|
||||
new Promise<'timed_out'>((resolve) => {
|
||||
timeout = setTimeout(() => resolve('timed_out'), this.stopTimeoutMs);
|
||||
}),
|
||||
]);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
return result;
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
if (!this.started || this.timer) return;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
if (this.timer === timer) this.timer = undefined;
|
||||
this.run();
|
||||
}, delayMs);
|
||||
this.timer = timer;
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
private run(): void {
|
||||
if (!this.started || this.inFlight) return;
|
||||
const inFlight = this.supervisor
|
||||
.run(this.cycleOptions)
|
||||
.then((summary) => this.notifyCycle(summary))
|
||||
.catch((error) => this.notifyError(error))
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
if (this.inFlight === inFlight) this.inFlight = undefined;
|
||||
if (this.started) this.schedule(this.intervalMs);
|
||||
});
|
||||
this.inFlight = inFlight;
|
||||
}
|
||||
|
||||
private notifyCycle(summary: PrimaryTimeoutSupervisorSummary): void {
|
||||
try {
|
||||
this.onCycle?.(summary);
|
||||
} catch (error) {
|
||||
this.notifyError(error);
|
||||
}
|
||||
}
|
||||
|
||||
private notifyError(error: unknown): void {
|
||||
try {
|
||||
this.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must never create another scheduler failure loop.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import type { RunCancellationReason } from '../domain/run';
|
||||
import type {
|
||||
PrimaryTimeoutCursor,
|
||||
PrimaryTimeoutSource,
|
||||
} from '../ports/primaryTimeoutSource';
|
||||
import type {
|
||||
RequestRunCancellationCommand,
|
||||
RequestRunCancellationResult,
|
||||
} from './runCommandService';
|
||||
|
||||
export interface PrimaryTimeoutClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface PrimaryTimeoutCommandPort {
|
||||
requestCancellation(
|
||||
command: RequestRunCancellationCommand,
|
||||
): Promise<RequestRunCancellationResult>;
|
||||
}
|
||||
|
||||
export interface PrimaryTimeoutRequestSummary {
|
||||
scanned: number;
|
||||
accepted: number;
|
||||
alreadyRequested: number;
|
||||
alreadyTerminal: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: PrimaryTimeoutCursor;
|
||||
}
|
||||
|
||||
/** One bounded timeout-intent pass. It never calls an Executor or sends signal. */
|
||||
export class PrimaryTimeoutRequester {
|
||||
private readonly clock: PrimaryTimeoutClock;
|
||||
|
||||
constructor(
|
||||
private readonly source: PrimaryTimeoutSource,
|
||||
private readonly commands: PrimaryTimeoutCommandPort,
|
||||
clock: PrimaryTimeoutClock = { now: Date.now },
|
||||
) {
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
async requestBatch(
|
||||
options: {
|
||||
nowMs?: number;
|
||||
cursor?: PrimaryTimeoutCursor;
|
||||
limit?: number;
|
||||
} = {},
|
||||
): Promise<PrimaryTimeoutRequestSummary> {
|
||||
const nowMs = options.nowMs ?? this.clock.now();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('nowMs must be a non-negative safe integer');
|
||||
}
|
||||
const page = await this.source.listOverdue({
|
||||
nowMs,
|
||||
...(options.cursor === undefined ? {} : { cursor: options.cursor }),
|
||||
...(options.limit === undefined ? {} : { limit: options.limit }),
|
||||
});
|
||||
const summary: PrimaryTimeoutRequestSummary = {
|
||||
scanned: page.candidates.length,
|
||||
accepted: 0,
|
||||
alreadyRequested: 0,
|
||||
alreadyTerminal: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
if (candidate.deadlineAtMs > nowMs) {
|
||||
summary.failed += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const result = await this.commands.requestCancellation({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
atMs: nowMs,
|
||||
reason: 'timeout' satisfies RunCancellationReason,
|
||||
actor: { type: 'system', id: 'runtime:timeout' },
|
||||
});
|
||||
if (result.status === 'accepted') summary.accepted += 1;
|
||||
else if (result.status === 'already_requested') {
|
||||
summary.alreadyRequested += 1;
|
||||
} else {
|
||||
summary.alreadyTerminal += 1;
|
||||
}
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import {
|
||||
MAX_PRIMARY_TIMEOUT_BATCH_SIZE,
|
||||
type PrimaryTimeoutCursor,
|
||||
} from '../ports/primaryTimeoutSource';
|
||||
import type {
|
||||
PrimaryTimeoutRequester,
|
||||
PrimaryTimeoutRequestSummary,
|
||||
} from './primaryTimeoutRequester';
|
||||
|
||||
export const MAX_PRIMARY_TIMEOUT_SUPERVISOR_PAGES = 64;
|
||||
|
||||
export type PrimaryTimeoutStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface PrimaryTimeoutSupervisorSummary {
|
||||
pages: number;
|
||||
scanned: number;
|
||||
accepted: number;
|
||||
alreadyRequested: number;
|
||||
alreadyTerminal: number;
|
||||
failed: number;
|
||||
stopReason: PrimaryTimeoutStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: PrimaryTimeoutCursor;
|
||||
}
|
||||
|
||||
export interface PrimaryTimeoutSupervisorOptions {
|
||||
nowMs?: number;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
cursor?: PrimaryTimeoutCursor;
|
||||
}
|
||||
|
||||
export interface PrimaryTimeoutSupervisorClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: PrimaryTimeoutCursor | undefined,
|
||||
right: PrimaryTimeoutCursor | undefined,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
right !== undefined &&
|
||||
left.deadlineAtMs === right.deadlineAtMs &&
|
||||
left.attemptId === right.attemptId
|
||||
);
|
||||
}
|
||||
|
||||
export class PrimaryTimeoutSupervisor {
|
||||
constructor(
|
||||
private readonly requester: Pick<PrimaryTimeoutRequester, 'requestBatch'>,
|
||||
private readonly clock: PrimaryTimeoutSupervisorClock = { now: Date.now },
|
||||
) {}
|
||||
|
||||
async run(
|
||||
options: PrimaryTimeoutSupervisorOptions = {},
|
||||
): Promise<PrimaryTimeoutSupervisorSummary> {
|
||||
const pageSize = options.pageSize ?? 32;
|
||||
const maxPages = options.maxPages ?? 4;
|
||||
const nowMs = options.nowMs ?? this.clock.now();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
||||
throw new RangeError('nowMs must be a non-negative safe integer');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(pageSize) ||
|
||||
pageSize < 1 ||
|
||||
pageSize > MAX_PRIMARY_TIMEOUT_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'pageSize must be between 1 and MAX_PRIMARY_TIMEOUT_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxPages) ||
|
||||
maxPages < 1 ||
|
||||
maxPages > MAX_PRIMARY_TIMEOUT_SUPERVISOR_PAGES
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_PRIMARY_TIMEOUT_SUPERVISOR_PAGES',
|
||||
);
|
||||
}
|
||||
|
||||
const aggregate: PrimaryTimeoutSupervisorSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
accepted: 0,
|
||||
alreadyRequested: 0,
|
||||
alreadyTerminal: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
|
||||
const page: PrimaryTimeoutRequestSummary =
|
||||
await this.requester.requestBatch({
|
||||
nowMs,
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: pageSize,
|
||||
});
|
||||
aggregate.pages += 1;
|
||||
aggregate.scanned += page.scanned;
|
||||
aggregate.accepted += page.accepted;
|
||||
aggregate.alreadyRequested += page.alreadyRequested;
|
||||
aggregate.alreadyTerminal += page.alreadyTerminal;
|
||||
aggregate.failed += page.failed;
|
||||
|
||||
if (!page.truncated) return aggregate;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
aggregate.stopReason = 'cursor_stalled';
|
||||
aggregate.remaining = true;
|
||||
if (page.nextCursor) aggregate.nextCursor = page.nextCursor;
|
||||
return aggregate;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
}
|
||||
aggregate.stopReason = 'page_limit';
|
||||
aggregate.remaining = true;
|
||||
if (cursor) aggregate.nextCursor = cursor;
|
||||
return aggregate;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import {
|
||||
assertAuthenticatedPrincipalActive,
|
||||
normalizeAuthenticatedPrincipal,
|
||||
type AuthenticatedPrincipal,
|
||||
} from '../domain/authenticatedPrincipal';
|
||||
import { assertProjectPolicyProjectId } from '../domain/projectPolicy';
|
||||
import {
|
||||
OWNER_BOOTSTRAP_CHALLENGE_ID_BYTES,
|
||||
OWNER_BOOTSTRAP_DEFAULT_TTL_MS,
|
||||
OWNER_BOOTSTRAP_SYSTEM_SUBJECT,
|
||||
OWNER_BOOTSTRAP_TOKEN_BYTES,
|
||||
ProjectOwnerBootstrapUnauthorizedError,
|
||||
assertProjectOwnerBootstrapChallengeId,
|
||||
assertProjectOwnerBootstrapToken,
|
||||
assertProjectOwnerBootstrapTtl,
|
||||
digestProjectOwnerBootstrapToken,
|
||||
} from '../domain/projectOwnerBootstrap';
|
||||
import type { ProjectOwnerBootstrapRepository } from '../ports/projectOwnerBootstrapRepository';
|
||||
|
||||
export interface IssueProjectOwnerBootstrapRequest {
|
||||
projectId: string;
|
||||
issuer: AuthenticatedPrincipal;
|
||||
nowMs: number;
|
||||
ttlMs?: number;
|
||||
}
|
||||
|
||||
export interface IssuedProjectOwnerBootstrapChallenge {
|
||||
projectId: string;
|
||||
challengeId: string;
|
||||
token: string;
|
||||
expiresAtMs: number;
|
||||
}
|
||||
|
||||
export interface ClaimProjectOwnerBootstrapRequest {
|
||||
projectId: string;
|
||||
challengeId: string;
|
||||
token: string;
|
||||
principal: AuthenticatedPrincipal;
|
||||
nowMs: number;
|
||||
}
|
||||
|
||||
export interface ProjectOwnerBootstrapRandomSource {
|
||||
bytes(size: number): Uint8Array;
|
||||
}
|
||||
|
||||
const CRYPTO_RANDOM_SOURCE: ProjectOwnerBootstrapRandomSource = {
|
||||
bytes: randomBytes,
|
||||
};
|
||||
|
||||
function encodeRandom(
|
||||
source: ProjectOwnerBootstrapRandomSource,
|
||||
size: number,
|
||||
): string {
|
||||
const bytes = source.bytes(size);
|
||||
if (!(bytes instanceof Uint8Array) || bytes.byteLength !== size) {
|
||||
throw new TypeError('Project owner bootstrap random source is invalid');
|
||||
}
|
||||
try {
|
||||
return Buffer.from(bytes).toString('base64url');
|
||||
} finally {
|
||||
bytes.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
function assertExactRequestKeys(
|
||||
value: object,
|
||||
expected: readonly string[],
|
||||
): void {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
keys.length !== canonical.length ||
|
||||
keys.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
throw new TypeError('Project owner bootstrap request shape is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
export class ProjectOwnerBootstrapService {
|
||||
constructor(
|
||||
private readonly repository: ProjectOwnerBootstrapRepository,
|
||||
private readonly randomSource: ProjectOwnerBootstrapRandomSource = CRYPTO_RANDOM_SOURCE,
|
||||
) {}
|
||||
|
||||
async issue(
|
||||
request: IssueProjectOwnerBootstrapRequest,
|
||||
): Promise<Readonly<IssuedProjectOwnerBootstrapChallenge>> {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError(
|
||||
'Project owner bootstrap issue request must be an object',
|
||||
);
|
||||
}
|
||||
assertExactRequestKeys(
|
||||
request,
|
||||
request.ttlMs === undefined
|
||||
? ['projectId', 'issuer', 'nowMs']
|
||||
: ['projectId', 'issuer', 'nowMs', 'ttlMs'],
|
||||
);
|
||||
assertProjectPolicyProjectId(request.projectId);
|
||||
const issuer = normalizeAuthenticatedPrincipal(request.issuer);
|
||||
assertAuthenticatedPrincipalActive(issuer, request.nowMs);
|
||||
if (
|
||||
issuer.subject.type !== OWNER_BOOTSTRAP_SYSTEM_SUBJECT.type ||
|
||||
issuer.subject.id !== OWNER_BOOTSTRAP_SYSTEM_SUBJECT.id ||
|
||||
issuer.assurance !== 'local_console'
|
||||
) {
|
||||
throw new ProjectOwnerBootstrapUnauthorizedError();
|
||||
}
|
||||
const ttlMs = request.ttlMs ?? OWNER_BOOTSTRAP_DEFAULT_TTL_MS;
|
||||
assertProjectOwnerBootstrapTtl(ttlMs);
|
||||
const expiresAtMs = request.nowMs + ttlMs;
|
||||
if (!Number.isSafeInteger(expiresAtMs)) {
|
||||
throw new TypeError('Project owner bootstrap expiry is invalid');
|
||||
}
|
||||
const challengeId = encodeRandom(
|
||||
this.randomSource,
|
||||
OWNER_BOOTSTRAP_CHALLENGE_ID_BYTES,
|
||||
);
|
||||
const token = encodeRandom(this.randomSource, OWNER_BOOTSTRAP_TOKEN_BYTES);
|
||||
assertProjectOwnerBootstrapChallengeId(challengeId);
|
||||
assertProjectOwnerBootstrapToken(token);
|
||||
const tokenDigest = digestProjectOwnerBootstrapToken(
|
||||
request.projectId,
|
||||
challengeId,
|
||||
token,
|
||||
);
|
||||
await this.repository.issue({
|
||||
projectId: request.projectId,
|
||||
challengeId,
|
||||
tokenDigest,
|
||||
issuedAtMs: request.nowMs,
|
||||
expiresAtMs,
|
||||
});
|
||||
return Object.freeze({
|
||||
projectId: request.projectId,
|
||||
challengeId,
|
||||
token,
|
||||
expiresAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
async claim(request: ClaimProjectOwnerBootstrapRequest) {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError(
|
||||
'Project owner bootstrap claim request must be an object',
|
||||
);
|
||||
}
|
||||
assertExactRequestKeys(request, [
|
||||
'projectId',
|
||||
'challengeId',
|
||||
'token',
|
||||
'principal',
|
||||
'nowMs',
|
||||
]);
|
||||
assertProjectPolicyProjectId(request.projectId);
|
||||
assertProjectOwnerBootstrapChallengeId(request.challengeId);
|
||||
assertProjectOwnerBootstrapToken(request.token);
|
||||
const principal = normalizeAuthenticatedPrincipal(request.principal);
|
||||
assertAuthenticatedPrincipalActive(principal, request.nowMs);
|
||||
if (principal.subject.type !== 'user') {
|
||||
throw new ProjectOwnerBootstrapUnauthorizedError();
|
||||
}
|
||||
return this.repository.claim({
|
||||
projectId: request.projectId,
|
||||
challengeId: request.challengeId,
|
||||
tokenDigest: digestProjectOwnerBootstrapToken(
|
||||
request.projectId,
|
||||
request.challengeId,
|
||||
request.token,
|
||||
),
|
||||
subject: principal.subject,
|
||||
claimedAtMs: request.nowMs,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import {
|
||||
assertProjectPolicyProjectId,
|
||||
normalizePolicySubject,
|
||||
normalizeProjectPermission,
|
||||
normalizeProjectPolicySnapshot,
|
||||
ProjectPolicyUnavailableError,
|
||||
type ProjectPermission,
|
||||
type ProjectPolicyDecision,
|
||||
type ProjectPolicyDecisionWithFence,
|
||||
type ProjectPolicyRequest,
|
||||
type ProjectRole,
|
||||
type StaticProjectPermission,
|
||||
} from '../domain/projectPolicy';
|
||||
import type { ProjectPolicyRepository } from '../ports/projectPolicyRepository';
|
||||
|
||||
const READ_ONLY_PERMISSIONS = new Set<ProjectPermission>([
|
||||
'project.read',
|
||||
'task.read',
|
||||
'run.read',
|
||||
'artifact.read',
|
||||
]);
|
||||
|
||||
const OPERATOR_PERMISSIONS = new Set<ProjectPermission>([
|
||||
...READ_ONLY_PERMISSIONS,
|
||||
'task.create',
|
||||
'task.update',
|
||||
'run.start',
|
||||
'run.stop',
|
||||
'run.retry',
|
||||
'secret.use',
|
||||
]);
|
||||
|
||||
const ADMIN_EXCLUDED_PERMISSIONS = new Set<StaticProjectPermission>([
|
||||
'project.manage',
|
||||
]);
|
||||
|
||||
const AGENT_APPROVAL_PERMISSIONS = new Set<ProjectPermission>([
|
||||
'project.manage',
|
||||
'task.create',
|
||||
'task.update',
|
||||
'task.delete',
|
||||
'run.start',
|
||||
'run.stop',
|
||||
'run.retry',
|
||||
'secret.use',
|
||||
'secret.manage',
|
||||
'worker.manage',
|
||||
'policy.manage',
|
||||
'approval.decide',
|
||||
]);
|
||||
|
||||
function decision(
|
||||
effect: ProjectPolicyDecision['effect'],
|
||||
reason: string,
|
||||
): Readonly<ProjectPolicyDecision> {
|
||||
return Object.freeze({ effect, reasons: Object.freeze([reason]) });
|
||||
}
|
||||
|
||||
function roleAllows(role: ProjectRole, permission: ProjectPermission): boolean {
|
||||
if (role === 'owner') return true;
|
||||
if (role === 'admin') {
|
||||
return (
|
||||
permission.startsWith('tool.call:') ||
|
||||
!ADMIN_EXCLUDED_PERMISSIONS.has(permission as StaticProjectPermission)
|
||||
);
|
||||
}
|
||||
if (role === 'operator') {
|
||||
return (
|
||||
permission.startsWith('tool.call:') ||
|
||||
OPERATOR_PERMISSIONS.has(permission)
|
||||
);
|
||||
}
|
||||
return READ_ONLY_PERMISSIONS.has(permission);
|
||||
}
|
||||
|
||||
function agentRequiresApproval(permission: ProjectPermission): boolean {
|
||||
return (
|
||||
permission.startsWith('tool.call:') ||
|
||||
AGENT_APPROVAL_PERMISSIONS.has(permission)
|
||||
);
|
||||
}
|
||||
|
||||
export class ProjectPolicyEngine {
|
||||
constructor(private readonly repository: ProjectPolicyRepository) {}
|
||||
|
||||
async decide(
|
||||
request: ProjectPolicyRequest,
|
||||
): Promise<Readonly<ProjectPolicyDecision>> {
|
||||
return (await this.decideWithFence(request)).decision;
|
||||
}
|
||||
|
||||
async decideWithFence(
|
||||
request: ProjectPolicyRequest,
|
||||
): Promise<Readonly<ProjectPolicyDecisionWithFence>> {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError('Project policy request must be an object');
|
||||
}
|
||||
const requestKeys = Object.keys(request).sort();
|
||||
if (
|
||||
requestKeys.length !== 3 ||
|
||||
requestKeys[0] !== 'permission' ||
|
||||
requestKeys[1] !== 'projectId' ||
|
||||
requestKeys[2] !== 'subject'
|
||||
) {
|
||||
throw new TypeError('Project policy request shape is invalid');
|
||||
}
|
||||
const subject = normalizePolicySubject(request.subject);
|
||||
assertProjectPolicyProjectId(request.projectId);
|
||||
const permission = normalizeProjectPermission(request.permission);
|
||||
let resolved;
|
||||
try {
|
||||
resolved = await this.repository.resolve(request.projectId, subject);
|
||||
} catch {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
if (!resolved) {
|
||||
return Object.freeze({
|
||||
decision: decision('deny', 'project_not_found'),
|
||||
fence: null,
|
||||
});
|
||||
}
|
||||
let snapshot;
|
||||
try {
|
||||
snapshot = normalizeProjectPolicySnapshot(resolved);
|
||||
} catch {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
if (snapshot.project.id !== request.projectId) {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
const fence = Object.freeze({
|
||||
projectVersion: snapshot.project.version,
|
||||
bindingVersion: snapshot.binding?.version ?? null,
|
||||
});
|
||||
if (!snapshot.binding || snapshot.binding.state === 'revoked') {
|
||||
return Object.freeze({
|
||||
decision: decision('deny', 'subject_unbound'),
|
||||
fence,
|
||||
});
|
||||
}
|
||||
if (
|
||||
snapshot.binding.subject.type !== subject.type ||
|
||||
snapshot.binding.subject.id !== subject.id
|
||||
) {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
if (
|
||||
snapshot.project.status === 'archived' &&
|
||||
!READ_ONLY_PERMISSIONS.has(permission)
|
||||
) {
|
||||
return Object.freeze({
|
||||
decision: decision('deny', 'project_archived'),
|
||||
fence,
|
||||
});
|
||||
}
|
||||
if (!roleAllows(snapshot.binding.role!, permission)) {
|
||||
return Object.freeze({
|
||||
decision: decision('deny', 'permission_missing'),
|
||||
fence,
|
||||
});
|
||||
}
|
||||
if (subject.type === 'agent' && agentRequiresApproval(permission)) {
|
||||
return Object.freeze({
|
||||
decision: decision(
|
||||
'require_approval',
|
||||
'agent_action_requires_approval',
|
||||
),
|
||||
fence,
|
||||
});
|
||||
}
|
||||
return Object.freeze({
|
||||
decision: decision('allow', 'role_grant'),
|
||||
fence,
|
||||
});
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user