feat(ql3): establish 3.0 incubation baseline

This commit is contained in:
whyour
2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
@@ -0,0 +1,110 @@
import { createHash, timingSafeEqual } from 'crypto';
import type {
LegacyPanelAuthSnapshot,
LegacyPanelAuthSnapshotReader,
LegacyPanelPlatform,
LegacyPanelSessionSource,
} from '../../ports/legacyPanelSessionSource';
export const MAX_LEGACY_PANEL_TOKEN_LENGTH = 4096;
export const MAX_LEGACY_PANEL_TOKENS_PER_PLATFORM = 64;
export class LegacyPanelSessionUnavailableError extends Error {
readonly code = 'LEGACY_PANEL_SESSION_UNAVAILABLE';
constructor() {
super('Legacy panel session state is unavailable');
this.name = 'LegacyPanelSessionUnavailableError';
}
}
function assertToken(value: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > MAX_LEGACY_PANEL_TOKEN_LENGTH
) {
throw new LegacyPanelSessionUnavailableError();
}
}
function tokenMatches(left: string, right: string): boolean {
assertToken(left);
assertToken(right);
return timingSafeEqual(
createHash('sha256').update(left, 'utf8').digest(),
createHash('sha256').update(right, 'utf8').digest(),
);
}
function candidates(
snapshot: Readonly<LegacyPanelAuthSnapshot>,
platform: LegacyPanelPlatform,
): string[] {
const result: string[] = [];
if (snapshot.token !== undefined && snapshot.token !== '') {
assertToken(snapshot.token);
result.push(snapshot.token);
}
if (snapshot.tokens === undefined) return result;
if (
!snapshot.tokens ||
typeof snapshot.tokens !== 'object' ||
Array.isArray(snapshot.tokens)
) {
throw new LegacyPanelSessionUnavailableError();
}
const platformTokens = snapshot.tokens[platform];
if (platformTokens === null || platformTokens === undefined) return result;
if (typeof platformTokens === 'string') {
if (platformTokens === '') return result;
assertToken(platformTokens);
result.push(platformTokens);
return result;
}
if (
!Array.isArray(platformTokens) ||
platformTokens.length > MAX_LEGACY_PANEL_TOKENS_PER_PLATFORM
) {
throw new LegacyPanelSessionUnavailableError();
}
for (const item of platformTokens) {
if (!item || typeof item !== 'object' || Array.isArray(item)) {
throw new LegacyPanelSessionUnavailableError();
}
assertToken(item.value);
result.push(item.value);
}
return result;
}
export class LegacyAuthInfoSessionSource implements LegacyPanelSessionSource {
constructor(private readonly read: LegacyPanelAuthSnapshotReader) {
if (typeof read !== 'function') {
throw new TypeError('Legacy panel auth snapshot reader is invalid');
}
}
async isActive(
token: string,
platform: LegacyPanelPlatform,
): Promise<boolean> {
assertToken(token);
if (platform !== 'desktop' && platform !== 'mobile') {
throw new TypeError('Legacy panel platform is invalid');
}
let snapshot: Readonly<LegacyPanelAuthSnapshot> | null;
try {
snapshot = await this.read();
} catch {
throw new LegacyPanelSessionUnavailableError();
}
if (!snapshot) return false;
if (typeof snapshot !== 'object' || Array.isArray(snapshot)) {
throw new LegacyPanelSessionUnavailableError();
}
return candidates(snapshot, platform).some((candidate) =>
tokenMatches(candidate, token),
);
}
}
@@ -0,0 +1,94 @@
import { createCipheriv, createDecipheriv, randomBytes } from 'crypto';
import { TextDecoder } from 'util';
import {
LOCAL_SECRET_ALGORITHM,
LocalSecretUnavailableError,
localSecretBinary,
localSecretEnvelopeAad,
normalizeLocalSecretEnvelope,
type LocalSecretEnvelope,
} from '../../domain/localSecret';
const SECRET_KEY_BYTES = 32;
const SECRET_NONCE_BYTES = 12;
const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
export type LocalSecretNonceFactory = () => Uint8Array;
function ownedSecretKey(key: Uint8Array): Buffer {
if (!(key instanceof Uint8Array) || key.byteLength !== SECRET_KEY_BYTES) {
throw new LocalSecretUnavailableError();
}
return Buffer.from(key);
}
export function encryptLocalSecretEnvelope(
metadata: Omit<LocalSecretEnvelope, 'nonce' | 'ciphertext' | 'authTag'>,
plaintext: string,
key: Uint8Array,
nonceFactory: LocalSecretNonceFactory = () => randomBytes(SECRET_NONCE_BYTES),
): LocalSecretEnvelope {
const ownedKey = ownedSecretKey(key);
const plaintextBuffer = Buffer.from(plaintext, 'utf8');
let nonce: Buffer | undefined;
try {
nonce = Buffer.from(nonceFactory());
if (nonce.length !== SECRET_NONCE_BYTES) {
throw new LocalSecretUnavailableError();
}
const cipher = createCipheriv(LOCAL_SECRET_ALGORITHM, ownedKey, nonce, {
authTagLength: 16,
});
cipher.setAAD(localSecretEnvelopeAad(metadata));
const ciphertext = Buffer.concat([
cipher.update(plaintextBuffer),
cipher.final(),
]);
return normalizeLocalSecretEnvelope({
...metadata,
nonce: nonce.toString('base64url'),
ciphertext: ciphertext.toString('base64url'),
authTag: cipher.getAuthTag().toString('base64url'),
});
} catch (error) {
if (error instanceof LocalSecretUnavailableError) throw error;
throw new LocalSecretUnavailableError();
} finally {
ownedKey.fill(0);
plaintextBuffer.fill(0);
nonce?.fill(0);
}
}
export function decryptLocalSecretEnvelopeToBuffer(
envelope: LocalSecretEnvelope,
key: Uint8Array,
): Buffer {
const normalized = normalizeLocalSecretEnvelope(envelope);
const ownedKey = ownedSecretKey(key);
const nonce = localSecretBinary('nonce', normalized.nonce);
const ciphertext = localSecretBinary('ciphertext', normalized.ciphertext);
const authTag = localSecretBinary('authTag', normalized.authTag);
try {
const decipher = createDecipheriv(LOCAL_SECRET_ALGORITHM, ownedKey, nonce, {
authTagLength: 16,
});
decipher.setAAD(localSecretEnvelopeAad(normalized));
decipher.setAuthTag(authTag);
return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
} catch {
throw new LocalSecretUnavailableError();
} finally {
ownedKey.fill(0);
nonce.fill(0);
authTag.fill(0);
}
}
export function decodeLocalSecretPlaintext(plaintext: Buffer): string {
try {
return UTF8_DECODER.decode(plaintext);
} catch {
throw new LocalSecretUnavailableError();
}
}
@@ -0,0 +1,24 @@
import type { CompletionReceipt } from '../../domain/completionReceipt';
import { matchesWorkerExecutionCompletionReceiptAuthentication } from '../../domain/workerExecutionCompletionReceiptAuthentication';
import type { WorkerExecutionOfferJournalRecord } from '../../domain/workerExecutionOffer';
import type { WorkerExecutionCompletionReceiptAuthenticator } from '../../ports/workerExecutionCompletionReceiptAuthenticator';
export class Sha256WorkerExecutionCompletionReceiptAuthenticator
implements WorkerExecutionCompletionReceiptAuthenticator
{
authenticate(
receipt: CompletionReceipt,
record: WorkerExecutionOfferJournalRecord,
): boolean {
if (
record.completionReceiptCallbackSequence === undefined ||
record.completionReceiptTokenDigest === undefined
) {
return false;
}
return matchesWorkerExecutionCompletionReceiptAuthentication(receipt, {
callbackSequence: record.completionReceiptCallbackSequence,
tokenDigest: record.completionReceiptTokenDigest,
});
}
}
@@ -0,0 +1,226 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import { randomBytes } from 'crypto';
import {
assertCompletionReceiptId,
InvalidCompletionReceiptError,
MAX_COMPLETION_RECEIPT_BYTES,
parseCompletionReceipt,
serializeCompletionReceipt,
type CompletionReceipt,
} from '../../domain/completionReceipt';
import type { CompletionReceiptStore } from '../../ports/completionReceiptStore';
export class CompletionReceiptAlreadyExistsError extends Error {
constructor(readonly attemptId: string) {
super(`Completion receipt already exists for Attempt ${attemptId}`);
this.name = 'CompletionReceiptAlreadyExistsError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
/**
* A bounded local journal. It does not scan directories or own a lifecycle;
* callers must discover active Attempts from the database.
*/
export class CompletionReceiptFileStore implements CompletionReceiptStore {
constructor(private readonly root: string) {
if (!path.isAbsolute(root)) {
throw new RangeError('Completion receipt root must be absolute');
}
}
async publish(receipt: CompletionReceipt): Promise<void> {
const serialized = serializeCompletionReceipt(receipt);
const directory = this.directory(receipt.attemptId);
const target = this.target(receipt.attemptId);
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
const temporary = path.join(
directory,
`.${receipt.attemptId}.${randomBytes(16).toString('hex')}.tmp`,
);
let handle: fs.FileHandle | undefined;
try {
handle = await fs.open(temporary, 'wx', 0o600);
await handle.writeFile(serialized, 'utf8');
await handle.sync();
await handle.close();
handle = undefined;
// A hard link publishes the fully written inode atomically and, unlike
// plain rename(), never replaces an existing completion fact.
try {
await fs.link(temporary, target);
} catch (error) {
if (isCode(error, 'EEXIST')) {
throw new CompletionReceiptAlreadyExistsError(receipt.attemptId);
}
throw error;
}
await this.bestEffortUnlink(temporary);
await this.bestEffortSyncDirectory(directory);
} catch (error) {
await handle?.close().catch(() => undefined);
await this.bestEffortUnlink(temporary);
throw error;
}
}
async read(attemptId: string): Promise<CompletionReceipt | undefined> {
const target = this.target(attemptId);
let handle: fs.FileHandle;
try {
handle = await fs.open(
target,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
if (isCode(error, 'ELOOP')) {
throw new InvalidCompletionReceiptError(
'Completion receipt must not be a symbolic link',
);
}
throw error;
}
try {
const stat = await handle.stat();
if (!stat.isFile()) {
throw new InvalidCompletionReceiptError(
'Completion receipt must be a regular file',
);
}
const bytes = Buffer.allocUnsafe(MAX_COMPLETION_RECEIPT_BYTES + 1);
const { bytesRead } = await handle.read(bytes, 0, bytes.length, 0);
if (bytesRead > MAX_COMPLETION_RECEIPT_BYTES) {
throw new InvalidCompletionReceiptError(
'Completion receipt exceeds the byte limit',
);
}
const receipt = parseCompletionReceipt(bytes.subarray(0, bytesRead));
if (receipt.attemptId !== attemptId) {
throw new InvalidCompletionReceiptError(
'Completion receipt path and Attempt do not match',
);
}
return receipt;
} finally {
await handle.close();
}
}
async remove(attemptId: string): Promise<boolean> {
const target = this.target(attemptId);
try {
await fs.unlink(target);
await this.bestEffortSyncDirectory(path.dirname(target));
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
throw error;
}
}
async quarantine(attemptId: string): Promise<string | undefined> {
const target = this.target(attemptId);
const reference = this.quarantineReference(attemptId);
const relativeDirectory = path.dirname(reference);
const directory = path.join(this.root, relativeDirectory);
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
await fs.chmod(directory, 0o700);
const quarantined = path.join(this.root, reference);
try {
await fs.link(target, quarantined);
} catch (error) {
if (isCode(error, 'ENOENT')) {
return (await this.pathExists(quarantined)) ? reference : undefined;
}
if (!isCode(error, 'EEXIST')) throw error;
}
await this.unlinkIfPresent(target);
await this.bestEffortSyncDirectory(path.dirname(target));
await this.bestEffortSyncDirectory(directory);
return reference;
}
quarantineReference(attemptId: string): string {
assertCompletionReceiptId(attemptId, 'attemptId');
return path.posix.join(
'.quarantine',
attemptId.slice(0, 2),
`${attemptId}.json`,
);
}
async purgeQuarantine(attemptId: string): Promise<boolean> {
const target = path.join(this.root, this.quarantineReference(attemptId));
try {
await fs.unlink(target);
await this.bestEffortSyncDirectory(path.dirname(target));
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
throw error;
}
}
private directory(attemptId: string): string {
assertCompletionReceiptId(attemptId, 'attemptId');
return path.join(this.root, attemptId.slice(0, 2));
}
private target(attemptId: string): string {
return path.join(this.directory(attemptId), `${attemptId}.json`);
}
private async bestEffortUnlink(value: string): Promise<void> {
try {
await fs.unlink(value);
} catch (error) {
if (!isCode(error, 'ENOENT')) {
// Temp cleanup is diagnostic-only; the immutable final fact wins.
}
}
}
private async unlinkIfPresent(value: string): Promise<void> {
try {
await fs.unlink(value);
} catch (error) {
if (!isCode(error, 'ENOENT')) throw error;
}
}
private async pathExists(value: string): Promise<boolean> {
try {
await fs.lstat(value);
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
throw error;
}
}
private async bestEffortSyncDirectory(directory: string): Promise<void> {
try {
const handle = await fs.open(directory, constants.O_RDONLY);
try {
await handle.sync();
} finally {
await handle.close();
}
} catch {
// Some supported filesystems cannot fsync directories. The receipt
// remains restart-safe, while power-loss durability is best effort.
}
}
}
@@ -0,0 +1,302 @@
import { createHash } from 'crypto';
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import { assertCompletionReceiptId } from '../../domain/completionReceipt';
import type {
CompletionReceiptDirectoryEntry,
CompletionReceiptDirectoryEntryKind,
CompletionReceiptOrphanDirectory,
CompletionReceiptOrphanQuarantineResult,
CompletionReceiptShardSnapshot,
} from '../../ports/completionReceiptOrphanMaintenance';
const SHARD_PATTERN = /^[0-9a-f]{2}$/;
const TEMPORARY_PATTERN = /^\.([0-9a-f-]{36})\.[0-9a-f]{32}\.tmp$/;
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
function receiptAttemptId(name: string): string | undefined {
if (!name.endsWith('.json')) return undefined;
const attemptId = name.slice(0, -'.json'.length);
try {
assertCompletionReceiptId(attemptId, 'attemptId');
return attemptId;
} catch {
return undefined;
}
}
function temporaryAttemptId(name: string): string | undefined {
const match = TEMPORARY_PATTERN.exec(name);
if (!match) return undefined;
try {
assertCompletionReceiptId(match[1], 'attemptId');
return match[1];
} catch {
return undefined;
}
}
function entryKind(
shard: string,
name: string,
regularFile: boolean,
): { kind: CompletionReceiptDirectoryEntryKind; attemptId?: string } {
if (!regularFile) return { kind: 'unsafe' };
const finalAttemptId = receiptAttemptId(name);
if (finalAttemptId && finalAttemptId.startsWith(shard)) {
return { kind: 'receipt', attemptId: finalAttemptId };
}
const tempAttemptId = temporaryAttemptId(name);
if (tempAttemptId && tempAttemptId.startsWith(shard)) {
return { kind: 'temporary', attemptId: tempAttemptId };
}
return { kind: 'unknown' };
}
function filesystemIdentity(
stat: Awaited<ReturnType<typeof fs.lstat>>,
): string {
return [stat.dev, stat.ino, stat.size, stat.mtimeMs].join(':');
}
export class CompletionReceiptOrphanFileDirectory
implements CompletionReceiptOrphanDirectory
{
constructor(private readonly root: string) {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new RangeError(
'Completion receipt orphan root must be an absolute path containing no NUL',
);
}
if (path.resolve(root) === path.parse(path.resolve(root)).root) {
throw new RangeError(
'Completion receipt orphan root must not be a filesystem root',
);
}
}
async inspectShard(
shard: string,
maxEntries: number,
): Promise<CompletionReceiptShardSnapshot> {
if (!SHARD_PATTERN.test(shard)) {
throw new RangeError(
'Completion receipt shard must be two lowercase hex digits',
);
}
if (
!Number.isSafeInteger(maxEntries) ||
maxEntries < 1 ||
maxEntries > 64
) {
throw new RangeError('maxEntries must be between 1 and 64');
}
const directoryPath = await this.resolveShardDirectory(shard);
if (!directoryPath) return { shard, entries: [], overflow: false };
const directory = await fs.opendir(directoryPath, { bufferSize: 1 });
const entries: CompletionReceiptDirectoryEntry[] = [];
let overflow = false;
try {
for await (const dirent of directory) {
if (entries.length === maxEntries) {
overflow = true;
break;
}
const entryPath = path.join(directoryPath, dirent.name);
let stat: Awaited<ReturnType<typeof fs.lstat>>;
try {
stat = await fs.lstat(entryPath);
} catch (error) {
if (isCode(error, 'ENOENT')) continue;
throw error;
}
const classified = entryKind(shard, dirent.name, stat.isFile());
entries.push({
shard,
name: dirent.name,
kind: classified.kind,
modifiedAtMs: Math.max(0, Math.trunc(stat.mtimeMs)),
sizeBytes: stat.size,
filesystemIdentity: filesystemIdentity(stat),
...(classified.attemptId ? { attemptId: classified.attemptId } : {}),
});
}
} finally {
await directory.close().catch((error) => {
if (!isCode(error, 'ERR_DIR_CLOSED')) throw error;
});
}
return { shard, entries, overflow };
}
async quarantine(
entry: CompletionReceiptDirectoryEntry,
): Promise<CompletionReceiptOrphanQuarantineResult> {
if (
!SHARD_PATTERN.test(entry.shard) ||
path.basename(entry.name) !== entry.name
) {
throw new RangeError('Completion receipt orphan entry path is invalid');
}
if (entry.kind === 'unsafe') return { status: 'changed' };
const shardDirectory = await this.resolveShardDirectory(entry.shard);
if (!shardDirectory) return { status: 'changed' };
const source = path.join(shardDirectory, entry.name);
let current: Awaited<ReturnType<typeof fs.lstat>>;
try {
current = await fs.lstat(source);
} catch (error) {
if (isCode(error, 'ENOENT')) return { status: 'changed' };
throw error;
}
if (
!current.isFile() ||
filesystemIdentity(current) !== entry.filesystemIdentity
) {
return { status: 'changed' };
}
const digest = createHash('sha256')
.update(`${entry.shard}/${entry.name}\0${entry.filesystemIdentity}`)
.digest('hex');
const reference = path.posix.join(
'.orphan-quarantine',
entry.shard,
`${digest}.entry`,
);
const canonicalRoot = path.dirname(shardDirectory);
const directory = await this.ensureQuarantineDirectory(
canonicalRoot,
entry.shard,
);
const target = path.join(directory, `${digest}.entry`);
let linkedByThisCall = false;
try {
await fs.link(source, target);
linkedByThisCall = true;
} catch (error) {
if (!isCode(error, 'EEXIST')) {
if (isCode(error, 'ENOENT')) return { status: 'changed' };
throw error;
}
const targetStat = await fs.lstat(target);
const sourceStat = await fs.lstat(source).catch(() => undefined);
if (
!sourceStat ||
targetStat.dev !== sourceStat.dev ||
targetStat.ino !== sourceStat.ino
) {
return { status: 'changed' };
}
}
let verifiedShardDirectory: string | undefined;
try {
verifiedShardDirectory = await this.resolveShardDirectory(entry.shard);
} catch (error) {
if (linkedByThisCall) await fs.unlink(target).catch(() => undefined);
throw error;
}
const verifiedSource = await fs.lstat(source).catch(() => undefined);
if (
verifiedShardDirectory !== shardDirectory ||
!verifiedSource ||
filesystemIdentity(verifiedSource) !== entry.filesystemIdentity
) {
if (linkedByThisCall) await fs.unlink(target).catch(() => undefined);
return { status: 'changed' };
}
await fs.unlink(source);
await this.bestEffortSync(path.dirname(source));
await this.bestEffortSync(directory);
return { status: 'quarantined', reference };
}
private async resolveShardDirectory(
shard: string,
): Promise<string | undefined> {
let canonicalRoot: string;
try {
canonicalRoot = await fs.realpath(this.root);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
throw error;
}
const candidate = path.join(canonicalRoot, shard);
let stat: Awaited<ReturnType<typeof fs.lstat>>;
try {
stat = await fs.lstat(candidate);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
throw error;
}
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new Error(
`Completion receipt shard ${shard} is not a safe directory`,
);
}
const canonicalDirectory = await fs.realpath(candidate);
if (
path.dirname(canonicalDirectory) !== canonicalRoot ||
path.basename(canonicalDirectory) !== shard
) {
throw new Error(`Completion receipt shard ${shard} escapes its root`);
}
return canonicalDirectory;
}
private async ensureQuarantineDirectory(
canonicalRoot: string,
shard: string,
): Promise<string> {
const quarantineRoot = path.join(canonicalRoot, '.orphan-quarantine');
await this.ensurePrivateDirectory(quarantineRoot, canonicalRoot);
const directory = path.join(quarantineRoot, shard);
await this.ensurePrivateDirectory(directory, quarantineRoot);
return directory;
}
private async ensurePrivateDirectory(
directory: string,
expectedParent: string,
): Promise<void> {
try {
await fs.mkdir(directory, { mode: 0o700 });
} catch (error) {
if (!isCode(error, 'EEXIST')) throw error;
}
const stat = await fs.lstat(directory);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new Error(
'Completion receipt quarantine path is not a safe directory',
);
}
const canonicalDirectory = await fs.realpath(directory);
if (path.dirname(canonicalDirectory) !== expectedParent) {
throw new Error('Completion receipt quarantine path escapes its root');
}
await fs.chmod(directory, 0o700);
}
private async bestEffortSync(directory: string): Promise<void> {
try {
const handle = await fs.open(directory, constants.O_RDONLY);
try {
await handle.sync();
} finally {
await handle.close();
}
} catch {
// Directory fsync is unavailable on some supported filesystems.
}
}
}
@@ -0,0 +1,122 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import { normalizeLocalArtifactReadRange } from '../../domain/artifactRead';
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
import type {
AvailableLocalArtifactByteRange,
LocalArtifactByteRangeReadResult,
LocalArtifactByteRangeReader as LocalArtifactByteRangeReaderPort,
} from '../../ports/localArtifactByteRangeReader';
export class UnsafeLocalArtifactReadTargetError extends Error {
constructor() {
super('Local Artifact read target is unsafe');
this.name = 'UnsafeLocalArtifactReadTargetError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
export class LocalArtifactByteRangeReader
implements LocalArtifactByteRangeReaderPort
{
private readonly root: string;
constructor(root: string) {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new TypeError('Local Artifact read root must be absolute');
}
this.root = path.resolve(root);
}
async read(
logArtifactId: string,
requestedRange: Parameters<LocalArtifactByteRangeReaderPort['read']>[1],
): Promise<LocalArtifactByteRangeReadResult> {
assertLocalExecutionArtifactId(logArtifactId);
const range = normalizeLocalArtifactReadRange(requestedRange);
const directory = path.join(this.root, logArtifactId.slice(6, 8));
await this.assertDirectory(this.root);
if (!(await this.optionalDirectory(directory)))
return { status: 'missing' };
const target = path.join(directory, `${logArtifactId}.log`);
let handle;
try {
handle = await fs.open(
target,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return { status: 'missing' };
throw new UnsafeLocalArtifactReadTargetError();
}
try {
const stat = await handle.stat();
if (!stat.isFile() || !Number.isSafeInteger(stat.size) || stat.size < 0) {
throw new UnsafeLocalArtifactReadTargetError();
}
const start = Math.min(range.offset, stat.size);
const expected = Math.min(range.length, stat.size - start);
const content = Buffer.allocUnsafe(expected);
let read = 0;
while (read < expected) {
const result = await handle.read(
content,
read,
expected - read,
start + read,
);
if (result.bytesRead < 1) {
throw new UnsafeLocalArtifactReadTargetError();
}
read += result.bytesRead;
}
const endExclusive = start + expected;
const result: AvailableLocalArtifactByteRange = {
status: 'available',
content,
start,
endExclusive,
totalBytes: stat.size,
...(endExclusive < stat.size ? { nextOffset: endExclusive } : {}),
};
return Object.freeze(result);
} finally {
await handle.close();
}
}
private async assertDirectory(value: string): Promise<void> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactReadTargetError();
}
} catch (error) {
if (error instanceof UnsafeLocalArtifactReadTargetError) throw error;
throw new UnsafeLocalArtifactReadTargetError();
}
}
private async optionalDirectory(value: string): Promise<boolean> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactReadTargetError();
}
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
if (error instanceof UnsafeLocalArtifactReadTargetError) throw error;
throw new UnsafeLocalArtifactReadTargetError();
}
}
}
@@ -0,0 +1,179 @@
import { constants, type Stats } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import { MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES } from '../../domain/localArtifactTruncation';
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
import type {
LocalArtifactFileRetirementResult,
LocalArtifactFileRetirementStore as LocalArtifactFileRetirementStorePort,
} from '../../ports/localArtifactFileRetirementStore';
import { localArtifactTruncationFactFileName } from './localArtifactTruncationFactStore';
export class UnsafeLocalArtifactRetirementError extends Error {
constructor() {
super('Local Artifact retirement target is unsafe');
this.name = 'UnsafeLocalArtifactRetirementError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
export class LocalArtifactFileRetirementStore
implements LocalArtifactFileRetirementStorePort
{
private readonly root: string;
constructor(root: string) {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new TypeError('Local Artifact retirement root must be absolute');
}
this.root = path.resolve(root);
}
async retire(
logArtifactId: string,
): Promise<LocalArtifactFileRetirementResult> {
assertLocalExecutionArtifactId(logArtifactId);
const directory = path.join(this.root, logArtifactId.slice(6, 8));
await this.assertDirectory(this.root);
if (!(await this.optionalDirectory(directory))) {
return Object.freeze({
disposition: 'already_absent',
bytesReclaimed: 0,
});
}
const target = path.join(directory, `${logArtifactId}.log`);
const fifo = path.join(directory, `.${logArtifactId}.log.fifo`);
const truncation = path.join(
directory,
localArtifactTruncationFactFileName(logArtifactId),
);
const truncationTemporary = path.join(
directory,
`.${logArtifactId}.log.truncated.tmp`,
);
const [targetStat, fifoStat, truncationStat, truncationTemporaryStat] =
await Promise.all([
this.lstat(target),
this.lstat(fifo),
this.lstat(truncation),
this.lstat(truncationTemporary),
]);
if (targetStat && (!targetStat.isFile() || targetStat.isSymbolicLink())) {
throw new UnsafeLocalArtifactRetirementError();
}
if (
targetStat &&
(!Number.isSafeInteger(targetStat.size) || targetStat.size < 0)
) {
throw new UnsafeLocalArtifactRetirementError();
}
if (fifoStat && (!fifoStat.isFIFO() || fifoStat.isSymbolicLink())) {
throw new UnsafeLocalArtifactRetirementError();
}
for (const stat of [truncationStat, truncationTemporaryStat]) {
if (
stat &&
(!stat.isFile() ||
stat.isSymbolicLink() ||
!Number.isSafeInteger(stat.size) ||
stat.size < 0 ||
stat.size > MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES)
) {
throw new UnsafeLocalArtifactRetirementError();
}
}
let targetDeleted = false;
let bytesReclaimed = 0;
if (targetStat) {
try {
await fs.unlink(target);
targetDeleted = true;
bytesReclaimed = targetStat.size;
} catch (error) {
if (!isCode(error, 'ENOENT')) throw error;
}
}
let auxiliaryRemoved = false;
for (const [auxiliary, stat] of [
[fifo, fifoStat],
[truncation, truncationStat],
[truncationTemporary, truncationTemporaryStat],
] as const) {
if (!stat) continue;
try {
await fs.unlink(auxiliary);
auxiliaryRemoved = true;
} catch (error) {
if (!isCode(error, 'ENOENT')) throw error;
}
}
if (targetDeleted || auxiliaryRemoved) {
await this.syncDirectory(directory);
}
if (!targetDeleted) {
return Object.freeze({
disposition: 'already_absent',
bytesReclaimed: 0,
});
}
return Object.freeze({
disposition: 'deleted',
bytesReclaimed,
});
}
private async assertDirectory(value: string): Promise<void> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactRetirementError();
}
} catch (error) {
if (error instanceof UnsafeLocalArtifactRetirementError) throw error;
throw new UnsafeLocalArtifactRetirementError();
}
}
private async optionalDirectory(value: string): Promise<boolean> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactRetirementError();
}
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
if (error instanceof UnsafeLocalArtifactRetirementError) throw error;
throw new UnsafeLocalArtifactRetirementError();
}
}
private async lstat(value: string): Promise<Stats | null> {
try {
return await fs.lstat(value);
} catch (error) {
if (isCode(error, 'ENOENT')) return null;
throw error;
}
}
private async syncDirectory(directory: string): Promise<void> {
const handle = await fs.open(
directory,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
try {
await handle.sync();
} finally {
await handle.close();
}
}
}
@@ -0,0 +1,113 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import {
decodeLocalArtifactTruncationFact,
MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES,
type LocalArtifactTruncationFact,
} from '../../domain/localArtifactTruncation';
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
import type { LocalArtifactTruncationFactStore as LocalArtifactTruncationFactStorePort } from '../../ports/localArtifactTruncationFactStore';
export function localArtifactTruncationFactFileName(
logArtifactId: string,
): string {
assertLocalExecutionArtifactId(logArtifactId);
return `.${logArtifactId}.log.truncated.json`;
}
export class UnsafeLocalArtifactTruncationFactError extends Error {
constructor() {
super('Local Artifact truncation fact target is unsafe');
this.name = 'UnsafeLocalArtifactTruncationFactError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
export class LocalArtifactTruncationFactStore
implements LocalArtifactTruncationFactStorePort
{
private readonly root: string;
constructor(root: string) {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new TypeError('Local Artifact truncation root must be absolute');
}
this.root = path.resolve(root);
}
async read(
logArtifactId: string,
): Promise<Readonly<LocalArtifactTruncationFact> | null> {
assertLocalExecutionArtifactId(logArtifactId);
const directory = path.join(this.root, logArtifactId.slice(6, 8));
await this.assertDirectory(this.root);
if (!(await this.optionalDirectory(directory))) return null;
const target = path.join(
directory,
localArtifactTruncationFactFileName(logArtifactId),
);
let handle;
try {
handle = await fs.open(
target,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return null;
throw new UnsafeLocalArtifactTruncationFactError();
}
try {
const stat = await handle.stat();
if (
!stat.isFile() ||
!Number.isSafeInteger(stat.size) ||
stat.size < 1 ||
stat.size > MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES
) {
throw new UnsafeLocalArtifactTruncationFactError();
}
const fact = decodeLocalArtifactTruncationFact(await handle.readFile());
if (fact.logArtifactId !== logArtifactId) {
throw new UnsafeLocalArtifactTruncationFactError();
}
return fact;
} finally {
await handle.close();
}
}
private async assertDirectory(value: string): Promise<void> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactTruncationFactError();
}
} catch (error) {
if (error instanceof UnsafeLocalArtifactTruncationFactError) throw error;
throw new UnsafeLocalArtifactTruncationFactError();
}
}
private async optionalDirectory(value: string): Promise<boolean> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactTruncationFactError();
}
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
if (error instanceof UnsafeLocalArtifactTruncationFactError) throw error;
throw new UnsafeLocalArtifactTruncationFactError();
}
}
}
@@ -0,0 +1,175 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import type {
ExecutionOutputChunk,
ExecutionOutputSink,
} from '../../domain/execution';
import {
LocalArtifactCapacityUnavailableError,
LocalArtifactQuotaExceededError,
normalizeLocalArtifactCapacityPolicy,
type LocalArtifactCapacityPolicy,
} from '../../domain/localArtifactCapacity';
import {
localExecutionArtifactId,
assertLocalExecutionArtifactId,
} from '../../domain/localExecutionArtifact';
import type { RunDispatchCandidate } from '../../domain/runDispatchCandidate';
import type {
LocalExecutionArtifactAllocator,
PreparedLocalExecutionArtifact,
} from '../../ports/localExecutionArtifactAllocator';
import type { LocalArtifactCapacityProbe } from '../../ports/localArtifactCapacityProbe';
import { LocalFileSystemCapacityProbe } from './localFileSystemCapacityProbe';
import { enableDurableLocalProcessOutput } from '../local-process/durableLocalProcessOutput';
async function privateDirectory(directory: string): Promise<void> {
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
const stat = await fs.lstat(directory);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new TypeError('Local execution artifact directory is unsafe');
}
await fs.chmod(directory, 0o700);
}
class LocalFileExecutionOutput implements ExecutionOutputSink {
private pending: Promise<unknown> = Promise.resolve();
private closed = false;
private remainingBytes: number;
constructor(
private readonly file: fs.FileHandle,
maximumBytes: number,
existingBytes: number,
) {
this.remainingBytes = maximumBytes - existingBytes;
}
write(output: ExecutionOutputChunk): Promise<void> {
if (this.closed) {
return Promise.reject(new Error('Local execution artifact is closed'));
}
const chunk = Buffer.from(output.chunk);
const operation = this.pending.then(async () => {
if (this.remainingBytes <= 0) {
throw new LocalArtifactQuotaExceededError();
}
const accepted = chunk.subarray(
0,
Math.min(chunk.length, this.remainingBytes),
);
let written = 0;
while (written < accepted.length) {
const result = await this.file.write(accepted.subarray(written));
if (result.bytesWritten < 1) {
throw new Error('Local execution artifact write made no progress');
}
written += result.bytesWritten;
this.remainingBytes -= result.bytesWritten;
}
if (accepted.length !== chunk.length) {
throw new LocalArtifactQuotaExceededError();
}
});
this.pending = operation.catch(() => undefined);
return operation;
}
async close(): Promise<void> {
if (this.closed) return;
this.closed = true;
await this.pending.catch(() => undefined);
await this.file.close();
}
}
export class LocalFileExecutionArtifactAllocator
implements LocalExecutionArtifactAllocator
{
private readonly artifactRoot: string;
private readonly completionReceiptRoot: string;
private readonly policy: Readonly<LocalArtifactCapacityPolicy>;
private readonly capacity: LocalArtifactCapacityProbe;
constructor(
artifactRoot: string,
completionReceiptRoot: string,
policy: LocalArtifactCapacityPolicy,
capacity: LocalArtifactCapacityProbe = new LocalFileSystemCapacityProbe(),
) {
if (
!path.isAbsolute(artifactRoot) ||
artifactRoot.includes('\0') ||
!path.isAbsolute(completionReceiptRoot) ||
completionReceiptRoot.includes('\0')
) {
throw new TypeError('Local execution artifact roots must be absolute');
}
this.artifactRoot = path.resolve(artifactRoot);
this.completionReceiptRoot = path.resolve(completionReceiptRoot);
this.policy = normalizeLocalArtifactCapacityPolicy(policy);
this.capacity = capacity;
}
async prepare(
candidate: Readonly<RunDispatchCandidate>,
): Promise<PreparedLocalExecutionArtifact> {
const logArtifactId = localExecutionArtifactId(candidate);
assertLocalExecutionArtifactId(logArtifactId);
const shard = logArtifactId.slice('local-'.length, 'local-'.length + 2);
const directory = path.join(this.artifactRoot, shard);
await privateDirectory(this.artifactRoot);
const capacity = await this.capacity.inspect(this.artifactRoot);
const requiredBytes =
BigInt(this.policy.minimumFreeBytes) +
BigInt(this.policy.maximumAttemptBytes);
if (capacity.availableBytes < requiredBytes) {
throw new LocalArtifactCapacityUnavailableError();
}
await privateDirectory(directory);
await privateDirectory(this.completionReceiptRoot);
const outputFilePath = path.join(directory, `${logArtifactId}.log`);
const file = await fs.open(
outputFilePath,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_APPEND |
(constants.O_NOFOLLOW ?? 0),
0o600,
);
try {
const stat = await file.stat();
if (!stat.isFile()) {
throw new TypeError('Local execution artifact target is unsafe');
}
if (
!Number.isSafeInteger(stat.size) ||
stat.size < 0 ||
stat.size > this.policy.maximumAttemptBytes
) {
throw new LocalArtifactQuotaExceededError();
}
await file.chmod(0o600);
const output = new LocalFileExecutionOutput(
file,
this.policy.maximumAttemptBytes,
stat.size,
);
return {
logArtifactId,
output: enableDurableLocalProcessOutput(output, {
outputFilePath,
completionReceiptRoot: this.completionReceiptRoot,
maximumBytes: this.policy.maximumAttemptBytes,
logArtifactId,
}),
dispose: () => output.close(),
};
} catch (error) {
await file.close().catch(() => undefined);
throw error;
}
}
}
@@ -0,0 +1,68 @@
import fs from 'fs/promises';
import path from 'path';
import { LocalArtifactCapacityUnavailableError } from '../../domain/localArtifactCapacity';
import type {
LocalArtifactCapacityProbe,
LocalArtifactCapacitySnapshot,
LocalArtifactCapacitySource,
} from '../../ports/localArtifactCapacityProbe';
interface BigIntStatFs {
bavail: bigint;
blocks: bigint;
bsize: bigint;
}
export class RootedLocalFileSystemCapacitySource
implements LocalArtifactCapacitySource
{
private readonly root: string;
constructor(
root: string,
private readonly probe: LocalArtifactCapacityProbe = new LocalFileSystemCapacityProbe(),
) {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new TypeError('Local Artifact capacity root must be absolute');
}
this.root = path.resolve(root);
}
inspect(): Promise<LocalArtifactCapacitySnapshot> {
return this.probe.inspect(this.root);
}
}
interface StatFsPromises {
statfs(value: string, options: { bigint: true }): Promise<BigIntStatFs>;
}
export class LocalFileSystemCapacityProbe
implements LocalArtifactCapacityProbe
{
async inspect(root: string): Promise<LocalArtifactCapacitySnapshot> {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new LocalArtifactCapacityUnavailableError();
}
try {
const statfs = (fs as unknown as StatFsPromises).statfs;
if (typeof statfs !== 'function') {
throw new LocalArtifactCapacityUnavailableError();
}
const stat = await statfs.call(fs, root, { bigint: true });
const availableBytes = stat.bavail * stat.bsize;
const totalBytes = stat.blocks * stat.bsize;
if (
availableBytes < BigInt(0) ||
totalBytes < BigInt(1) ||
availableBytes > totalBytes
) {
throw new LocalArtifactCapacityUnavailableError();
}
return Object.freeze({ availableBytes, totalBytes });
} catch (error) {
if (error instanceof LocalArtifactCapacityUnavailableError) throw error;
throw new LocalArtifactCapacityUnavailableError();
}
}
}
@@ -0,0 +1,160 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import {
LocalSecretUnavailableError,
assertLocalSecretKeyId,
} from '../../domain/localSecret';
import type {
LocalSecretKeyMaterial,
LocalSecretKeyProvider,
} from '../../ports/localSecretKeyProvider';
const MAX_KEYRING_BYTES = 16 * 1024;
const MAX_KEY_COUNT = 16;
const BASE64URL_PATTERN = /^[A-Za-z0-9_-]+$/;
interface LocalSecretKeyringManifest {
version: 1;
activeKeyId: string;
keys: Record<string, string>;
}
function exactKeys(value: object, expected: readonly string[]): boolean {
const actual = Object.keys(value).sort();
return (
actual.length === expected.length &&
actual.every((key, index) => key === expected[index])
);
}
function parseManifest(value: Buffer): LocalSecretKeyringManifest {
let parsed: unknown;
try {
parsed = JSON.parse(value.toString('utf8'));
} catch {
throw new LocalSecretUnavailableError();
}
if (
!parsed ||
typeof parsed !== 'object' ||
Array.isArray(parsed) ||
!exactKeys(parsed, ['activeKeyId', 'keys', 'version'])
) {
throw new LocalSecretUnavailableError();
}
const record = parsed as Record<string, unknown>;
if (
record.version !== 1 ||
!record.keys ||
typeof record.keys !== 'object' ||
Array.isArray(record.keys)
) {
throw new LocalSecretUnavailableError();
}
assertLocalSecretKeyId(record.activeKeyId as string);
const entries = Object.entries(record.keys as Record<string, unknown>);
if (entries.length < 1 || entries.length > MAX_KEY_COUNT) {
throw new LocalSecretUnavailableError();
}
const keys: Record<string, string> = Object.create(null);
for (const [keyId, encoded] of entries) {
assertLocalSecretKeyId(keyId);
const decoded =
typeof encoded === 'string'
? Buffer.from(encoded, 'base64url')
: Buffer.alloc(0);
if (
typeof encoded !== 'string' ||
!BASE64URL_PATTERN.test(encoded) ||
decoded.length !== 32 ||
decoded.toString('base64url') !== encoded
) {
decoded.fill(0);
throw new LocalSecretUnavailableError();
}
decoded.fill(0);
keys[keyId] = encoded;
}
if (!keys[record.activeKeyId as string]) {
throw new LocalSecretUnavailableError();
}
return {
version: 1,
activeKeyId: record.activeKeyId as string,
keys,
};
}
export class LocalSecretKeyringFileProvider implements LocalSecretKeyProvider {
private readonly filePath: string;
constructor(filePath: string) {
if (!path.isAbsolute(filePath) || filePath.includes('\0')) {
throw new TypeError('Local Secret keyring path must be absolute');
}
this.filePath = path.resolve(filePath);
}
async active(): Promise<LocalSecretKeyMaterial> {
const manifest = await this.read();
return this.material(
manifest,
manifest.activeKeyId,
) as LocalSecretKeyMaterial;
}
async resolve(keyId: string): Promise<LocalSecretKeyMaterial | null> {
try {
assertLocalSecretKeyId(keyId);
} catch {
throw new LocalSecretUnavailableError();
}
const manifest = await this.read();
return this.material(manifest, keyId);
}
private material(
manifest: LocalSecretKeyringManifest,
keyId: string,
): LocalSecretKeyMaterial | null {
const encoded = manifest.keys[keyId];
return encoded
? Object.freeze({
keyId,
key: Uint8Array.from(Buffer.from(encoded, 'base64url')),
})
: null;
}
private async read(): Promise<LocalSecretKeyringManifest> {
let file: fs.FileHandle | undefined;
let contents: Buffer | undefined;
try {
file = await fs.open(
this.filePath,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
const stat = await file.stat();
if (
!stat.isFile() ||
stat.isSymbolicLink() ||
(stat.mode & 0o077) !== 0 ||
stat.size < 1 ||
stat.size > MAX_KEYRING_BYTES
) {
throw new LocalSecretUnavailableError();
}
contents = await file.readFile();
if (contents.length !== stat.size) {
throw new LocalSecretUnavailableError();
}
return parseManifest(contents);
} catch {
throw new LocalSecretUnavailableError();
} finally {
contents?.fill(0);
await file?.close().catch(() => undefined);
}
}
}
@@ -0,0 +1,106 @@
import { createHash } from 'crypto';
import fs from 'fs/promises';
import path from 'path';
import {
defaultOffRuntimeRolloutPolicy,
parseRuntimeRolloutManifest,
} from '../../domain/runtimeRolloutManifest';
import type {
RuntimeRolloutLoadAudit,
RuntimeRolloutLoadResult,
} from '../../ports/runtimeRolloutLoader';
export type {
RuntimeRolloutLoadAudit,
RuntimeRolloutLoadResult,
RuntimeRolloutLoadStatus,
} from '../../ports/runtimeRolloutLoader';
export const MAX_RUNTIME_ROLLOUT_MANIFEST_BYTES = 64 * 1024;
export interface RuntimeRolloutManifestLoaderOptions {
clock?: { now(): number };
maxBytes?: number;
}
function rejected(
audit: RuntimeRolloutLoadAudit,
reasonCode: NonNullable<RuntimeRolloutLoadAudit['reasonCode']>,
): RuntimeRolloutLoadResult {
return {
status: 'rejected',
policy: defaultOffRuntimeRolloutPolicy(),
audit: { ...audit, status: 'rejected', reasonCode },
};
}
export async function loadRuntimeRolloutManifest(
sourcePath: string,
options: RuntimeRolloutManifestLoaderOptions = {},
): Promise<RuntimeRolloutLoadResult> {
if (!path.isAbsolute(sourcePath)) {
throw new TypeError('Runtime rollout manifest path must be absolute');
}
const evaluatedAtMs = (options.clock ?? { now: Date.now }).now();
if (!Number.isSafeInteger(evaluatedAtMs) || evaluatedAtMs < 0) {
throw new TypeError('Runtime rollout clock returned an invalid timestamp');
}
const maxBytes = options.maxBytes ?? MAX_RUNTIME_ROLLOUT_MANIFEST_BYTES;
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
throw new TypeError('Runtime rollout maxBytes must be a positive integer');
}
const baseAudit: RuntimeRolloutLoadAudit = {
event: 'runtime.rollout_config_evaluated',
evaluatedAtMs,
sourcePath,
status: 'rejected',
};
let bytes: Buffer;
try {
bytes = await fs.readFile(sourcePath);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
return {
status: 'missing',
policy: defaultOffRuntimeRolloutPolicy(),
audit: {
...baseAudit,
status: 'missing',
reasonCode: 'FILE_MISSING',
},
};
}
return rejected(baseAudit, 'FILE_READ_FAILED');
}
const sourceSha256 = createHash('sha256').update(bytes).digest('hex');
const hashedAudit = { ...baseAudit, sourceSha256 };
if (bytes.byteLength > maxBytes) {
return rejected(hashedAudit, 'FILE_TOO_LARGE');
}
let value: unknown;
try {
value = JSON.parse(bytes.toString('utf8'));
} catch {
return rejected(hashedAudit, 'INVALID_JSON');
}
try {
const decision = parseRuntimeRolloutManifest(value, evaluatedAtMs);
const status = decision.manifest.enabled ? 'accepted' : 'disabled';
return {
status,
policy: decision.policy,
manifest: decision.manifest,
audit: {
...hashedAudit,
status,
revision: decision.manifest.revision,
},
};
} catch {
return rejected(hashedAudit, 'INVALID_MANIFEST');
}
}
@@ -0,0 +1,534 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import { randomBytes } from 'crypto';
import { lock } from 'proper-lockfile';
import {
MAX_WORKER_EXECUTION_OFFER_JOURNAL_ENTRIES,
MAX_WORKER_EXECUTION_OFFER_JOURNAL_PAGE_SIZE,
MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES,
cloneWorkerExecutionOfferJournalRecord,
parseWorkerExecutionOfferJournalRecord,
serializeWorkerExecutionOfferJournalRecord,
type WorkerExecutionOfferJournalRecord,
} from '../../domain/workerExecutionOffer';
import { assertRunDispatchOfferId } from '../../domain/runDispatchOffer';
import type {
WorkerExecutionOfferJournal,
WorkerExecutionOfferJournalCreateResult,
WorkerExecutionOfferJournalPage,
} from '../../ports/workerExecutionOfferJournal';
import type {
WorkerExecutionOfferJournalOwnership,
WorkerExecutionOfferJournalOwnershipState,
} from '../../ports/workerExecutionOfferJournalOwnership';
const JOURNAL_FILE_PATTERN = /^([0-9a-f]{64})\.json$/;
export const MIN_WORKER_OFFER_JOURNAL_LOCK_STALE_MS = 5_000;
export const MAX_WORKER_OFFER_JOURNAL_LOCK_STALE_MS = 5 * 60_000;
export interface WorkerExecutionOfferLockProvider {
acquire(options: {
root: string;
lockfilePath: string;
staleMs: number;
updateMs: number;
onCompromised(error: Error): void;
}): Promise<() => Promise<void>>;
}
const properLockProvider: WorkerExecutionOfferLockProvider = {
acquire(options) {
return lock(options.root, {
stale: options.staleMs,
update: options.updateMs,
retries: 0,
realpath: true,
lockfilePath: options.lockfilePath,
onCompromised: options.onCompromised,
});
},
};
export class WorkerExecutionOfferJournalCapacityError extends Error {
constructor(readonly maximumEntries: number) {
super(`Worker execution offer journal reached ${maximumEntries} entries`);
this.name = 'WorkerExecutionOfferJournalCapacityError';
}
}
export class WorkerExecutionOfferJournalRevisionError extends Error {
constructor(readonly offerId: string) {
super(`Worker execution offer journal revision changed for ${offerId}`);
this.name = 'WorkerExecutionOfferJournalRevisionError';
}
}
export class WorkerExecutionOfferJournalNotFoundError extends Error {
constructor(readonly offerId: string) {
super(`Worker execution offer journal entry ${offerId} was not found`);
this.name = 'WorkerExecutionOfferJournalNotFoundError';
}
}
export class WorkerExecutionOfferJournalOwnershipError extends Error {
constructor(
readonly reason: 'already_owned' | 'not_owned' | 'compromised',
readonly cause?: unknown,
) {
super(`Worker execution offer journal ownership failed: ${reason}`);
this.name = 'WorkerExecutionOfferJournalOwnershipError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
function assertIntegerBetween(
name: string,
value: number,
minimum: number,
maximum: number,
): void {
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
}
}
/**
* Worker-local restart journal. One bounded file per offer avoids a database or
* sidecar on edge devices. The Worker runtime must exclusively own this root.
*/
export class WorkerExecutionOfferFileJournal
implements WorkerExecutionOfferJournal, WorkerExecutionOfferJournalOwnership
{
private readonly maximumEntries: number;
private readonly ownershipStaleMs: number;
private readonly lockProvider: WorkerExecutionOfferLockProvider;
private readonly onOwnershipCompromised?: (error: Error) => void;
private ownerState: WorkerExecutionOfferJournalOwnershipState = 'unowned';
private releaseOwner?: () => Promise<void>;
private ownershipError?: Error;
private mutationTail: Promise<void> = Promise.resolve();
constructor(
private readonly root: string,
options: {
maximumEntries?: number;
ownershipStaleMs?: number;
lockProvider?: WorkerExecutionOfferLockProvider;
onOwnershipCompromised?: (error: Error) => void;
} = {},
) {
if (!path.isAbsolute(root)) {
throw new RangeError(
'Worker execution offer journal root must be absolute',
);
}
this.maximumEntries = options.maximumEntries ?? 64;
assertIntegerBetween(
'maximumEntries',
this.maximumEntries,
1,
MAX_WORKER_EXECUTION_OFFER_JOURNAL_ENTRIES,
);
this.ownershipStaleMs = options.ownershipStaleMs ?? 30_000;
assertIntegerBetween(
'ownershipStaleMs',
this.ownershipStaleMs,
MIN_WORKER_OFFER_JOURNAL_LOCK_STALE_MS,
MAX_WORKER_OFFER_JOURNAL_LOCK_STALE_MS,
);
this.lockProvider = options.lockProvider ?? properLockProvider;
this.onOwnershipCompromised = options.onOwnershipCompromised;
}
ownershipState(): WorkerExecutionOfferJournalOwnershipState {
return this.ownerState;
}
async acquireOwnership(): Promise<'acquired' | 'already_owned'> {
if (this.ownerState === 'owned') return 'already_owned';
if (this.ownerState === 'releasing') {
throw new WorkerExecutionOfferJournalOwnershipError('not_owned');
}
if (this.ownerState === 'compromised') {
throw new WorkerExecutionOfferJournalOwnershipError(
'compromised',
this.ownershipError,
);
}
await this.ensureRoot();
try {
const release = await this.lockProvider.acquire({
root: this.root,
lockfilePath: path.join(this.root, '.owner.lock'),
staleMs: this.ownershipStaleMs,
updateMs: Math.max(1_000, Math.floor(this.ownershipStaleMs / 2)),
onCompromised: (error) => this.compromiseOwnership(error),
});
this.releaseOwner = release;
this.ownerState = 'owned';
return 'acquired';
} catch (error) {
if (isCode(error, 'ELOCKED')) {
throw new WorkerExecutionOfferJournalOwnershipError(
'already_owned',
error,
);
}
throw error;
}
}
async releaseOwnership(): Promise<'released' | 'not_owned' | 'compromised'> {
if (this.ownerState === 'unowned') return 'not_owned';
if (this.ownerState === 'compromised') return 'compromised';
if (this.ownerState === 'releasing') return 'not_owned';
while (true) {
const pending = this.mutationTail;
await pending;
if (pending === this.mutationTail) break;
}
const ownerStateAfterMutations = this.ownershipState();
if (ownerStateAfterMutations === 'compromised') return 'compromised';
if (ownerStateAfterMutations !== 'owned') return 'not_owned';
const release = this.releaseOwner;
if (!release) {
this.compromiseOwnership(
new Error('Worker offer journal owner release capability is missing'),
);
return 'compromised';
}
this.ownerState = 'releasing';
try {
await release();
this.releaseOwner = undefined;
this.ownerState = 'unowned';
return 'released';
} catch (error) {
const compromised =
error instanceof Error
? error
: new Error('Worker offer journal owner release failed');
this.compromiseOwnership(compromised);
throw new WorkerExecutionOfferJournalOwnershipError('compromised', error);
}
}
async create(
record: WorkerExecutionOfferJournalRecord,
): Promise<WorkerExecutionOfferJournalCreateResult> {
this.assertOwned();
const candidate = cloneWorkerExecutionOfferJournalRecord(record);
if (candidate.revision !== 0 || candidate.state !== 'accepted') {
throw new TypeError(
'A new Worker offer journal entry must be accepted at revision zero',
);
}
return this.serializeMutation(async () => {
const target = this.target(candidate.offer.offerId);
if (await this.exists(target)) return 'exists';
const names = await this.entryNames();
if (names.length >= this.maximumEntries) {
throw new WorkerExecutionOfferJournalCapacityError(this.maximumEntries);
}
const temporary = this.temporary(candidate.offer.offerId);
try {
await this.writeTemporary(temporary, candidate);
this.assertOwned();
await fs.link(temporary, target);
await this.bestEffortSyncDirectory();
return 'created';
} catch (error) {
if (isCode(error, 'EEXIST')) return 'exists';
throw error;
} finally {
await this.bestEffortUnlink(temporary);
}
});
}
async read(
offerId: string,
): Promise<WorkerExecutionOfferJournalRecord | undefined> {
this.assertOwned();
assertRunDispatchOfferId(offerId);
let handle: fs.FileHandle;
try {
handle = await fs.open(
this.target(offerId),
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
throw error;
}
try {
const stat = await handle.stat();
if (!stat.isFile()) {
throw new TypeError(
'Worker execution offer journal entry must be a regular file',
);
}
const bytes = Buffer.allocUnsafe(
MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES + 1,
);
const { bytesRead } = await handle.read(bytes, 0, bytes.length, 0);
if (bytesRead > MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES) {
throw new TypeError(
'Worker execution offer journal entry exceeds the byte limit',
);
}
const record = parseWorkerExecutionOfferJournalRecord(
bytes.subarray(0, bytesRead),
);
if (record.offer.offerId !== offerId) {
throw new TypeError(
'Worker offer journal path and payload do not match',
);
}
return record;
} finally {
await handle.close();
}
}
async replace(
record: WorkerExecutionOfferJournalRecord,
expectedRevision: number,
): Promise<void> {
this.assertOwned();
const candidate = cloneWorkerExecutionOfferJournalRecord(record);
assertIntegerBetween(
'expectedRevision',
expectedRevision,
0,
Number.MAX_SAFE_INTEGER - 1,
);
if (candidate.revision !== expectedRevision + 1) {
throw new TypeError('Replacement journal revision must increment by one');
}
await this.serializeMutation(async () => {
const current = await this.read(candidate.offer.offerId);
if (!current) {
throw new WorkerExecutionOfferJournalNotFoundError(
candidate.offer.offerId,
);
}
if (current.revision !== expectedRevision) {
throw new WorkerExecutionOfferJournalRevisionError(
candidate.offer.offerId,
);
}
const temporary = this.temporary(candidate.offer.offerId);
try {
await this.writeTemporary(temporary, candidate);
this.assertOwned();
await fs.rename(temporary, this.target(candidate.offer.offerId));
await this.bestEffortSyncDirectory();
} finally {
await this.bestEffortUnlink(temporary);
}
});
}
async remove(offerId: string, expectedRevision?: number): Promise<boolean> {
this.assertOwned();
assertRunDispatchOfferId(offerId);
if (expectedRevision !== undefined) {
assertIntegerBetween(
'expectedRevision',
expectedRevision,
0,
Number.MAX_SAFE_INTEGER,
);
}
return this.serializeMutation(async () => {
if (expectedRevision !== undefined) {
const current = await this.read(offerId);
if (!current) return false;
if (current.revision !== expectedRevision) {
throw new WorkerExecutionOfferJournalRevisionError(offerId);
}
}
try {
this.assertOwned();
await fs.unlink(this.target(offerId));
await this.bestEffortSyncDirectory();
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
throw error;
}
});
}
async list(
options: { afterOfferId?: string; limit?: number } = {},
): Promise<WorkerExecutionOfferJournalPage> {
this.assertOwned();
if (options.afterOfferId !== undefined) {
assertRunDispatchOfferId(options.afterOfferId);
}
const limit = options.limit ?? 32;
assertIntegerBetween(
'limit',
limit,
1,
MAX_WORKER_EXECUTION_OFFER_JOURNAL_PAGE_SIZE,
);
const names = await this.entryNames();
const offerIds = names
.map((name) => JOURNAL_FILE_PATTERN.exec(name)?.[1])
.filter((value): value is string => value !== undefined)
.filter(
(offerId) =>
options.afterOfferId === undefined || offerId > options.afterOfferId,
)
.sort();
const selected = offerIds.slice(0, limit + 1);
const hasMore = selected.length > limit;
const pageIds = selected.slice(0, limit);
const records: WorkerExecutionOfferJournalRecord[] = [];
for (const offerId of pageIds) {
const record = await this.read(offerId);
if (!record) {
throw new WorkerExecutionOfferJournalRevisionError(offerId);
}
records.push(record);
}
return {
records,
...(hasMore && pageIds.length
? { nextAfterOfferId: pageIds[pageIds.length - 1] }
: {}),
};
}
private target(offerId: string): string {
assertRunDispatchOfferId(offerId);
return path.join(this.root, `${offerId}.json`);
}
private temporary(offerId: string): string {
return path.join(
this.root,
`.${offerId}.${randomBytes(16).toString('hex')}.tmp`,
);
}
private async ensureRoot(): Promise<void> {
await fs.mkdir(this.root, { recursive: true, mode: 0o700 });
const stat = await fs.lstat(this.root);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new TypeError(
'Worker execution offer journal root must be a real directory',
);
}
await fs.chmod(this.root, 0o700);
}
private assertOwned(): void {
if (this.ownerState === 'owned') return;
throw new WorkerExecutionOfferJournalOwnershipError(
this.ownerState === 'compromised' ? 'compromised' : 'not_owned',
this.ownershipError,
);
}
private compromiseOwnership(error: Error): void {
this.ownershipError = error;
this.releaseOwner = undefined;
this.ownerState = 'compromised';
try {
this.onOwnershipCompromised?.(error);
} catch {
// Ownership loss must remain visible even if diagnostics fail.
}
}
private async entryNames(): Promise<string[]> {
let names: string[];
try {
names = await fs.readdir(this.root);
} catch (error) {
if (isCode(error, 'ENOENT')) return [];
throw error;
}
const entries = names.filter((name) => JOURNAL_FILE_PATTERN.test(name));
if (entries.length > this.maximumEntries) {
throw new WorkerExecutionOfferJournalCapacityError(this.maximumEntries);
}
return entries;
}
private async writeTemporary(
temporary: string,
record: WorkerExecutionOfferJournalRecord,
): Promise<void> {
const serialized = serializeWorkerExecutionOfferJournalRecord(record);
let handle: fs.FileHandle | undefined;
try {
handle = await fs.open(temporary, 'wx', 0o600);
await handle.writeFile(serialized, 'utf8');
await handle.sync();
} finally {
await handle?.close().catch(() => undefined);
}
}
private async exists(target: string): Promise<boolean> {
try {
const stat = await fs.lstat(target);
if (!stat.isFile()) {
throw new TypeError(
'Worker execution offer journal target must be a regular file',
);
}
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
throw error;
}
}
private async bestEffortUnlink(target: string): Promise<void> {
try {
await fs.unlink(target);
} catch (error) {
if (!isCode(error, 'ENOENT')) {
// Temp cleanup is diagnostic-only; the atomically published record wins.
}
}
}
private async bestEffortSyncDirectory(): Promise<void> {
try {
const handle = await fs.open(this.root, constants.O_RDONLY);
try {
await handle.sync();
} finally {
await handle.close();
}
} catch {
// Some supported filesystems cannot fsync directories.
}
}
private serializeMutation<T>(operation: () => Promise<T>): Promise<T> {
const result = this.mutationTail.then(operation, operation);
this.mutationTail = result.then(
() => undefined,
() => undefined,
);
return result;
}
}
@@ -0,0 +1,925 @@
import {
DataTypes,
Model,
ModelStatic,
QueryTypes,
Sequelize,
Transaction,
UniqueConstraintError,
} from 'sequelize';
import {
APPROVAL_REQUEST_TABLE,
APPROVED_ACTION_DISPATCH_TABLE,
} from '../../../migrations/0020-approval-requests';
import { APPROVED_ACTION_DISPATCH_EXECUTION_TABLE } from '../../../migrations/0021-approved-action-dispatch-executions';
import { DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS } from '../../domain/approvedActionDispatchExecution';
import {
ApprovalMutationConflictError,
ApprovalPolicyFenceConflictError,
ApprovalRequestExpiredError,
ApprovalRequestNotFoundError,
ApprovalRequestStateConflictError,
ApprovalRequestVersionConflictError,
ApprovalUnavailableError,
InvalidApprovalValueError,
normalizeApprovalActionBinding,
normalizeApprovalPolicyFence,
normalizeApprovalRequestRecord,
normalizeApprovedActionDispatchRecord,
sameApprovalAction,
sameApprovalSubject,
type ApprovalRequestRecord,
type ApprovedActionDispatchRecord,
} from '../../domain/approvalRequest';
import {
normalizePolicySubject,
type ProjectPolicyFence,
} from '../../domain/projectPolicy';
import type {
ApprovalRequestRepository,
ConsumeApprovalRequestCommand,
ConsumeApprovalRequestResult,
CreateApprovalRequestCommand,
CreateApprovalRequestResult,
DecideApprovalRequestCommand,
DecideApprovalRequestResult,
} from '../../ports/approvalRequestRepository';
import {
PROJECT_ROLE_BINDING_TABLE,
PROJECT_TABLE,
} from '../../../migrations/0017-project-policy';
const RETRY_ATTEMPTS = 5;
interface ApprovalRequestRow {
id: string;
projectId: string;
version: number;
state: string;
permission: string;
actionType: string;
actionRef: string;
actionDigest: string;
previewDigest: string;
risk: string;
requestedByType: string;
requestedById: string;
requestedAtMs: number | string;
expiresAtMs: number | string;
decisionId: string | null;
decision: string | null;
decisionReasonCode: string | null;
decidedByType: string | null;
decidedById: string | null;
decidedAtMs: number | string | null;
consumptionId: string | null;
dispatchId: string | null;
consumedByType: string | null;
consumedById: string | null;
consumedAtMs: number | string | null;
}
interface ApprovalRequestInstance
extends Model<ApprovalRequestRow, ApprovalRequestRow>,
ApprovalRequestRow {}
interface ApprovedActionDispatchRow {
id: string;
approvalRequestId: string;
approvalRequestVersion: number;
projectId: string;
state: string;
permission: string;
actionType: string;
actionRef: string;
actionDigest: string;
previewDigest: string;
requestedByType: string;
requestedById: string;
consumedByType: string;
consumedById: string;
createdAtMs: number | string;
}
interface ApprovedActionDispatchInstance
extends Model<ApprovedActionDispatchRow, ApprovedActionDispatchRow>,
ApprovedActionDispatchRow {}
interface PolicyFenceRow {
project_version: number | string;
binding_version: number | string | null;
}
function defineApprovalRequestModel(
database: Sequelize,
): ModelStatic<ApprovalRequestInstance> {
return database.define<ApprovalRequestInstance>(
'Ql3ApprovalRequest',
{
id: { type: DataTypes.STRING(64), allowNull: false, primaryKey: true },
projectId: {
field: 'project_id',
type: DataTypes.STRING(128),
allowNull: false,
},
version: { type: DataTypes.INTEGER, allowNull: false },
state: { type: DataTypes.STRING(16), allowNull: false },
permission: { type: DataTypes.STRING(255), allowNull: false },
actionType: {
field: 'action_type',
type: DataTypes.STRING(64),
allowNull: false,
},
actionRef: {
field: 'action_ref',
type: DataTypes.STRING(255),
allowNull: false,
},
actionDigest: {
field: 'action_digest',
type: DataTypes.STRING(64),
allowNull: false,
},
previewDigest: {
field: 'preview_digest',
type: DataTypes.STRING(64),
allowNull: false,
},
risk: { type: DataTypes.STRING(16), allowNull: false },
requestedByType: {
field: 'requested_by_type',
type: DataTypes.STRING(32),
allowNull: false,
},
requestedById: {
field: 'requested_by_id',
type: DataTypes.STRING(255),
allowNull: false,
},
requestedAtMs: {
field: 'requested_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
expiresAtMs: {
field: 'expires_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
decisionId: {
field: 'decision_id',
type: DataTypes.STRING(64),
allowNull: true,
},
decision: { type: DataTypes.STRING(16), allowNull: true },
decisionReasonCode: {
field: 'decision_reason_code',
type: DataTypes.STRING(64),
allowNull: true,
},
decidedByType: {
field: 'decided_by_type',
type: DataTypes.STRING(32),
allowNull: true,
},
decidedById: {
field: 'decided_by_id',
type: DataTypes.STRING(255),
allowNull: true,
},
decidedAtMs: {
field: 'decided_at_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
consumptionId: {
field: 'consumption_id',
type: DataTypes.STRING(64),
allowNull: true,
},
dispatchId: {
field: 'dispatch_id',
type: DataTypes.STRING(64),
allowNull: true,
},
consumedByType: {
field: 'consumed_by_type',
type: DataTypes.STRING(32),
allowNull: true,
},
consumedById: {
field: 'consumed_by_id',
type: DataTypes.STRING(255),
allowNull: true,
},
consumedAtMs: {
field: 'consumed_at_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
},
{
tableName: APPROVAL_REQUEST_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function defineApprovedActionDispatchModel(
database: Sequelize,
): ModelStatic<ApprovedActionDispatchInstance> {
return database.define<ApprovedActionDispatchInstance>(
'Ql3ApprovedActionDispatch',
{
id: { type: DataTypes.STRING(64), allowNull: false, primaryKey: true },
approvalRequestId: {
field: 'approval_request_id',
type: DataTypes.STRING(64),
allowNull: false,
},
approvalRequestVersion: {
field: 'approval_request_version',
type: DataTypes.INTEGER,
allowNull: false,
},
projectId: {
field: 'project_id',
type: DataTypes.STRING(128),
allowNull: false,
},
state: { type: DataTypes.STRING(16), allowNull: false },
permission: { type: DataTypes.STRING(255), allowNull: false },
actionType: {
field: 'action_type',
type: DataTypes.STRING(64),
allowNull: false,
},
actionRef: {
field: 'action_ref',
type: DataTypes.STRING(255),
allowNull: false,
},
actionDigest: {
field: 'action_digest',
type: DataTypes.STRING(64),
allowNull: false,
},
previewDigest: {
field: 'preview_digest',
type: DataTypes.STRING(64),
allowNull: false,
},
requestedByType: {
field: 'requested_by_type',
type: DataTypes.STRING(32),
allowNull: false,
},
requestedById: {
field: 'requested_by_id',
type: DataTypes.STRING(255),
allowNull: false,
},
consumedByType: {
field: 'consumed_by_type',
type: DataTypes.STRING(32),
allowNull: false,
},
consumedById: {
field: 'consumed_by_id',
type: DataTypes.STRING(255),
allowNull: false,
},
createdAtMs: {
field: 'created_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: APPROVED_ACTION_DISPATCH_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function rowToRequest(
row: ApprovalRequestRow,
): Readonly<ApprovalRequestRecord> {
try {
return normalizeApprovalRequestRecord({
id: row.id,
projectId: row.projectId,
version: Number(row.version),
state: row.state as ApprovalRequestRecord['state'],
action: {
permission:
row.permission as ApprovalRequestRecord['action']['permission'],
actionType: row.actionType,
actionRef: row.actionRef,
actionDigest: row.actionDigest,
previewDigest: row.previewDigest,
},
risk: row.risk as ApprovalRequestRecord['risk'],
requestedBy: {
type: row.requestedByType as ApprovalRequestRecord['requestedBy']['type'],
id: row.requestedById,
},
requestedAtMs: Number(row.requestedAtMs),
expiresAtMs: Number(row.expiresAtMs),
decisionId: row.decisionId,
decision: row.decision as ApprovalRequestRecord['decision'],
decisionReasonCode: row.decisionReasonCode,
decidedBy:
row.decidedByType === null || row.decidedById === null
? null
: {
type: row.decidedByType as NonNullable<
ApprovalRequestRecord['decidedBy']
>['type'],
id: row.decidedById,
},
decidedAtMs: row.decidedAtMs === null ? null : Number(row.decidedAtMs),
consumptionId: row.consumptionId,
dispatchId: row.dispatchId,
consumedBy:
row.consumedByType === null || row.consumedById === null
? null
: {
type: row.consumedByType as NonNullable<
ApprovalRequestRecord['consumedBy']
>['type'],
id: row.consumedById,
},
consumedAtMs: row.consumedAtMs === null ? null : Number(row.consumedAtMs),
});
} catch {
throw new ApprovalUnavailableError();
}
}
function requestToRow(
request: Readonly<ApprovalRequestRecord>,
): ApprovalRequestRow {
return {
id: request.id,
projectId: request.projectId,
version: request.version,
state: request.state,
permission: request.action.permission,
actionType: request.action.actionType,
actionRef: request.action.actionRef,
actionDigest: request.action.actionDigest,
previewDigest: request.action.previewDigest,
risk: request.risk,
requestedByType: request.requestedBy.type,
requestedById: request.requestedBy.id,
requestedAtMs: request.requestedAtMs,
expiresAtMs: request.expiresAtMs,
decisionId: request.decisionId,
decision: request.decision,
decisionReasonCode: request.decisionReasonCode,
decidedByType: request.decidedBy?.type ?? null,
decidedById: request.decidedBy?.id ?? null,
decidedAtMs: request.decidedAtMs,
consumptionId: request.consumptionId,
dispatchId: request.dispatchId,
consumedByType: request.consumedBy?.type ?? null,
consumedById: request.consumedBy?.id ?? null,
consumedAtMs: request.consumedAtMs,
};
}
function rowToDispatch(
row: ApprovedActionDispatchRow,
): Readonly<ApprovedActionDispatchRecord> {
try {
return normalizeApprovedActionDispatchRecord({
id: row.id,
approvalRequestId: row.approvalRequestId,
approvalRequestVersion: Number(row.approvalRequestVersion),
projectId: row.projectId,
state: row.state as ApprovedActionDispatchRecord['state'],
action: {
permission:
row.permission as ApprovedActionDispatchRecord['action']['permission'],
actionType: row.actionType,
actionRef: row.actionRef,
actionDigest: row.actionDigest,
previewDigest: row.previewDigest,
},
requestedBy: {
type: row.requestedByType as ApprovedActionDispatchRecord['requestedBy']['type'],
id: row.requestedById,
},
consumedBy: {
type: row.consumedByType as ApprovedActionDispatchRecord['consumedBy']['type'],
id: row.consumedById,
},
createdAtMs: Number(row.createdAtMs),
});
} catch {
throw new ApprovalUnavailableError();
}
}
function dispatchToRow(
dispatch: Readonly<ApprovedActionDispatchRecord>,
): ApprovedActionDispatchRow {
return {
id: dispatch.id,
approvalRequestId: dispatch.approvalRequestId,
approvalRequestVersion: dispatch.approvalRequestVersion,
projectId: dispatch.projectId,
state: dispatch.state,
permission: dispatch.action.permission,
actionType: dispatch.action.actionType,
actionRef: dispatch.action.actionRef,
actionDigest: dispatch.action.actionDigest,
previewDigest: dispatch.action.previewDigest,
requestedByType: dispatch.requestedBy.type,
requestedById: dispatch.requestedBy.id,
consumedByType: dispatch.consumedBy.type,
consumedById: dispatch.consumedBy.id,
createdAtMs: dispatch.createdAtMs,
};
}
function sameRequestCreation(
left: Readonly<ApprovalRequestRecord>,
right: Readonly<ApprovalRequestRecord>,
): boolean {
return (
left.id === right.id &&
left.projectId === right.projectId &&
sameApprovalAction(left.action, right.action) &&
left.risk === right.risk &&
sameApprovalSubject(left.requestedBy, right.requestedBy) &&
left.requestedAtMs === right.requestedAtMs &&
left.expiresAtMs === right.expiresAtMs
);
}
function sameDispatch(
left: Readonly<ApprovedActionDispatchRecord>,
right: Readonly<ApprovedActionDispatchRecord>,
): boolean {
return (
left.id === right.id &&
left.approvalRequestId === right.approvalRequestId &&
left.approvalRequestVersion === right.approvalRequestVersion &&
left.projectId === right.projectId &&
left.state === right.state &&
sameApprovalAction(left.action, right.action) &&
sameApprovalSubject(left.requestedBy, right.requestedBy) &&
sameApprovalSubject(left.consumedBy, right.consumedBy) &&
left.createdAtMs === right.createdAtMs
);
}
function errorCode(error: unknown): string | undefined {
if (!error || typeof error !== 'object') return undefined;
for (const candidate of [
error,
'original' in error ? error.original : undefined,
'parent' in error ? error.parent : undefined,
]) {
if (
candidate &&
typeof candidate === 'object' &&
'code' in candidate &&
typeof candidate.code === 'string'
) {
return candidate.code;
}
}
return undefined;
}
function retryDelay(attempt: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
}
function isApprovalError(error: unknown): boolean {
return (
error instanceof ApprovalMutationConflictError ||
error instanceof InvalidApprovalValueError ||
error instanceof ApprovalPolicyFenceConflictError ||
error instanceof ApprovalRequestExpiredError ||
error instanceof ApprovalRequestNotFoundError ||
error instanceof ApprovalRequestStateConflictError ||
error instanceof ApprovalRequestVersionConflictError ||
error instanceof ApprovalUnavailableError
);
}
export class LegacySequelizeApprovalRequestRepository
implements ApprovalRequestRepository
{
private readonly requests: ModelStatic<ApprovalRequestInstance>;
private readonly dispatches: ModelStatic<ApprovedActionDispatchInstance>;
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Approval request repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
this.requests = defineApprovalRequestModel(database);
this.dispatches = defineApprovedActionDispatchModel(database);
}
async findById(id: string): Promise<Readonly<ApprovalRequestRecord> | null> {
const row = await this.requests.findByPk(id, { raw: true });
return row ? rowToRequest(row) : null;
}
private async assertFence(
projectId: string,
subject: Readonly<ApprovalRequestRecord['requestedBy']>,
requestedFence: Readonly<ProjectPolicyFence>,
transaction: Transaction,
): Promise<void> {
const normalizedSubject = normalizePolicySubject(subject);
const fence = normalizeApprovalPolicyFence(requestedFence);
const rows = await this.database.query<PolicyFenceRow>(
`SELECT project.version AS project_version,
(SELECT MAX(binding.version)
FROM "${PROJECT_ROLE_BINDING_TABLE}" AS binding
WHERE binding.project_id = project.id
AND binding.subject_type = :subjectType
AND binding.subject_id = :subjectId) AS binding_version
FROM "${PROJECT_TABLE}" AS project
WHERE project.id = :projectId
LIMIT 2`,
{
type: QueryTypes.SELECT,
replacements: {
projectId,
subjectType: normalizedSubject.type,
subjectId: normalizedSubject.id,
},
transaction,
},
);
if (rows.length !== 1) throw new ApprovalPolicyFenceConflictError();
const currentProjectVersion = Number(rows[0].project_version);
const currentBindingVersion =
rows[0].binding_version === null ? null : Number(rows[0].binding_version);
if (
currentProjectVersion !== fence.projectVersion ||
currentBindingVersion !== fence.bindingVersion
) {
throw new ApprovalPolicyFenceConflictError();
}
}
private async findDecisionReplay(
decisionId: string,
transaction: Transaction,
): Promise<Readonly<ApprovalRequestRecord> | null> {
const row = await this.requests.findOne({
where: { decisionId },
raw: true,
transaction,
});
return row ? rowToRequest(row) : null;
}
private async findConsumptionReplay(
consumptionId: string,
transaction: Transaction,
): Promise<{
request: Readonly<ApprovalRequestRecord>;
dispatch: Readonly<ApprovedActionDispatchRecord>;
} | null> {
const row = await this.requests.findOne({
where: { consumptionId },
raw: true,
transaction,
});
if (!row) return null;
const request = rowToRequest(row);
if (!request.dispatchId) throw new ApprovalUnavailableError();
const dispatchRow = await this.dispatches.findByPk(request.dispatchId, {
raw: true,
transaction,
});
if (!dispatchRow) throw new ApprovalUnavailableError();
const executionRows = await this.database.query<{
dispatch_id: string;
project_id: string;
}>(
`SELECT dispatch_id, project_id
FROM "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
WHERE dispatch_id = :dispatchId
LIMIT 2`,
{
type: QueryTypes.SELECT,
replacements: { dispatchId: request.dispatchId },
transaction,
},
);
if (
executionRows.length !== 1 ||
executionRows[0].project_id !== request.projectId
) {
throw new ApprovalUnavailableError();
}
return { request, dispatch: rowToDispatch(dispatchRow) };
}
async create(
command: CreateApprovalRequestCommand,
): Promise<CreateApprovalRequestResult> {
const request = normalizeApprovalRequestRecord(command.request);
const fence = normalizeApprovalPolicyFence(command.authorizationFence);
if (request.state !== 'pending' || request.version !== 1) {
throw new ApprovalRequestStateConflictError();
}
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
try {
return await this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const existing = await this.requests.findByPk(request.id, {
raw: true,
transaction,
});
if (existing) {
const previous = rowToRequest(existing);
if (!sameRequestCreation(previous, request)) {
throw new ApprovalMutationConflictError();
}
return { status: 'existing', request: previous };
}
await this.assertFence(
request.projectId,
request.requestedBy,
fence,
transaction,
);
await this.requests.create(requestToRow(request), { transaction });
return { status: 'created', request };
},
);
} catch (error) {
if (isApprovalError(error)) throw error;
if (
(error instanceof UniqueConstraintError ||
errorCode(error) === 'SQLITE_BUSY') &&
attempt < RETRY_ATTEMPTS - 1
) {
await retryDelay(attempt);
continue;
}
throw new ApprovalUnavailableError();
}
}
throw new ApprovalUnavailableError();
}
async decide(
command: DecideApprovalRequestCommand,
): Promise<DecideApprovalRequestResult> {
const decidedBy = normalizePolicySubject(command.decidedBy);
const fence = normalizeApprovalPolicyFence(command.authorizationFence);
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
try {
return await this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const replay = await this.findDecisionReplay(
command.decisionId,
transaction,
);
if (replay) {
if (
replay.id !== command.requestId ||
command.expectedVersion !== 1 ||
replay.decisionId !== command.decisionId ||
replay.decision !== command.decision ||
replay.decisionReasonCode !== command.reasonCode ||
!replay.decidedBy ||
!sameApprovalSubject(replay.decidedBy, decidedBy) ||
replay.decidedAtMs !== command.decidedAtMs
) {
throw new ApprovalMutationConflictError();
}
return { status: 'existing', request: replay };
}
const row = await this.requests.findByPk(command.requestId, {
raw: true,
transaction,
});
if (!row) throw new ApprovalRequestNotFoundError();
const current = rowToRequest(row);
if (command.decidedAtMs >= current.expiresAtMs) {
throw new ApprovalRequestExpiredError();
}
if (current.version !== command.expectedVersion) {
throw new ApprovalRequestVersionConflictError();
}
if (current.state !== 'pending') {
throw new ApprovalRequestStateConflictError();
}
await this.assertFence(
current.projectId,
decidedBy,
fence,
transaction,
);
const decided = normalizeApprovalRequestRecord({
...current,
version: 2,
state: command.decision,
decisionId: command.decisionId,
decision: command.decision,
decisionReasonCode: command.reasonCode,
decidedBy,
decidedAtMs: command.decidedAtMs,
});
const [updated] = await this.requests.update(
requestToRow(decided),
{
where: {
id: current.id,
version: command.expectedVersion,
state: 'pending',
},
transaction,
},
);
if (updated !== 1) {
throw new ApprovalRequestVersionConflictError();
}
return { status: 'decided', request: decided };
},
);
} catch (error) {
if (isApprovalError(error)) throw error;
if (
(error instanceof UniqueConstraintError ||
errorCode(error) === 'SQLITE_BUSY') &&
attempt < RETRY_ATTEMPTS - 1
) {
await retryDelay(attempt);
continue;
}
throw new ApprovalUnavailableError();
}
}
throw new ApprovalUnavailableError();
}
async consume(
command: ConsumeApprovalRequestCommand,
): Promise<ConsumeApprovalRequestResult> {
const action = normalizeApprovalActionBinding(command.action);
const requestedBy = normalizePolicySubject(command.requestedBy);
const consumedBy = normalizePolicySubject(command.consumedBy);
const fence = normalizeApprovalPolicyFence(command.authorizationFence);
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
try {
return await this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const replay = await this.findConsumptionReplay(
command.consumptionId,
transaction,
);
if (replay) {
const expectedDispatch = normalizeApprovedActionDispatchRecord({
id: command.dispatchId,
approvalRequestId: command.requestId,
approvalRequestVersion: 3,
projectId: replay.request.projectId,
state: 'pending',
action,
requestedBy,
consumedBy,
createdAtMs: command.consumedAtMs,
});
if (
command.expectedVersion !== 2 ||
replay.request.id !== command.requestId ||
replay.request.consumptionId !== command.consumptionId ||
!sameDispatch(replay.dispatch, expectedDispatch)
) {
throw new ApprovalMutationConflictError();
}
return {
status: 'existing',
request: replay.request,
dispatch: replay.dispatch,
};
}
const row = await this.requests.findByPk(command.requestId, {
raw: true,
transaction,
});
if (!row) throw new ApprovalRequestNotFoundError();
const current = rowToRequest(row);
if (command.consumedAtMs >= current.expiresAtMs) {
throw new ApprovalRequestExpiredError();
}
if (current.version !== command.expectedVersion) {
throw new ApprovalRequestVersionConflictError();
}
if (current.state !== 'approved') {
throw new ApprovalRequestStateConflictError();
}
if (
!sameApprovalAction(current.action, action) ||
!sameApprovalSubject(current.requestedBy, requestedBy)
) {
throw new ApprovalMutationConflictError();
}
await this.assertFence(
current.projectId,
requestedBy,
fence,
transaction,
);
const dispatch = normalizeApprovedActionDispatchRecord({
id: command.dispatchId,
approvalRequestId: current.id,
approvalRequestVersion: 3,
projectId: current.projectId,
state: 'pending',
action,
requestedBy,
consumedBy,
createdAtMs: command.consumedAtMs,
});
const dispatchCollision = await this.dispatches.findByPk(
dispatch.id,
{ raw: true, transaction },
);
if (dispatchCollision) throw new ApprovalMutationConflictError();
await this.dispatches.create(dispatchToRow(dispatch), {
transaction,
});
await this.database.query(
`INSERT INTO "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
(dispatch_id, project_id, status, version, attempt_count,
max_attempts, eligible_at_ms, next_attempt_at_ms,
lease_owner, lease_token, lease_expires_at_ms, started_at_ms,
last_result_code, completed_at_ms, created_at_ms, updated_at_ms)
VALUES
(:dispatchId, :projectId, 'pending', 0, 0, :maxAttempts,
:createdAtMs, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
:createdAtMs, :createdAtMs)`,
{
replacements: {
dispatchId: dispatch.id,
projectId: dispatch.projectId,
maxAttempts: DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS,
createdAtMs: dispatch.createdAtMs,
},
transaction,
},
);
const consumed = normalizeApprovalRequestRecord({
...current,
version: 3,
state: 'consumed',
consumptionId: command.consumptionId,
dispatchId: command.dispatchId,
consumedBy,
consumedAtMs: command.consumedAtMs,
});
const [updated] = await this.requests.update(
requestToRow(consumed),
{
where: {
id: current.id,
version: command.expectedVersion,
state: 'approved',
},
transaction,
},
);
if (updated !== 1) {
throw new ApprovalRequestVersionConflictError();
}
return { status: 'consumed', request: consumed, dispatch };
},
);
} catch (error) {
if (isApprovalError(error)) throw error;
if (
(error instanceof UniqueConstraintError ||
errorCode(error) === 'SQLITE_BUSY') &&
attempt < RETRY_ATTEMPTS - 1
) {
await retryDelay(attempt);
continue;
}
throw new ApprovalUnavailableError();
}
}
throw new ApprovalUnavailableError();
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,427 @@
import {
QueryTypes,
Sequelize,
Transaction,
type Transaction as SequelizeTransaction,
} from 'sequelize';
import { APPROVED_RUN_ACTION_RECEIPT_TABLE } from '../../../migrations/0023-approved-run-action-receipts';
import { APPROVED_ACTION_DISPATCH_EXECUTION_TABLE } from '../../../migrations/0021-approved-action-dispatch-executions';
import {
APPROVED_RUN_ACTION_TYPE,
APPROVED_RUN_RECEIPT_RESULT_CODE,
ApprovedRunActionBindingConflictError,
ApprovedRunActionRepositoryError,
InvalidApprovedRunActionError,
digestApprovedRunCreationPlan,
digestApprovedRunCreationReceipt,
normalizeApprovedRunCreationPlan,
normalizeApprovedRunCreationReceipt,
type ApprovedRunCreationReceipt,
} from '../../domain/approvedRunAction';
import {
normalizeApprovedActionDispatchExecutionRecord,
type ApprovedActionDispatchExecutionSnapshot,
} from '../../domain/approvedActionDispatchExecution';
import { normalizeApprovedActionDispatchRecord } from '../../domain/approvalRequest';
import { DuplicateIdempotencyKeyError } from '../../domain/repositoryErrors';
import type { RunRecord } from '../../domain/run';
import {
PrimaryRunCreator,
type PrimaryRunIdFactory,
} from '../../application/primaryRunCreator';
import type {
ApprovedRunActionRepository,
ApprovedRunReference,
CreateApprovedRunCommand,
} from '../../ports/approvedRunActionRepository';
import type { RunRepositoryTransaction } from '../../ports/runRepository';
import {
LegacySequelizeRunRepository,
LegacySequelizeRunTransaction,
} from './runRepository';
interface ApprovedRunReceiptRow {
schema_version: number;
dispatch_id: string;
approval_request_id: string;
project_id: string;
action_type: string;
action_digest: string;
execution_attempt: number;
execution_version: number;
started_at_ms: number;
idempotency_key: string;
outcome: string;
result_code: string;
resource_type: string;
resource_id: string;
finished_at_ms: number;
evidence_digest: string;
created_at_ms: number;
}
interface ReceiptBinding {
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>;
clock: () => number;
}
interface ExecutionFenceRow {
project_id: string;
status: string;
version: number;
attempt_count: number;
lease_owner: string | null;
lease_token: string | null;
started_at_ms: number | null;
}
function rowToReceipt(
row: ApprovedRunReceiptRow,
): Readonly<ApprovedRunCreationReceipt> {
return normalizeApprovedRunCreationReceipt({
schemaVersion: row.schema_version as 1,
dispatchId: row.dispatch_id,
approvalRequestId: row.approval_request_id,
projectId: row.project_id,
actionType: row.action_type as typeof APPROVED_RUN_ACTION_TYPE,
actionDigest: row.action_digest,
executionAttempt: row.execution_attempt,
executionVersion: row.execution_version,
startedAtMs: row.started_at_ms,
idempotencyKey: row.idempotency_key,
outcome: row.outcome as 'succeeded',
resultCode: row.result_code as typeof APPROVED_RUN_RECEIPT_RESULT_CODE,
resourceType: row.resource_type as 'run',
resourceId: row.resource_id,
finishedAtMs: row.finished_at_ms,
evidenceDigest: row.evidence_digest,
createdAtMs: row.created_at_ms,
});
}
function normalizeSnapshot(
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
): Readonly<ApprovedActionDispatchExecutionSnapshot> {
if (!snapshot || typeof snapshot !== 'object' || Array.isArray(snapshot)) {
throw new InvalidApprovedRunActionError('execution snapshot is invalid');
}
const dispatch = normalizeApprovedActionDispatchRecord(snapshot.dispatch);
const execution = normalizeApprovedActionDispatchExecutionRecord(
snapshot.execution,
);
if (
execution.dispatchId !== dispatch.id ||
execution.projectId !== dispatch.projectId ||
execution.status !== 'executing' ||
execution.startedAtMs === null
) {
throw new ApprovedRunActionBindingConflictError();
}
return Object.freeze({ dispatch, execution });
}
function receiptMatches(
receipt: Readonly<ApprovedRunCreationReceipt>,
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
): boolean {
return (
receipt.dispatchId === snapshot.dispatch.id &&
receipt.approvalRequestId === snapshot.dispatch.approvalRequestId &&
receipt.projectId === snapshot.dispatch.projectId &&
receipt.actionType === snapshot.dispatch.action.actionType &&
receipt.actionDigest === snapshot.dispatch.action.actionDigest &&
receipt.executionAttempt === snapshot.execution.attemptCount &&
receipt.startedAtMs === snapshot.execution.startedAtMs &&
receipt.idempotencyKey === snapshot.dispatch.id
);
}
class AtomicApprovedRunRepository extends LegacySequelizeRunRepository {
constructor(
private readonly approvedDatabase: Sequelize,
private readonly binding: Readonly<ReceiptBinding>,
) {
super(approvedDatabase);
}
override async transaction<T>(
work: (transaction: RunRepositoryTransaction) => Promise<T>,
): Promise<T> {
return this.approvedDatabase.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const executionVersion = await this.requireCurrentExecutionFence(
transaction,
);
const result = await work(
new LegacySequelizeRunTransaction(this.models, transaction),
);
const run = this.requireCreatedRun(result);
await this.insertReceipt(run, executionVersion, transaction);
return result;
},
);
}
private async requireCurrentExecutionFence(
transaction: SequelizeTransaction,
): Promise<number> {
const { dispatch, execution } = this.binding.snapshot;
const rows = await this.approvedDatabase.query<ExecutionFenceRow>(
`SELECT project_id, status, version, attempt_count, lease_owner,
lease_token, started_at_ms
FROM "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
WHERE dispatch_id = :dispatchId
LIMIT 2`,
{
type: QueryTypes.SELECT,
replacements: { dispatchId: dispatch.id },
transaction,
},
);
const current = rows[0];
if (
rows.length !== 1 ||
current.project_id !== dispatch.projectId ||
current.status !== 'executing' ||
current.version < execution.version ||
current.attempt_count !== execution.attemptCount ||
current.lease_owner !== execution.leaseOwner ||
current.lease_token !== execution.leaseToken ||
current.started_at_ms !== execution.startedAtMs
) {
throw new ApprovedRunActionBindingConflictError();
}
return current.version;
}
private requireCreatedRun(value: unknown): Readonly<RunRecord> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!('id' in value) ||
typeof value.id !== 'string' ||
!('projectId' in value) ||
value.projectId !== this.binding.snapshot.dispatch.projectId ||
!('idempotencyKey' in value) ||
value.idempotencyKey !== this.binding.snapshot.dispatch.id ||
!('requestId' in value) ||
value.requestId !== this.binding.snapshot.dispatch.approvalRequestId ||
!('status' in value) ||
value.status !== 'queued'
) {
throw new ApprovedRunActionBindingConflictError();
}
return value as Readonly<RunRecord>;
}
private async insertReceipt(
run: Readonly<RunRecord>,
executionVersion: number,
transaction: SequelizeTransaction,
): Promise<void> {
const { dispatch, execution } = this.binding.snapshot;
const finishedAtMs = this.nowAtOrAfter(execution.startedAtMs!);
const unsigned: Omit<ApprovedRunCreationReceipt, 'evidenceDigest'> = {
schemaVersion: 1,
dispatchId: dispatch.id,
approvalRequestId: dispatch.approvalRequestId,
projectId: dispatch.projectId,
actionType: APPROVED_RUN_ACTION_TYPE,
actionDigest: dispatch.action.actionDigest,
executionAttempt: execution.attemptCount,
executionVersion,
startedAtMs: execution.startedAtMs!,
idempotencyKey: dispatch.id,
outcome: 'succeeded',
resultCode: APPROVED_RUN_RECEIPT_RESULT_CODE,
resourceType: 'run',
resourceId: run.id,
finishedAtMs,
createdAtMs: finishedAtMs,
};
const receipt = normalizeApprovedRunCreationReceipt({
...unsigned,
evidenceDigest: digestApprovedRunCreationReceipt(unsigned),
});
await this.approvedDatabase.query(
`INSERT INTO "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
(dispatch_id, approval_request_id, project_id, schema_version,
action_type, action_digest, execution_attempt, execution_version,
started_at_ms, idempotency_key, outcome, result_code, resource_type,
resource_id, finished_at_ms, evidence_digest, created_at_ms)
VALUES
(:dispatchId, :approvalRequestId, :projectId, :schemaVersion,
:actionType, :actionDigest, :executionAttempt, :executionVersion,
:startedAtMs, :idempotencyKey, :outcome, :resultCode, :resourceType,
:resourceId, :finishedAtMs, :evidenceDigest, :createdAtMs)`,
{ replacements: receipt, transaction },
);
}
private nowAtOrAfter(minimum: number): number {
const nowMs = this.binding.clock();
if (!Number.isSafeInteger(nowMs) || nowMs < minimum) {
throw new RangeError('clock must not precede the action start barrier');
}
return nowMs;
}
}
export interface LegacySequelizeApprovedRunActionRepositoryOptions {
clock?: () => number;
createId?: PrimaryRunIdFactory;
}
export class LegacySequelizeApprovedRunActionRepository
implements ApprovedRunActionRepository
{
private readonly runs: LegacySequelizeRunRepository;
private readonly clock: () => number;
private readonly createId?: PrimaryRunIdFactory;
constructor(
private readonly database: Sequelize,
options: LegacySequelizeApprovedRunActionRepositoryOptions = {},
) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Approved Run action repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
this.runs = new LegacySequelizeRunRepository(database);
this.clock = options.clock ?? Date.now;
this.createId = options.createId;
}
async create(
command: Readonly<CreateApprovedRunCommand>,
): Promise<Readonly<ApprovedRunReference>> {
try {
const snapshot = normalizeSnapshot(command.snapshot);
const plan = normalizeApprovedRunCreationPlan(command.plan);
this.assertPlanBinding(snapshot, plan);
const replay = await this.findReplay(snapshot);
if (replay) return replay;
const atomic = new AtomicApprovedRunRepository(this.database, {
snapshot,
clock: this.clock,
});
const creator = new PrimaryRunCreator(atomic, this.createId);
try {
return await creator.create(
{
projectId: plan.projectId,
taskId: plan.taskId,
taskRevision: plan.taskRevision,
...(plan.taskName === undefined ? {} : { taskName: plan.taskName }),
...(plan.taskSnapshotRef === undefined
? {}
: { taskSnapshotRef: plan.taskSnapshotRef }),
triggerType: 'approved_action',
executionOrigin: 'system',
triggeredBy: `approved-action:${snapshot.dispatch.id}`,
requestId: snapshot.dispatch.approvalRequestId,
priority: plan.priority,
idempotencyKey: snapshot.dispatch.id,
...(plan.inputRef === undefined ? {} : { inputRef: plan.inputRef }),
acceptedAtMs: snapshot.execution.startedAtMs!,
actor: { type: 'system', id: 'approved-action-dispatcher' },
},
plan.executorType,
);
} catch (error) {
if (!(error instanceof DuplicateIdempotencyKeyError)) throw error;
const raced = await this.findReplay(snapshot);
if (raced) return raced;
throw new ApprovedRunActionBindingConflictError();
}
} catch (error) {
if (
error instanceof ApprovedRunActionBindingConflictError ||
error instanceof InvalidApprovedRunActionError ||
error instanceof RangeError ||
error instanceof TypeError
) {
throw error;
}
throw new ApprovedRunActionRepositoryError();
}
}
private assertPlanBinding(
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
plan: Readonly<ReturnType<typeof normalizeApprovedRunCreationPlan>>,
): void {
if (
snapshot.dispatch.action.actionType !== APPROVED_RUN_ACTION_TYPE ||
snapshot.dispatch.action.actionRef !== plan.actionRef ||
snapshot.dispatch.projectId !== plan.projectId ||
snapshot.dispatch.action.actionDigest !==
digestApprovedRunCreationPlan(plan)
) {
throw new ApprovedRunActionBindingConflictError();
}
}
private async findReplay(
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
): Promise<Readonly<ApprovedRunReference> | null> {
const rows = await this.database.query<ApprovedRunReceiptRow>(
`SELECT schema_version, dispatch_id, approval_request_id, project_id,
action_type, action_digest, execution_attempt, execution_version,
started_at_ms, idempotency_key, outcome, result_code,
resource_type, resource_id, finished_at_ms, evidence_digest,
created_at_ms
FROM "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
WHERE dispatch_id = :dispatchId
LIMIT 2`,
{
type: QueryTypes.SELECT,
replacements: { dispatchId: snapshot.dispatch.id },
},
);
if (rows.length > 1) throw new ApprovedRunActionBindingConflictError();
if (rows.length === 0) {
const collisions = await this.database.query<{ id: string }>(
`SELECT id FROM "Runs"
WHERE project_id = :projectId AND idempotency_key = :idempotencyKey
LIMIT 1`,
{
type: QueryTypes.SELECT,
replacements: {
projectId: snapshot.dispatch.projectId,
idempotencyKey: snapshot.dispatch.id,
},
},
);
if (collisions.length > 0) {
throw new ApprovedRunActionBindingConflictError();
}
return null;
}
const receipt = rowToReceipt(rows[0]);
if (!receiptMatches(receipt, snapshot)) {
throw new ApprovedRunActionBindingConflictError();
}
const run = await this.runs.findRunById(receipt.resourceId);
const attempt = await this.runs.findLatestAttemptByRunId(
receipt.resourceId,
);
if (
!run ||
!attempt ||
run.projectId !== receipt.projectId ||
run.idempotencyKey !== receipt.idempotencyKey ||
run.requestId !== receipt.approvalRequestId ||
run.executionOwner !== 'runtime' ||
run.executionOrigin !== 'system' ||
run.triggerType !== 'approved_action'
) {
throw new ApprovedRunActionBindingConflictError();
}
return Object.freeze({ run, attempt });
}
}
@@ -0,0 +1,173 @@
import { QueryTypes, Sequelize } from 'sequelize';
import { APPROVED_RUN_ACTION_RECEIPT_TABLE } from '../../../migrations/0023-approved-run-action-receipts';
import {
APPROVED_RUN_ACTION_TYPE,
APPROVED_RUN_RECEIPT_RESULT_CODE,
InvalidApprovedRunActionError,
normalizeApprovedRunCreationReceipt,
type ApprovedRunCreationReceipt,
} from '../../domain/approvedRunAction';
import type {
ApprovedActionRecoveryEvidence,
ApprovedActionRecoveryEvidenceContext,
ApprovedActionRecoveryEvidenceProvider,
} from '../../ports/approvedActionRecoveryEvidenceProvider';
import { LegacySequelizeRunRepository } from './runRepository';
interface ReceiptRow {
schema_version: number;
dispatch_id: string;
approval_request_id: string;
project_id: string;
action_type: string;
action_digest: string;
execution_attempt: number;
execution_version: number;
started_at_ms: number;
idempotency_key: string;
outcome: string;
result_code: string;
resource_type: string;
resource_id: string;
finished_at_ms: number;
evidence_digest: string;
created_at_ms: number;
}
function normalizeRow(row: ReceiptRow): Readonly<ApprovedRunCreationReceipt> {
return normalizeApprovedRunCreationReceipt({
schemaVersion: row.schema_version as 1,
dispatchId: row.dispatch_id,
approvalRequestId: row.approval_request_id,
projectId: row.project_id,
actionType: row.action_type as typeof APPROVED_RUN_ACTION_TYPE,
actionDigest: row.action_digest,
executionAttempt: row.execution_attempt,
executionVersion: row.execution_version,
startedAtMs: row.started_at_ms,
idempotencyKey: row.idempotency_key,
outcome: row.outcome as 'succeeded',
resultCode: row.result_code as typeof APPROVED_RUN_RECEIPT_RESULT_CODE,
resourceType: row.resource_type as 'run',
resourceId: row.resource_id,
finishedAtMs: row.finished_at_ms,
evidenceDigest: row.evidence_digest,
createdAtMs: row.created_at_ms,
});
}
const CONFLICT: ApprovedActionRecoveryEvidence = Object.freeze({
finding: 'conflict',
resultCode: 'approved_run_receipt_conflict',
});
export class LegacySequelizeApprovedRunRecoveryEvidenceProvider
implements ApprovedActionRecoveryEvidenceProvider
{
readonly actionType = APPROVED_RUN_ACTION_TYPE;
readonly capability = 'automatic' as const;
private readonly runs: LegacySequelizeRunRepository;
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Approved Run recovery provider is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
this.runs = new LegacySequelizeRunRepository(database);
}
async inspect(
context: Readonly<ApprovedActionRecoveryEvidenceContext>,
): Promise<ApprovedActionRecoveryEvidence> {
const snapshot = context.snapshot;
const dispatch = snapshot.action.dispatch;
const execution = snapshot.action.execution;
if (
dispatch.action.actionType !== this.actionType ||
context.idempotencyKey !== dispatch.id ||
execution.dispatchId !== dispatch.id ||
execution.projectId !== dispatch.projectId ||
execution.startedAtMs === null
) {
return CONFLICT;
}
const rows = await this.database.query<ReceiptRow>(
`SELECT schema_version, dispatch_id, approval_request_id, project_id,
action_type, action_digest, execution_attempt, execution_version,
started_at_ms, idempotency_key, outcome, result_code,
resource_type, resource_id, finished_at_ms, evidence_digest,
created_at_ms
FROM "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
WHERE dispatch_id = :dispatchId
LIMIT 2`,
{
type: QueryTypes.SELECT,
replacements: { dispatchId: dispatch.id },
},
);
if (rows.length > 1) return CONFLICT;
if (rows.length === 0) {
const collisions = await this.database.query<{ id: string }>(
`SELECT id FROM "Runs"
WHERE project_id = :projectId AND idempotency_key = :idempotencyKey
LIMIT 1`,
{
type: QueryTypes.SELECT,
replacements: {
projectId: dispatch.projectId,
idempotencyKey: dispatch.id,
},
},
);
return collisions.length === 0
? {
finding: 'missing',
resultCode: 'approved_run_receipt_missing',
}
: CONFLICT;
}
let receipt: Readonly<ApprovedRunCreationReceipt>;
try {
receipt = normalizeRow(rows[0]);
} catch (error) {
if (error instanceof InvalidApprovedRunActionError) return CONFLICT;
throw error;
}
if (
receipt.dispatchId !== dispatch.id ||
receipt.approvalRequestId !== dispatch.approvalRequestId ||
receipt.projectId !== dispatch.projectId ||
receipt.actionType !== dispatch.action.actionType ||
receipt.actionDigest !== dispatch.action.actionDigest ||
receipt.executionAttempt !== execution.attemptCount ||
receipt.executionVersion > execution.version ||
receipt.startedAtMs !== execution.startedAtMs ||
receipt.idempotencyKey !== context.idempotencyKey
) {
return CONFLICT;
}
const run = await this.runs.findRunById(receipt.resourceId);
const attempt = await this.runs.findLatestAttemptByRunId(
receipt.resourceId,
);
if (
!run ||
!attempt ||
run.projectId !== receipt.projectId ||
run.idempotencyKey !== receipt.idempotencyKey ||
run.requestId !== receipt.approvalRequestId ||
run.executionOwner !== 'runtime' ||
run.executionOrigin !== 'system' ||
run.triggerType !== 'approved_action'
) {
return CONFLICT;
}
return {
finding: 'verified_succeeded',
resultCode: 'approved_run_receipt_verified',
evidenceDigest: receipt.evidenceDigest,
};
}
}
@@ -0,0 +1,735 @@
import {
DataTypes,
Model,
ModelStatic,
Sequelize,
Transaction,
UniqueConstraintError,
} from 'sequelize';
import {
RUN_EVENT_TABLE,
RUN_TABLE,
RUN_ATTEMPT_TABLE,
} from '../../../migrations/0002-run-schema';
import { RUN_CANCELLATION_DISPATCH_TABLE } from '../../../migrations/0005-run-cancellation-dispatch';
import {
CANCELLATION_DISPATCH_RESULTS,
CANCELLATION_DISPATCH_STATUSES,
type CancellationDispatchRecord,
type CancellationDispatchResult,
type CancellationDispatchStatus,
} from '../../domain/cancellationDispatch';
import {
CancellationDispatchBindingConflictError,
CancellationDispatchFenceRejectedError,
CancellationDispatchRepositoryError,
InvalidCancellationDispatchCommandError,
} from '../../domain/cancellationDispatchErrors';
import type { RunEventRecord, RunStatus } from '../../domain/run';
import type {
CancellationDispatchRepository,
ClaimCancellationDispatchCommand,
ClaimCancellationDispatchResult,
RecordCancellationDispatchResult,
RecordCancellationDispatchResultCommand,
} from '../../ports/cancellationDispatchRepository';
const ACTIVE_RUN_STATUSES: readonly RunStatus[] = [
'created',
'queued',
'dispatching',
'running',
'waiting_approval',
'retry_wait',
'lost',
];
const ACTIVE_ATTEMPT_STATUSES = ['claimed', 'starting', 'running'] as const;
const RETRYABLE_RESULTS: readonly CancellationDispatchResult[] = [
'controller_missing',
'handle_missing',
'dispatch_error',
];
const BLOCKING_RESULTS: readonly CancellationDispatchResult[] = [
'identity_mismatch',
'pid_mismatch',
'unsupported',
'invalid',
];
interface CancellationDispatchRow {
runId: string;
attemptId: string;
status: string;
version: number;
dispatchCount: number;
nextAttemptAtMs: number | null;
leaseOwner: string | null;
leaseToken: string | null;
leaseExpiresAtMs: number | null;
lastResult: string | null;
lastDispatchedAtMs: number | null;
createdAtMs: number;
updatedAtMs: number;
}
interface CancellationDispatchRunRow {
id: string;
executionOwner: string;
status: string;
version: number;
eventSequence: number;
cancelRequestedAtMs: number | null;
}
interface CancellationDispatchAttemptRow {
id: string;
runId: string;
status: string;
}
interface CancellationDispatchEventRow {
id: string;
runId: string;
sequence: number;
type: string;
dedupeKey: string;
actorType: string;
actorId: string;
attemptId: string;
payload: Readonly<Record<string, unknown>>;
createdAtMs: number;
}
interface CancellationDispatchInstance
extends Model<CancellationDispatchRow, CancellationDispatchRow>,
CancellationDispatchRow {}
interface CancellationDispatchRunInstance
extends Model<CancellationDispatchRunRow, CancellationDispatchRunRow>,
CancellationDispatchRunRow {}
interface CancellationDispatchAttemptInstance
extends Model<CancellationDispatchAttemptRow, CancellationDispatchAttemptRow>,
CancellationDispatchAttemptRow {}
interface CancellationDispatchEventInstance
extends Model<CancellationDispatchEventRow, CancellationDispatchEventRow>,
CancellationDispatchEventRow {}
function defineDispatchModel(
database: Sequelize,
): ModelStatic<CancellationDispatchInstance> {
return database.define<CancellationDispatchInstance>(
'Ql3CancellationDispatch',
{
runId: { field: 'run_id', type: DataTypes.STRING(36), primaryKey: true },
attemptId: {
field: 'attempt_id',
type: DataTypes.STRING(36),
allowNull: false,
},
status: { type: DataTypes.STRING(32), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false },
dispatchCount: {
field: 'dispatch_count',
type: DataTypes.INTEGER,
allowNull: false,
},
nextAttemptAtMs: {
field: 'next_attempt_at_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
leaseOwner: {
field: 'lease_owner',
type: DataTypes.STRING(128),
allowNull: true,
},
leaseToken: {
field: 'lease_token',
type: DataTypes.STRING(128),
allowNull: true,
},
leaseExpiresAtMs: {
field: 'lease_expires_at_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
lastResult: {
field: 'last_result',
type: DataTypes.STRING(64),
allowNull: true,
},
lastDispatchedAtMs: {
field: 'last_dispatched_at_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
createdAtMs: {
field: 'created_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
updatedAtMs: {
field: 'updated_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: RUN_CANCELLATION_DISPATCH_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function defineRunModel(
database: Sequelize,
): ModelStatic<CancellationDispatchRunInstance> {
return database.define<CancellationDispatchRunInstance>(
'Ql3CancellationDispatchRun',
{
id: { type: DataTypes.STRING(36), primaryKey: true },
executionOwner: {
field: 'execution_owner',
type: DataTypes.STRING(16),
allowNull: false,
},
status: { type: DataTypes.STRING(32), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false },
eventSequence: {
field: 'event_sequence',
type: DataTypes.INTEGER,
allowNull: false,
},
cancelRequestedAtMs: {
field: 'cancel_requested_at_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
},
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
);
}
function defineAttemptModel(
database: Sequelize,
): ModelStatic<CancellationDispatchAttemptInstance> {
return database.define<CancellationDispatchAttemptInstance>(
'Ql3CancellationDispatchAttempt',
{
id: { type: DataTypes.STRING(36), primaryKey: true },
runId: { field: 'run_id', type: DataTypes.STRING(36), allowNull: false },
status: { type: DataTypes.STRING(32), allowNull: false },
},
{ tableName: RUN_ATTEMPT_TABLE, timestamps: false, freezeTableName: true },
);
}
function defineEventModel(
database: Sequelize,
): ModelStatic<CancellationDispatchEventInstance> {
return database.define<CancellationDispatchEventInstance>(
'Ql3CancellationDispatchEvent',
{
id: { type: DataTypes.STRING(36), primaryKey: true },
runId: { field: 'run_id', type: DataTypes.STRING(36), allowNull: false },
sequence: { type: DataTypes.INTEGER, allowNull: false },
type: { type: DataTypes.STRING(128), allowNull: false },
dedupeKey: {
field: 'dedupe_key',
type: DataTypes.STRING(255),
allowNull: false,
},
actorType: {
field: 'actor_type',
type: DataTypes.STRING(64),
allowNull: false,
},
actorId: {
field: 'actor_id',
type: DataTypes.STRING(255),
allowNull: false,
},
attemptId: {
field: 'attempt_id',
type: DataTypes.STRING(36),
allowNull: false,
},
payload: { type: DataTypes.JSON, allowNull: false },
createdAtMs: {
field: 'created_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{ tableName: RUN_EVENT_TABLE, timestamps: false, freezeTableName: true },
);
}
function assertId(name: string, value: string, maxLength = 36): void {
if (!value || value.length > maxLength) {
throw new InvalidCancellationDispatchCommandError(
`${name} must be between 1 and ${maxLength} characters`,
);
}
}
function assertTimestamp(name: string, value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new InvalidCancellationDispatchCommandError(
`${name} must be a non-negative safe integer`,
);
}
}
function assertClaim(command: ClaimCancellationDispatchCommand): void {
assertId('runId', command.runId);
assertId('attemptId', command.attemptId);
assertId('owner', command.owner, 128);
assertId('leaseToken', command.leaseToken, 128);
assertTimestamp('requestedAtMs', command.requestedAtMs);
assertTimestamp('nowMs', command.nowMs);
if (
!Number.isSafeInteger(command.leaseDurationMs) ||
command.leaseDurationMs < 1
) {
throw new InvalidCancellationDispatchCommandError(
'leaseDurationMs must be a positive safe integer',
);
}
if (!Number.isSafeInteger(command.nowMs + command.leaseDurationMs)) {
throw new InvalidCancellationDispatchCommandError(
'lease expiry exceeds the supported range',
);
}
}
function assertRecordResult(
command: RecordCancellationDispatchResultCommand,
): void {
assertId('runId', command.runId);
assertId('attemptId', command.attemptId);
assertId('owner', command.owner, 128);
assertId('leaseToken', command.leaseToken, 128);
assertId('eventId', command.eventId);
assertTimestamp('atMs', command.atMs);
if (
!Number.isSafeInteger(command.expectedVersion) ||
command.expectedVersion < 1
) {
throw new InvalidCancellationDispatchCommandError(
'expectedVersion must be a positive safe integer',
);
}
if (!CANCELLATION_DISPATCH_RESULTS.includes(command.result)) {
throw new InvalidCancellationDispatchCommandError(
'result is not supported',
);
}
if (RETRYABLE_RESULTS.includes(command.result)) {
if (
command.nextAttemptAtMs === undefined ||
!Number.isSafeInteger(command.nextAttemptAtMs) ||
command.nextAttemptAtMs <= command.atMs
) {
throw new InvalidCancellationDispatchCommandError(
'retryable results require nextAttemptAtMs greater than atMs',
);
}
} else if (command.nextAttemptAtMs !== undefined) {
throw new InvalidCancellationDispatchCommandError(
'terminal results must not include nextAttemptAtMs',
);
}
}
function rowToDispatch(
row: CancellationDispatchRow,
): CancellationDispatchRecord {
if (
!CANCELLATION_DISPATCH_STATUSES.includes(
row.status as CancellationDispatchStatus,
)
) {
throw new CancellationDispatchRepositoryError(
new Error(`Unsupported cancellation dispatch status: ${row.status}`),
);
}
if (
row.lastResult !== null &&
!CANCELLATION_DISPATCH_RESULTS.includes(
row.lastResult as CancellationDispatchResult,
)
) {
throw new CancellationDispatchRepositoryError(
new Error(`Unsupported cancellation dispatch result: ${row.lastResult}`),
);
}
for (const [name, value] of [
['version', row.version],
['dispatchCount', row.dispatchCount],
['createdAtMs', row.createdAtMs],
['updatedAtMs', row.updatedAtMs],
['nextAttemptAtMs', row.nextAttemptAtMs],
['leaseExpiresAtMs', row.leaseExpiresAtMs],
['lastDispatchedAtMs', row.lastDispatchedAtMs],
] as const) {
if (
value !== null &&
(!Number.isSafeInteger(Number(value)) || Number(value) < 0)
) {
throw new CancellationDispatchRepositoryError(
new Error(`Invalid cancellation dispatch ${name}`),
);
}
}
const status = row.status as CancellationDispatchStatus;
const hasCompleteLease =
row.leaseOwner !== null &&
row.leaseToken !== null &&
row.leaseExpiresAtMs !== null;
const hasAnyLease =
row.leaseOwner !== null ||
row.leaseToken !== null ||
row.leaseExpiresAtMs !== null;
if (
(status === 'leased' && !hasCompleteLease) ||
(status !== 'leased' && hasAnyLease) ||
((status === 'pending' || status === 'retry_wait') &&
row.nextAttemptAtMs === null) ||
((status === 'dispatched' || status === 'blocked' || status === 'leased') &&
row.nextAttemptAtMs !== null) ||
((status === 'dispatched' || status === 'blocked') &&
row.lastResult === null)
) {
throw new CancellationDispatchRepositoryError(
new Error('Cancellation dispatch lease/status fields are inconsistent'),
);
}
return {
runId: row.runId,
attemptId: row.attemptId,
status,
version: Number(row.version),
dispatchCount: Number(row.dispatchCount),
createdAtMs: Number(row.createdAtMs),
updatedAtMs: Number(row.updatedAtMs),
...(row.nextAttemptAtMs === null
? {}
: { nextAttemptAtMs: Number(row.nextAttemptAtMs) }),
...(row.leaseOwner === null ? {} : { leaseOwner: row.leaseOwner }),
...(row.leaseToken === null ? {} : { leaseToken: row.leaseToken }),
...(row.leaseExpiresAtMs === null
? {}
: { leaseExpiresAtMs: Number(row.leaseExpiresAtMs) }),
...(row.lastResult === null
? {}
: { lastResult: row.lastResult as CancellationDispatchResult }),
...(row.lastDispatchedAtMs === null
? {}
: { lastDispatchedAtMs: Number(row.lastDispatchedAtMs) }),
};
}
function resultState(result: CancellationDispatchResult): {
status: CancellationDispatchStatus;
eventType: string;
} {
if (RETRYABLE_RESULTS.includes(result)) {
return { status: 'retry_wait', eventType: 'run.cancel_dispatch_failed' };
}
if (BLOCKING_RESULTS.includes(result)) {
return { status: 'blocked', eventType: 'run.cancel_dispatch_blocked' };
}
return { status: 'dispatched', eventType: 'run.cancel_dispatched' };
}
function withoutScheduleAndLease(
dispatch: CancellationDispatchRecord,
): Omit<
CancellationDispatchRecord,
'nextAttemptAtMs' | 'leaseOwner' | 'leaseToken' | 'leaseExpiresAtMs'
> {
const {
nextAttemptAtMs: _nextAttemptAtMs,
leaseOwner: _leaseOwner,
leaseToken: _leaseToken,
leaseExpiresAtMs: _leaseExpiresAtMs,
...rest
} = dispatch;
return rest;
}
export class LegacySequelizeCancellationDispatchRepository
implements CancellationDispatchRepository
{
private readonly dispatch: ModelStatic<CancellationDispatchInstance>;
private readonly run: ModelStatic<CancellationDispatchRunInstance>;
private readonly attempt: ModelStatic<CancellationDispatchAttemptInstance>;
private readonly event: ModelStatic<CancellationDispatchEventInstance>;
constructor(private readonly database: Sequelize) {
this.dispatch = defineDispatchModel(database);
this.run = defineRunModel(database);
this.attempt = defineAttemptModel(database);
this.event = defineEventModel(database);
}
async findByRunId(runId: string): Promise<CancellationDispatchRecord | null> {
assertId('runId', runId);
const row = (await this.dispatch.findByPk(runId, {
raw: true,
})) as unknown as CancellationDispatchRow | null;
return row === null ? null : rowToDispatch(row);
}
async claim(
command: ClaimCancellationDispatchCommand,
): Promise<ClaimCancellationDispatchResult> {
assertClaim(command);
return this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const [run, attempt] = await Promise.all([
this.run.findByPk(command.runId, { raw: true, transaction }),
this.attempt.findByPk(command.attemptId, { raw: true, transaction }),
]);
const runRow = run as unknown as CancellationDispatchRunRow | null;
const attemptRow =
attempt as unknown as CancellationDispatchAttemptRow | null;
if (
runRow === null ||
attemptRow === null ||
runRow.executionOwner !== 'runtime' ||
!ACTIVE_RUN_STATUSES.includes(runRow.status as RunStatus) ||
runRow.cancelRequestedAtMs === null ||
Number(runRow.cancelRequestedAtMs) !== command.requestedAtMs ||
attemptRow.runId !== command.runId ||
!ACTIVE_ATTEMPT_STATUSES.includes(
attemptRow.status as (typeof ACTIVE_ATTEMPT_STATUSES)[number],
)
) {
return { status: 'not_eligible' as const };
}
let row = (await this.dispatch.findByPk(command.runId, {
raw: true,
transaction,
})) as unknown as CancellationDispatchRow | null;
if (row === null) {
try {
const created = await this.dispatch.create(
{
runId: command.runId,
attemptId: command.attemptId,
status: 'pending',
version: 0,
dispatchCount: 0,
nextAttemptAtMs: command.requestedAtMs,
leaseOwner: null,
leaseToken: null,
leaseExpiresAtMs: null,
lastResult: null,
lastDispatchedAtMs: null,
createdAtMs: command.nowMs,
updatedAtMs: command.nowMs,
},
{ transaction },
);
row = created.get({ plain: true }) as CancellationDispatchRow;
} catch (error) {
if (error instanceof UniqueConstraintError) {
throw new CancellationDispatchRepositoryError(error);
}
throw error;
}
}
if (row.attemptId !== command.attemptId) {
throw new CancellationDispatchBindingConflictError(
command.runId,
command.attemptId,
);
}
const dispatch = rowToDispatch(row);
if (dispatch.status === 'dispatched' || dispatch.status === 'blocked') {
return { status: dispatch.status, dispatch };
}
if (
dispatch.status === 'leased' &&
dispatch.leaseExpiresAtMs !== undefined &&
dispatch.leaseExpiresAtMs > command.nowMs
) {
return { status: 'leased', dispatch };
}
if (
dispatch.status !== 'leased' &&
dispatch.nextAttemptAtMs !== undefined &&
dispatch.nextAttemptAtMs > command.nowMs
) {
return { status: 'not_due', dispatch };
}
const nextVersion = dispatch.version + 1;
const nextCount = dispatch.dispatchCount + 1;
const leaseExpiresAtMs = command.nowMs + command.leaseDurationMs;
const [affected] = await this.dispatch.update(
{
status: 'leased',
version: nextVersion,
dispatchCount: nextCount,
nextAttemptAtMs: null,
leaseOwner: command.owner,
leaseToken: command.leaseToken,
leaseExpiresAtMs,
updatedAtMs: command.nowMs,
},
{
where: { runId: command.runId, version: dispatch.version },
transaction,
},
);
if (affected !== 1) {
throw new CancellationDispatchFenceRejectedError(command.runId);
}
return {
status: 'claimed',
dispatch: {
...withoutScheduleAndLease(dispatch),
status: 'leased',
version: nextVersion,
dispatchCount: nextCount,
leaseOwner: command.owner,
leaseToken: command.leaseToken,
leaseExpiresAtMs,
updatedAtMs: command.nowMs,
},
};
},
);
}
async recordResult(
command: RecordCancellationDispatchResultCommand,
): Promise<RecordCancellationDispatchResult> {
assertRecordResult(command);
return this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const row = (await this.dispatch.findByPk(command.runId, {
raw: true,
transaction,
})) as unknown as CancellationDispatchRow | null;
if (
row === null ||
row.attemptId !== command.attemptId ||
row.status !== 'leased' ||
row.version !== command.expectedVersion ||
row.leaseOwner !== command.owner ||
row.leaseToken !== command.leaseToken
) {
throw new CancellationDispatchFenceRejectedError(command.runId);
}
const run = (await this.run.findByPk(command.runId, {
raw: true,
transaction,
})) as unknown as CancellationDispatchRunRow | null;
if (run === null) {
throw new CancellationDispatchRepositoryError(
new Error('Run disappeared while recording cancellation result'),
);
}
const state = resultState(command.result);
const controllerInvoked = ![
'controller_missing',
'handle_missing',
].includes(command.result);
const nextVersion = row.version + 1;
const nextSequence = Number(run.eventSequence) + 1;
const [runAffected] = await this.run.update(
{ version: Number(run.version) + 1, eventSequence: nextSequence },
{ where: { id: command.runId, version: run.version }, transaction },
);
if (runAffected !== 1) {
throw new CancellationDispatchFenceRejectedError(command.runId);
}
const [dispatchAffected] = await this.dispatch.update(
{
status: state.status,
version: nextVersion,
nextAttemptAtMs: command.nextAttemptAtMs ?? null,
leaseOwner: null,
leaseToken: null,
leaseExpiresAtMs: null,
lastResult: command.result,
lastDispatchedAtMs: controllerInvoked
? command.atMs
: row.lastDispatchedAtMs,
updatedAtMs: command.atMs,
},
{
where: {
runId: command.runId,
attemptId: command.attemptId,
status: 'leased',
version: command.expectedVersion,
leaseOwner: command.owner,
leaseToken: command.leaseToken,
},
transaction,
},
);
if (dispatchAffected !== 1) {
throw new CancellationDispatchFenceRejectedError(command.runId);
}
const event: RunEventRecord = {
id: command.eventId,
runId: command.runId,
sequence: nextSequence,
type: state.eventType,
dedupeKey: `cancel-dispatch:${command.attemptId}:${row.dispatchCount}`,
actorType: 'worker',
actorId: command.owner,
attemptId: command.attemptId,
payload: {
attempt_id: command.attemptId,
dispatch_count: row.dispatchCount,
result: command.result,
},
createdAtMs: command.atMs,
};
await this.event.create(
{
id: event.id,
runId: event.runId,
sequence: event.sequence,
type: event.type,
dedupeKey: event.dedupeKey!,
actorType: event.actorType,
actorId: event.actorId!,
attemptId: event.attemptId!,
payload: event.payload,
createdAtMs: event.createdAtMs,
},
{ transaction },
);
return {
dispatch: {
...withoutScheduleAndLease(rowToDispatch(row)),
status: state.status,
version: nextVersion,
...(command.nextAttemptAtMs === undefined
? {}
: { nextAttemptAtMs: command.nextAttemptAtMs }),
lastResult: command.result,
...(controllerInvoked
? { lastDispatchedAtMs: command.atMs }
: row.lastDispatchedAtMs === null
? {}
: { lastDispatchedAtMs: Number(row.lastDispatchedAtMs) }),
updatedAtMs: command.atMs,
},
event,
};
},
);
}
}
@@ -0,0 +1,322 @@
import {
DataTypes,
Model,
ModelStatic,
Op,
Sequelize,
UniqueConstraintError,
type WhereOptions,
} from 'sequelize';
import { RUN_ATTEMPT_TABLE } from '../../../migrations/0002-run-schema';
import { COMPLETION_RECEIPT_JOURNAL_TABLE } from '../../../migrations/0007-completion-receipt-journal';
import {
COMPLETION_RECEIPT_JOURNAL_STATES,
type CompletionReceiptJournalCandidate,
type CompletionReceiptJournalCursor,
type CompletionReceiptJournalRecord,
type CompletionReceiptJournalState,
} from '../../domain/completionReceiptJournal';
import { assertCompletionReceiptId } from '../../domain/completionReceipt';
import type { RunAttemptStatus } from '../../domain/run';
import {
MAX_COMPLETION_RECEIPT_JOURNAL_BATCH_SIZE,
type CompletionReceiptJournal,
type QuarantineCompletionReceiptCommand,
type RegisterCompletionReceiptCommand,
} from '../../ports/completionReceiptJournal';
interface JournalRow {
attemptId: string;
runId: string;
state: string;
quarantineRef: string | null;
purgeAfterMs: number | null;
registeredAtMs: number;
updatedAtMs: number;
}
interface AttemptRow {
id: string;
status: string;
executorType: string;
finishedAtMs: number | null;
}
interface JournalInstance extends Model<JournalRow, JournalRow>, JournalRow {}
interface AttemptInstance extends Model<AttemptRow, AttemptRow>, AttemptRow {}
function defineJournalModel(database: Sequelize): ModelStatic<JournalInstance> {
return database.define<JournalInstance>(
'Ql3CompletionReceiptJournal',
{
attemptId: {
field: 'attempt_id',
type: DataTypes.STRING(36),
primaryKey: true,
},
runId: {
field: 'run_id',
type: DataTypes.STRING(36),
allowNull: false,
},
state: { type: DataTypes.STRING(16), allowNull: false },
quarantineRef: {
field: 'quarantine_ref',
type: DataTypes.STRING(255),
allowNull: true,
},
purgeAfterMs: {
field: 'purge_after_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
registeredAtMs: {
field: 'registered_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
updatedAtMs: {
field: 'updated_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: COMPLETION_RECEIPT_JOURNAL_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function defineAttemptModel(database: Sequelize): ModelStatic<AttemptInstance> {
return database.define<AttemptInstance>(
'Ql3CompletionReceiptJournalAttempt',
{
id: { type: DataTypes.STRING(36), primaryKey: true },
status: { type: DataTypes.STRING(32), allowNull: false },
executorType: {
field: 'executor_type',
type: DataTypes.STRING(64),
allowNull: false,
},
finishedAtMs: {
field: 'finished_at_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
},
{ tableName: RUN_ATTEMPT_TABLE, timestamps: false, freezeTableName: true },
);
}
function assertNonNegativeTimestamp(name: string, value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new RangeError(`${name} must be a non-negative safe integer`);
}
}
function assertCursor(cursor: CompletionReceiptJournalCursor): void {
assertNonNegativeTimestamp('cursor.updatedAtMs', cursor.updatedAtMs);
assertCompletionReceiptId(cursor.attemptId, 'attemptId');
}
function assertQuarantineRef(value: string): void {
if (
value.length < 1 ||
value.length > 255 ||
!value.startsWith('.quarantine/') ||
value.includes('..') ||
value.includes('\\') ||
value.includes('\0')
) {
throw new TypeError('quarantineRef is invalid');
}
}
function toRecord(row: JournalRow): CompletionReceiptJournalRecord {
if (!COMPLETION_RECEIPT_JOURNAL_STATES.includes(row.state as never)) {
throw new Error('Completion receipt journal state is corrupt');
}
return {
attemptId: row.attemptId,
runId: row.runId,
state: row.state as CompletionReceiptJournalState,
registeredAtMs: Number(row.registeredAtMs),
updatedAtMs: Number(row.updatedAtMs),
...(row.quarantineRef === null ? {} : { quarantineRef: row.quarantineRef }),
...(row.purgeAfterMs === null
? {}
: { purgeAfterMs: Number(row.purgeAfterMs) }),
};
}
export class LegacySequelizeCompletionReceiptJournal
implements CompletionReceiptJournal
{
private readonly journal: ModelStatic<JournalInstance>;
private readonly attempt: ModelStatic<AttemptInstance>;
constructor(database: Sequelize) {
this.journal = defineJournalModel(database);
this.attempt = defineAttemptModel(database);
}
async register(command: RegisterCompletionReceiptCommand): Promise<void> {
assertCompletionReceiptId(command.attemptId, 'attemptId');
assertCompletionReceiptId(command.runId, 'runId');
assertNonNegativeTimestamp('registeredAtMs', command.registeredAtMs);
const values: JournalRow = {
attemptId: command.attemptId,
runId: command.runId,
state: 'pending',
quarantineRef: null,
purgeAfterMs: null,
registeredAtMs: command.registeredAtMs,
updatedAtMs: command.registeredAtMs,
};
try {
await this.journal.create(values);
return;
} catch (error) {
if (!(error instanceof UniqueConstraintError)) throw error;
}
const current = (await this.journal.findByPk(command.attemptId, {
raw: true,
})) as unknown as JournalRow | null;
if (
!current ||
current.runId !== command.runId ||
Number(current.registeredAtMs) !== command.registeredAtMs
) {
throw new Error('Completion receipt journal registration conflicts');
}
}
async markQuarantined(
command: QuarantineCompletionReceiptCommand,
): Promise<void> {
assertCompletionReceiptId(command.attemptId, 'attemptId');
assertQuarantineRef(command.quarantineRef);
assertNonNegativeTimestamp('updatedAtMs', command.updatedAtMs);
assertNonNegativeTimestamp('purgeAfterMs', command.purgeAfterMs);
if (command.purgeAfterMs < command.updatedAtMs) {
throw new RangeError('purgeAfterMs must not precede updatedAtMs');
}
const [updated] = await this.journal.update(
{
state: 'quarantined',
quarantineRef: command.quarantineRef,
purgeAfterMs: command.purgeAfterMs,
updatedAtMs: command.updatedAtMs,
},
{ where: { attemptId: command.attemptId, state: 'pending' } },
);
if (updated === 1) return;
const current = (await this.journal.findByPk(command.attemptId, {
raw: true,
})) as unknown as JournalRow | null;
if (
current?.state === 'quarantined' &&
current.quarantineRef === command.quarantineRef &&
Number(current.purgeAfterMs) === command.purgeAfterMs
) {
return;
}
throw new Error('Completion receipt journal quarantine transition failed');
}
async resolve(attemptId: string): Promise<boolean> {
assertCompletionReceiptId(attemptId, 'attemptId');
return (await this.journal.destroy({ where: { attemptId } })) === 1;
}
async listCandidates({
observedAtMs,
cursor,
limit = 32,
}: {
observedAtMs: number;
cursor?: CompletionReceiptJournalCursor;
limit?: number;
}) {
assertNonNegativeTimestamp('observedAtMs', observedAtMs);
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_COMPLETION_RECEIPT_JOURNAL_BATCH_SIZE
) {
throw new RangeError(
'limit must be between 1 and MAX_COMPLETION_RECEIPT_JOURNAL_BATCH_SIZE',
);
}
if (cursor) assertCursor(cursor);
const eligible: WhereOptions<JournalRow> = {
[Op.or]: [
{ state: 'pending' },
{
state: 'quarantined',
purgeAfterMs: { [Op.lte]: observedAtMs },
},
],
};
const afterCursor: WhereOptions<JournalRow> | undefined = cursor
? {
[Op.or]: [
{ updatedAtMs: { [Op.gt]: cursor.updatedAtMs } },
{
updatedAtMs: cursor.updatedAtMs,
attemptId: { [Op.gt]: cursor.attemptId },
},
],
}
: undefined;
const where: WhereOptions<JournalRow> = afterCursor
? { [Op.and]: [eligible, afterCursor] }
: eligible;
const rows = (await this.journal.findAll({
where,
order: [
['updatedAtMs', 'ASC'],
['attemptId', 'ASC'],
],
limit: limit + 1,
raw: true,
})) as unknown as JournalRow[];
const truncated = rows.length > limit;
const bounded = rows.slice(0, limit);
if (bounded.length === 0) return { candidates: [], truncated: false };
const attempts = (await this.attempt.findAll({
where: { id: { [Op.in]: bounded.map((row) => row.attemptId) } },
raw: true,
})) as unknown as AttemptRow[];
const attemptById = new Map(attempts.map((row) => [row.id, row]));
const candidates: CompletionReceiptJournalCandidate[] = bounded.map(
(row) => {
const attempt = attemptById.get(row.attemptId);
if (!attempt) {
throw new Error('Completion receipt journal Attempt is missing');
}
return {
...toRecord(row),
attemptStatus: attempt.status as RunAttemptStatus,
executorType: attempt.executorType,
...(attempt.finishedAtMs === null
? {}
: { finishedAtMs: Number(attempt.finishedAtMs) }),
};
},
);
const last = bounded[bounded.length - 1];
return {
candidates,
truncated,
nextCursor: {
updatedAtMs: Number(last.updatedAtMs),
attemptId: last.attemptId,
},
};
}
}
@@ -0,0 +1,116 @@
import { QueryTypes, Sequelize } from 'sequelize';
import {
IDENTITY_AUTHENTICATION_BINDING_TABLE,
IDENTITY_SUBJECT_TABLE,
} from '../../../migrations/0019-identity-directory';
import {
IdentityDirectoryUnavailableError,
assertIdentityProvider,
assertIdentityProviderSubject,
normalizeIdentityAuthenticationBindingRecord,
normalizeIdentitySubjectRecord,
} from '../../domain/identityDirectory';
import type { PolicySubject } from '../../domain/projectPolicy';
import type { IdentityDirectoryRepository } from '../../ports/identityDirectoryRepository';
interface IdentityAuthenticationRow {
provider: string;
provider_subject: string;
binding_version: number;
binding_state: string;
binding_subject_id: string;
binding_created_at_ms: number | string;
subject_id: string | null;
subject_type: string | null;
subject_status: string | null;
subject_version: number | null;
subject_created_at_ms: number | string | null;
subject_updated_at_ms: number | string | null;
}
export class LegacySequelizeIdentityDirectoryRepository
implements IdentityDirectoryRepository
{
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Identity directory repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
}
async resolveAuthenticationSubject(
provider: string,
providerSubject: string,
): Promise<Readonly<PolicySubject> | null> {
assertIdentityProvider(provider);
assertIdentityProviderSubject(providerSubject);
try {
const rows = await this.database.query<IdentityAuthenticationRow>(
`SELECT binding.provider AS provider,
binding.provider_subject AS provider_subject,
binding.version AS binding_version,
binding.state AS binding_state,
binding.subject_id AS binding_subject_id,
binding.created_at_ms AS binding_created_at_ms,
subject.id AS subject_id,
subject.type AS subject_type,
subject.status AS subject_status,
subject.version AS subject_version,
subject.created_at_ms AS subject_created_at_ms,
subject.updated_at_ms AS subject_updated_at_ms
FROM "${IDENTITY_AUTHENTICATION_BINDING_TABLE}" AS binding
LEFT JOIN "${IDENTITY_SUBJECT_TABLE}" AS subject
ON subject.id = binding.subject_id
WHERE binding.provider = :provider
AND binding.provider_subject = :providerSubject
AND binding.version = (
SELECT MAX(current.version)
FROM "${IDENTITY_AUTHENTICATION_BINDING_TABLE}" AS current
WHERE current.provider = binding.provider
AND current.provider_subject = binding.provider_subject
)
LIMIT 2`,
{
type: QueryTypes.SELECT,
replacements: { provider, providerSubject },
},
);
if (rows.length === 0) return null;
if (rows.length !== 1) throw new IdentityDirectoryUnavailableError();
const row = rows[0];
const binding = normalizeIdentityAuthenticationBindingRecord({
provider: row.provider,
providerSubject: row.provider_subject,
version: Number(row.binding_version),
state: row.binding_state as 'active' | 'revoked',
subjectId: row.binding_subject_id,
createdAtMs: Number(row.binding_created_at_ms),
});
const subject = normalizeIdentitySubjectRecord({
subject: {
type: row.subject_type as PolicySubject['type'],
id: row.subject_id!,
},
status: row.subject_status as 'active' | 'disabled',
version: Number(row.subject_version),
createdAtMs: Number(row.subject_created_at_ms),
updatedAtMs: Number(row.subject_updated_at_ms),
});
if (binding.subjectId !== subject.subject.id) {
throw new IdentityDirectoryUnavailableError();
}
if (
binding.state !== 'active' ||
subject.status !== 'active' ||
subject.subject.type !== 'user'
) {
return null;
}
return subject.subject;
} catch (error) {
if (error instanceof IdentityDirectoryUnavailableError) throw error;
throw new IdentityDirectoryUnavailableError();
}
}
}
@@ -0,0 +1,136 @@
import { QueryTypes, Sequelize } from 'sequelize';
import {
RUN_ATTEMPT_TABLE,
RUN_TABLE,
} from '../../../migrations/0002-run-schema';
import { LOCAL_ARTIFACT_RETENTION_TABLE } from '../../../migrations/0015-local-artifact-retention';
import {
normalizeLocalArtifactReadMetadata,
type LocalArtifactReadMetadata,
} from '../../domain/artifactRead';
import type { LocalArtifactReadMetadataRepository } from '../../ports/localArtifactReadMetadataRepository';
interface ArtifactMetadataRow {
project_id: string;
run_id: string;
attempt_id: string;
attempt_finished_at_ms: number | string | null;
log_artifact_id: string;
retention_log_artifact_id: string | null;
retention_disposition: string | null;
retention_finished_at_ms: number | string | null;
retention_eligible_at_ms: number | string | null;
retention_bytes_reclaimed: number | string | null;
retention_recorded_at_ms: number | string | null;
}
export class CorruptLocalArtifactReadMetadataError extends Error {
constructor() {
super('Local Artifact read metadata is corrupt or ambiguous');
this.name = 'CorruptLocalArtifactReadMetadataError';
}
}
function rowToMetadata(
row: ArtifactMetadataRow,
): Readonly<LocalArtifactReadMetadata> {
const retentionValues = [
row.retention_log_artifact_id,
row.retention_disposition,
row.retention_finished_at_ms,
row.retention_eligible_at_ms,
row.retention_bytes_reclaimed,
row.retention_recorded_at_ms,
];
const hasRetention = retentionValues.every((value) => value !== null);
if (!hasRetention && retentionValues.some((value) => value !== null)) {
throw new CorruptLocalArtifactReadMetadataError();
}
if (hasRetention && row.retention_log_artifact_id !== row.log_artifact_id) {
throw new CorruptLocalArtifactReadMetadataError();
}
if (
hasRetention &&
(row.attempt_finished_at_ms === null ||
Number(row.retention_finished_at_ms) !==
Number(row.attempt_finished_at_ms))
) {
throw new CorruptLocalArtifactReadMetadataError();
}
try {
return normalizeLocalArtifactReadMetadata({
projectId: row.project_id,
runId: row.run_id,
attemptId: row.attempt_id,
logArtifactId: row.log_artifact_id,
...(hasRetention
? {
retention: {
disposition: row.retention_disposition as NonNullable<
LocalArtifactReadMetadata['retention']
>['disposition'],
finishedAtMs: Number(row.retention_finished_at_ms),
eligibleAtMs: Number(row.retention_eligible_at_ms),
bytesReclaimed: Number(row.retention_bytes_reclaimed),
recordedAtMs: Number(row.retention_recorded_at_ms),
},
}
: {}),
});
} catch (error) {
if (error instanceof CorruptLocalArtifactReadMetadataError) throw error;
throw new CorruptLocalArtifactReadMetadataError();
}
}
export class LegacySequelizeLocalArtifactReadMetadataRepository
implements LocalArtifactReadMetadataRepository
{
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Local Artifact read metadata repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
}
async find({
projectId,
runId,
logArtifactId,
}: Parameters<
LocalArtifactReadMetadataRepository['find']
>[0]): Promise<Readonly<LocalArtifactReadMetadata> | null> {
const rows = await this.database.query<ArtifactMetadataRow>(
`SELECT run.project_id,
run.id AS run_id,
attempt.id AS attempt_id,
attempt.finished_at_ms AS attempt_finished_at_ms,
attempt.log_artifact_id,
retained.log_artifact_id AS retention_log_artifact_id,
retained.disposition AS retention_disposition,
retained.finished_at_ms AS retention_finished_at_ms,
retained.eligible_at_ms AS retention_eligible_at_ms,
retained.bytes_reclaimed AS retention_bytes_reclaimed,
retained.recorded_at_ms AS retention_recorded_at_ms
FROM "${RUN_TABLE}" AS run
JOIN "${RUN_ATTEMPT_TABLE}" AS attempt ON attempt.run_id = run.id
LEFT JOIN "${LOCAL_ARTIFACT_RETENTION_TABLE}" AS retained
ON retained.attempt_id = attempt.id
WHERE run.project_id = :projectId
AND run.id = :runId
AND run.execution_owner = 'runtime'
AND attempt.executor_type = 'local_process'
AND attempt.log_artifact_id = :logArtifactId
AND attempt.log_artifact_id LIKE 'local-%'
LIMIT 2`,
{
type: QueryTypes.SELECT,
replacements: { projectId, runId, logArtifactId },
},
);
if (rows.length === 0) return null;
if (rows.length !== 1) throw new CorruptLocalArtifactReadMetadataError();
return rowToMetadata(rows[0]);
}
}
@@ -0,0 +1,132 @@
import {
DataTypes,
Model,
type ModelStatic,
Sequelize,
UniqueConstraintError,
} from 'sequelize';
import { LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE } from '../../../migrations/0016-local-artifact-maintenance-cursor';
import {
normalizeLocalArtifactRetentionCursor,
assertLocalArtifactRetentionTimestamp,
} from '../../domain/localArtifactRetention';
import {
normalizeLocalArtifactRetentionCheckpoint,
type LocalArtifactRetentionCheckpoint,
} from '../../domain/localArtifactRetentionCheckpoint';
import type { LocalArtifactRetentionCheckpointStore } from '../../ports/localArtifactRetentionCheckpointStore';
const RETENTION_SCOPE = 'retention';
interface CursorRow {
scope: string;
cursorFinishedAtMs: number | string | null;
cursorAttemptId: string | null;
version: number | string;
updatedAtMs: number | string;
}
interface CursorInstance extends Model<CursorRow, CursorRow>, CursorRow {}
function defineCursorModel(database: Sequelize): ModelStatic<CursorInstance> {
return database.define<CursorInstance>(
'Ql3LocalArtifactMaintenanceCursor',
{
scope: { type: DataTypes.STRING(32), allowNull: false, primaryKey: true },
cursorFinishedAtMs: {
field: 'cursor_finished_at_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
cursorAttemptId: {
field: 'cursor_attempt_id',
type: DataTypes.STRING(36),
allowNull: true,
},
version: { type: DataTypes.BIGINT, allowNull: false },
updatedAtMs: {
field: 'updated_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function rowToCheckpoint(
row: CursorRow,
): Readonly<LocalArtifactRetentionCheckpoint> {
const finishedAtMs =
row.cursorFinishedAtMs === null ? null : Number(row.cursorFinishedAtMs);
const attemptId = row.cursorAttemptId;
if ((finishedAtMs === null) !== (attemptId === null)) {
throw new TypeError('Local Artifact retention cursor row is corrupt');
}
return normalizeLocalArtifactRetentionCheckpoint({
version: Number(row.version),
...(finishedAtMs === null || attemptId === null
? {}
: { cursor: { finishedAtMs, attemptId } }),
});
}
export class LegacySequelizeLocalArtifactRetentionCheckpointStore
implements LocalArtifactRetentionCheckpointStore
{
private readonly cursors: ModelStatic<CursorInstance>;
constructor(database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Local Artifact retention checkpoint store is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
this.cursors = defineCursorModel(database);
}
async load(): Promise<Readonly<LocalArtifactRetentionCheckpoint>> {
const row = await this.cursors.findByPk(RETENTION_SCOPE, { raw: true });
return row
? rowToCheckpoint(row)
: normalizeLocalArtifactRetentionCheckpoint({ version: 0 });
}
async compareAndSet({
expectedVersion,
cursor,
updatedAtMs,
}: Parameters<
LocalArtifactRetentionCheckpointStore['compareAndSet']
>[0]): Promise<boolean> {
const checkpoint = normalizeLocalArtifactRetentionCheckpoint({
version: expectedVersion,
...(cursor ? { cursor } : {}),
});
assertLocalArtifactRetentionTimestamp('updatedAtMs', updatedAtMs);
const next = {
scope: RETENTION_SCOPE,
cursorFinishedAtMs: checkpoint.cursor?.finishedAtMs ?? null,
cursorAttemptId: checkpoint.cursor?.attemptId ?? null,
version: checkpoint.version + 1,
updatedAtMs,
};
if (checkpoint.version === 0) {
try {
await this.cursors.create(next);
return true;
} catch (error) {
if (error instanceof UniqueConstraintError) return false;
throw error;
}
}
const [updated] = await this.cursors.update(next, {
where: { scope: RETENTION_SCOPE, version: checkpoint.version },
});
return updated === 1;
}
}
@@ -0,0 +1,252 @@
import {
DataTypes,
Model,
ModelStatic,
QueryTypes,
Sequelize,
UniqueConstraintError,
} from 'sequelize';
import { COMPLETION_RECEIPT_JOURNAL_TABLE } from '../../../migrations/0007-completion-receipt-journal';
import {
RUN_ATTEMPT_TABLE,
RUN_TABLE,
} from '../../../migrations/0002-run-schema';
import { LOCAL_ARTIFACT_RETENTION_TABLE } from '../../../migrations/0015-local-artifact-retention';
import {
normalizeLocalArtifactRetentionCandidate,
normalizeLocalArtifactRetentionCursor,
normalizeLocalArtifactRetentionRecord,
assertLocalArtifactRetentionTimestamp,
type LocalArtifactRetentionCandidate,
type LocalArtifactRetentionRecord,
} from '../../domain/localArtifactRetention';
import type {
LocalArtifactRetentionPage,
LocalArtifactRetentionRepository,
} from '../../ports/localArtifactRetentionRepository';
import { MAX_LOCAL_ARTIFACT_RETENTION_PAGE_SIZE } from '../../ports/localArtifactRetentionRepository';
interface LocalArtifactRetentionRow {
attemptId: string;
logArtifactId: string;
finishedAtMs: number | string;
eligibleAtMs: number | string;
disposition: string;
bytesReclaimed: number | string;
recordedAtMs: number | string;
}
interface LocalArtifactRetentionInstance
extends Model<LocalArtifactRetentionRow, LocalArtifactRetentionRow>,
LocalArtifactRetentionRow {}
interface CandidateRow {
attempt_id: string;
log_artifact_id: string;
finished_at_ms: number | string;
}
function defineRetentionModel(
database: Sequelize,
): ModelStatic<LocalArtifactRetentionInstance> {
return database.define<LocalArtifactRetentionInstance>(
'Ql3LocalArtifactRetention',
{
attemptId: {
field: 'attempt_id',
type: DataTypes.STRING(36),
allowNull: false,
primaryKey: true,
},
logArtifactId: {
field: 'log_artifact_id',
type: DataTypes.STRING(36),
allowNull: false,
},
finishedAtMs: {
field: 'finished_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
eligibleAtMs: {
field: 'eligible_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
disposition: { type: DataTypes.STRING(16), allowNull: false },
bytesReclaimed: {
field: 'bytes_reclaimed',
type: DataTypes.BIGINT,
allowNull: false,
},
recordedAtMs: {
field: 'recorded_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: LOCAL_ARTIFACT_RETENTION_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function rowToRecord(
row: LocalArtifactRetentionRow,
): LocalArtifactRetentionRecord {
return normalizeLocalArtifactRetentionRecord({
attemptId: row.attemptId,
logArtifactId: row.logArtifactId,
finishedAtMs: Number(row.finishedAtMs),
eligibleAtMs: Number(row.eligibleAtMs),
disposition: row.disposition as LocalArtifactRetentionRecord['disposition'],
bytesReclaimed: Number(row.bytesReclaimed),
recordedAtMs: Number(row.recordedAtMs),
});
}
function sameRetirementIdentity(
left: LocalArtifactRetentionRecord,
right: LocalArtifactRetentionRecord,
): boolean {
return (
left.attemptId === right.attemptId &&
left.logArtifactId === right.logArtifactId &&
left.finishedAtMs === right.finishedAtMs
);
}
export class LocalArtifactRetentionRecordConflictError extends Error {
constructor() {
super('Local Artifact retention record conflicts with existing evidence');
this.name = 'LocalArtifactRetentionRecordConflictError';
}
}
export class LegacySequelizeLocalArtifactRetentionRepository
implements LocalArtifactRetentionRepository
{
private readonly retention: ModelStatic<LocalArtifactRetentionInstance>;
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Local Artifact retention repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
this.retention = defineRetentionModel(database);
}
async list({
cutoffMs,
cursor,
limit,
}: Parameters<
LocalArtifactRetentionRepository['list']
>[0]): Promise<LocalArtifactRetentionPage> {
assertLocalArtifactRetentionTimestamp('cutoffMs', cutoffMs);
const normalizedCursor = cursor
? normalizeLocalArtifactRetentionCursor(cursor)
: undefined;
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_LOCAL_ARTIFACT_RETENTION_PAGE_SIZE
) {
throw new RangeError('Local Artifact retention page size is invalid');
}
const replacements: Record<string, string | number> = {
cutoffMs,
fetchLimit: limit + 1,
...(normalizedCursor
? {
cursorFinishedAtMs: normalizedCursor.finishedAtMs,
cursorAttemptId: normalizedCursor.attemptId,
}
: {}),
};
const cursorPredicate = normalizedCursor
? `AND (
attempt.finished_at_ms > :cursorFinishedAtMs OR
(attempt.finished_at_ms = :cursorFinishedAtMs AND attempt.id > :cursorAttemptId)
)`
: '';
const rows = await this.database.query<CandidateRow>(
`SELECT attempt.id AS attempt_id,
attempt.log_artifact_id,
attempt.finished_at_ms
FROM "${RUN_ATTEMPT_TABLE}" AS attempt
JOIN "${RUN_TABLE}" AS run ON run.id = attempt.run_id
WHERE run.execution_owner = 'runtime'
AND run.status IN ('succeeded','failed','cancelled','timed_out')
AND attempt.status IN ('succeeded','failed','cancelled','timed_out')
AND attempt.executor_type = 'local_process'
AND attempt.log_artifact_id LIKE 'local-%'
AND attempt.finished_at_ms IS NOT NULL
AND attempt.finished_at_ms <= :cutoffMs
AND NOT EXISTS (
SELECT 1 FROM "${COMPLETION_RECEIPT_JOURNAL_TABLE}" AS receipt
WHERE receipt.attempt_id = attempt.id
)
AND NOT EXISTS (
SELECT 1 FROM "${LOCAL_ARTIFACT_RETENTION_TABLE}" AS retained
WHERE retained.attempt_id = attempt.id
)
${cursorPredicate}
ORDER BY attempt.finished_at_ms ASC, attempt.id ASC
LIMIT :fetchLimit`,
{ type: QueryTypes.SELECT, replacements },
);
const truncated = rows.length > limit;
const selected = truncated ? rows.slice(0, limit) : rows;
const candidates: LocalArtifactRetentionCandidate[] = selected.map((row) =>
normalizeLocalArtifactRetentionCandidate({
attemptId: row.attempt_id,
logArtifactId: row.log_artifact_id,
finishedAtMs: Number(row.finished_at_ms),
}),
);
const last = candidates[candidates.length - 1];
return Object.freeze({
candidates: Object.freeze(candidates),
truncated,
...(truncated && last
? {
nextCursor: Object.freeze({
finishedAtMs: last.finishedAtMs,
attemptId: last.attemptId,
}),
}
: {}),
});
}
async record(
value: LocalArtifactRetentionRecord,
): Promise<'inserted' | 'existing'> {
const record = normalizeLocalArtifactRetentionRecord(value);
const row: LocalArtifactRetentionRow = {
attemptId: record.attemptId,
logArtifactId: record.logArtifactId,
finishedAtMs: record.finishedAtMs,
eligibleAtMs: record.eligibleAtMs,
disposition: record.disposition,
bytesReclaimed: record.bytesReclaimed,
recordedAtMs: record.recordedAtMs,
};
try {
await this.retention.create(row);
return 'inserted';
} catch (error) {
if (!(error instanceof UniqueConstraintError)) throw error;
}
const existing = await this.retention.findByPk(record.attemptId, {
raw: true,
});
if (existing && sameRetirementIdentity(rowToRecord(existing), record))
return 'existing';
throw new LocalArtifactRetentionRecordConflictError();
}
}
@@ -0,0 +1,207 @@
import {
DataTypes,
Model,
ModelStatic,
Sequelize,
UniqueConstraintError,
} from 'sequelize';
import { LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE } from '../../../migrations/0013-local-execution-context-recipes';
import {
assertLocalExecutionContextRef,
createLocalExecutionContextRecipeRecord,
localExecutionContextRecipeDigest,
normalizeLocalExecutionContextRecipe,
type LocalExecutionContextRecipe,
} from '../../domain/localExecutionContextRecipe';
import type {
InsertLocalExecutionContextRecipeResult,
LocalExecutionContextRecipeRepository,
} from '../../ports/localExecutionContextRecipeRepository';
interface LocalExecutionContextRecipeRow {
contextRef: string;
environmentRecipe: string;
contentDigest: string;
createdAtMs: number | string;
}
interface LocalExecutionContextRecipeInstance
extends Model<LocalExecutionContextRecipeRow, LocalExecutionContextRecipeRow>,
LocalExecutionContextRecipeRow {}
export class LocalExecutionContextRecipeConflictError extends Error {
constructor(readonly contextRef: string) {
super(`Local execution context recipe ${contextRef} is immutable`);
this.name = 'LocalExecutionContextRecipeConflictError';
}
}
export class LocalExecutionContextRecipeCorruptError extends Error {
constructor(message: string) {
super(message);
this.name = 'LocalExecutionContextRecipeCorruptError';
}
}
function defineRecipeModel(
database: Sequelize,
): ModelStatic<LocalExecutionContextRecipeInstance> {
return database.define<LocalExecutionContextRecipeInstance>(
'Ql3LocalExecutionContextRecipe',
{
contextRef: {
field: 'context_ref',
type: DataTypes.STRING(512),
allowNull: false,
primaryKey: true,
},
environmentRecipe: {
field: 'environment_recipe',
type: DataTypes.TEXT,
allowNull: false,
},
contentDigest: {
field: 'content_digest',
type: DataTypes.STRING(64),
allowNull: false,
},
createdAtMs: {
field: 'created_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function errorCode(error: unknown): string | undefined {
if (!error || typeof error !== 'object') return undefined;
for (const candidate of [
error,
'original' in error ? error.original : undefined,
'parent' in error ? error.parent : undefined,
]) {
if (
candidate &&
typeof candidate === 'object' &&
'code' in candidate &&
typeof candidate.code === 'string'
) {
return candidate.code;
}
}
return undefined;
}
function retryDelay(attempt: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
}
function rowToRecipe(
row: LocalExecutionContextRecipeRow,
): LocalExecutionContextRecipe {
let environment: unknown;
try {
environment = JSON.parse(row.environmentRecipe);
} catch {
throw new LocalExecutionContextRecipeCorruptError(
'Stored local execution context recipe is not valid JSON',
);
}
try {
const normalized = normalizeLocalExecutionContextRecipe({
contextRef: row.contextRef,
environment: environment as LocalExecutionContextRecipe['environment'],
});
if (JSON.stringify(normalized.environment) !== row.environmentRecipe) {
throw new LocalExecutionContextRecipeCorruptError(
'Stored local execution context recipe is not canonical',
);
}
if (
!/^[0-9a-f]{64}$/.test(row.contentDigest) ||
localExecutionContextRecipeDigest(normalized) !== row.contentDigest
) {
throw new LocalExecutionContextRecipeCorruptError(
'Stored local execution context recipe digest does not match',
);
}
return createLocalExecutionContextRecipeRecord(
normalized,
Number(row.createdAtMs),
);
} catch (error) {
if (error instanceof LocalExecutionContextRecipeCorruptError) throw error;
throw new LocalExecutionContextRecipeCorruptError(
`Stored local execution context recipe is invalid: ${
error instanceof Error ? error.message : 'unknown validation error'
}`,
);
}
}
export class LegacySequelizeLocalExecutionContextRecipeRepository
implements LocalExecutionContextRecipeRepository
{
private readonly recipe: ModelStatic<LocalExecutionContextRecipeInstance>;
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Legacy local context recipe repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
this.recipe = defineRecipeModel(database);
}
async resolve(
contextRef: string,
): Promise<LocalExecutionContextRecipe | null> {
assertLocalExecutionContextRef(contextRef);
const row = (await this.recipe.findByPk(contextRef, {
raw: true,
})) as unknown as LocalExecutionContextRecipeRow | null;
return row ? rowToRecipe(row) : null;
}
async insert(
recipe: LocalExecutionContextRecipe,
createdAtMs: number,
): Promise<InsertLocalExecutionContextRecipeResult> {
const record = createLocalExecutionContextRecipeRecord(recipe, createdAtMs);
const values: LocalExecutionContextRecipeRow = {
contextRef: record.contextRef,
environmentRecipe: JSON.stringify(record.environment),
contentDigest: record.contentDigest,
createdAtMs: record.createdAtMs,
};
for (let attempt = 0; attempt < 5; attempt += 1) {
try {
await this.recipe.create(values);
return 'inserted';
} catch (error) {
if (error instanceof UniqueConstraintError) {
const existing = await this.resolve(record.contextRef);
if (
existing &&
localExecutionContextRecipeDigest(existing) === record.contentDigest
) {
return 'idempotent';
}
throw new LocalExecutionContextRecipeConflictError(record.contextRef);
}
if (errorCode(error) === 'SQLITE_BUSY' && attempt < 4) {
await retryDelay(attempt);
continue;
}
throw error;
}
}
throw new Error('Local context recipe insert retry budget exhausted');
}
}
@@ -0,0 +1,350 @@
import {
DataTypes,
Model,
ModelStatic,
QueryTypes,
Sequelize,
Transaction,
UniqueConstraintError,
} from 'sequelize';
import { LOCAL_SECRET_ENVELOPE_TABLE } from '../../../migrations/0014-local-secret-envelopes';
import {
LOCAL_SECRET_ALGORITHM,
LocalSecretUnavailableError,
LocalSecretVersionConflictError,
assertLocalSecretMutationId,
assertLocalSecretName,
assertLocalSecretProjectId,
createLocalSecretRef,
normalizeLocalSecretEnvelope,
type LocalSecretEnvelope,
type LocalSecretReference,
} from '../../domain/localSecret';
import type {
AppendLocalSecretEnvelopeCommand,
AppendLocalSecretEnvelopeResult,
LocalSecretEnvelopeRepository,
} from '../../ports/localSecretEnvelopeRepository';
const MAX_BATCH_SIZE = 64;
const RETRY_ATTEMPTS = 5;
interface LocalSecretEnvelopeRow {
projectId: string;
name: string;
version: number;
mutationId: string;
keyId: string;
algorithm: string;
nonce: Buffer;
ciphertext: Buffer;
authTag: Buffer;
createdAtMs: number | string;
}
interface LocalSecretEnvelopeInstance
extends Model<LocalSecretEnvelopeRow, LocalSecretEnvelopeRow>,
LocalSecretEnvelopeRow {}
interface ResolvedSecretRow {
position: number;
project_id: string | null;
secret_name: string | null;
version: number | null;
mutation_id: string | null;
key_id: string | null;
algorithm: string | null;
nonce: Buffer | null;
ciphertext: Buffer | null;
auth_tag: Buffer | null;
created_at_ms: number | string | null;
}
function defineLocalSecretEnvelopeModel(
database: Sequelize,
): ModelStatic<LocalSecretEnvelopeInstance> {
return database.define<LocalSecretEnvelopeInstance>(
'Ql3LocalSecretEnvelope',
{
projectId: {
field: 'project_id',
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
},
name: {
field: 'secret_name',
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
},
version: {
type: DataTypes.INTEGER,
allowNull: false,
primaryKey: true,
},
mutationId: {
field: 'mutation_id',
type: DataTypes.STRING(64),
allowNull: false,
},
keyId: {
field: 'key_id',
type: DataTypes.STRING(128),
allowNull: false,
},
algorithm: {
type: DataTypes.STRING(32),
allowNull: false,
},
nonce: { type: DataTypes.BLOB, allowNull: false },
ciphertext: { type: DataTypes.BLOB, allowNull: false },
authTag: {
field: 'auth_tag',
type: DataTypes.BLOB,
allowNull: false,
},
createdAtMs: {
field: 'created_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: LOCAL_SECRET_ENVELOPE_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function rowToEnvelope(row: LocalSecretEnvelopeRow): LocalSecretEnvelope {
try {
return normalizeLocalSecretEnvelope({
projectId: row.projectId,
name: row.name,
version: Number(row.version),
mutationId: row.mutationId,
keyId: row.keyId,
algorithm: row.algorithm as typeof LOCAL_SECRET_ALGORITHM,
nonce: Buffer.from(row.nonce).toString('base64url'),
ciphertext: Buffer.from(row.ciphertext).toString('base64url'),
authTag: Buffer.from(row.authTag).toString('base64url'),
createdAtMs: Number(row.createdAtMs),
});
} catch {
throw new LocalSecretUnavailableError();
}
}
function resolvedRowToEnvelope(
row: ResolvedSecretRow,
): LocalSecretEnvelope | null {
if (row.version === null) return null;
if (
row.project_id === null ||
row.secret_name === null ||
row.mutation_id === null ||
row.key_id === null ||
row.algorithm === null ||
row.nonce === null ||
row.ciphertext === null ||
row.auth_tag === null ||
row.created_at_ms === null
) {
throw new LocalSecretUnavailableError();
}
return rowToEnvelope({
projectId: row.project_id,
name: row.secret_name,
version: row.version,
mutationId: row.mutation_id,
keyId: row.key_id,
algorithm: row.algorithm,
nonce: row.nonce,
ciphertext: row.ciphertext,
authTag: row.auth_tag,
createdAtMs: row.created_at_ms,
});
}
function errorCode(error: unknown): string | undefined {
if (!error || typeof error !== 'object') return undefined;
for (const candidate of [
error,
'original' in error ? error.original : undefined,
'parent' in error ? error.parent : undefined,
]) {
if (
candidate &&
typeof candidate === 'object' &&
'code' in candidate &&
typeof candidate.code === 'string'
) {
return candidate.code;
}
}
return undefined;
}
function retryDelay(attempt: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
}
function assertExpectedVersion(value: number): void {
if (!Number.isSafeInteger(value) || value < 0 || value >= 2_147_483_647) {
throw new TypeError('Local Secret expected current version is invalid');
}
}
export class LegacySequelizeLocalSecretEnvelopeRepository
implements LocalSecretEnvelopeRepository
{
private readonly envelope: ModelStatic<LocalSecretEnvelopeInstance>;
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Local Secret envelope repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
this.envelope = defineLocalSecretEnvelopeModel(database);
}
async append(
command: AppendLocalSecretEnvelopeCommand,
): Promise<AppendLocalSecretEnvelopeResult> {
assertExpectedVersion(command.expectedCurrentVersion);
const envelope = normalizeLocalSecretEnvelope(command.envelope);
if (envelope.version !== command.expectedCurrentVersion + 1) {
throw new LocalSecretVersionConflictError();
}
const values = this.values(envelope);
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
try {
return await this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const replay = await this.envelope.findOne({
where: {
projectId: envelope.projectId,
name: envelope.name,
mutationId: envelope.mutationId,
},
raw: true,
transaction,
});
if (replay) {
return { status: 'existing', envelope: rowToEnvelope(replay) };
}
const current = await this.envelope.findOne({
where: { projectId: envelope.projectId, name: envelope.name },
order: [['version', 'DESC']],
raw: true,
transaction,
});
const currentVersion = current ? Number(current.version) : 0;
if (currentVersion !== command.expectedCurrentVersion) {
throw new LocalSecretVersionConflictError();
}
await this.envelope.create(values, { transaction });
return { status: 'inserted', envelope };
},
);
} catch (error) {
if (error instanceof LocalSecretVersionConflictError) throw error;
if (
(error instanceof UniqueConstraintError ||
errorCode(error) === 'SQLITE_BUSY') &&
attempt < RETRY_ATTEMPTS - 1
) {
await retryDelay(attempt);
continue;
}
throw error;
}
}
throw new LocalSecretUnavailableError();
}
async findByMutation(
projectId: string,
name: string,
mutationId: string,
): Promise<LocalSecretEnvelope | null> {
assertLocalSecretProjectId(projectId);
assertLocalSecretName(name);
assertLocalSecretMutationId(mutationId);
const row = await this.envelope.findOne({
where: { projectId, name, mutationId },
raw: true,
});
return row ? rowToEnvelope(row) : null;
}
async resolveMany(
references: readonly LocalSecretReference[],
): Promise<readonly (LocalSecretEnvelope | null)[]> {
if (!Array.isArray(references) || references.length > MAX_BATCH_SIZE) {
throw new RangeError('Local Secret batch is too large');
}
if (references.length === 0) return Object.freeze([]);
const replacements: Record<string, string | number | null> = {};
const requestedValues = references.map((reference, position) => {
createLocalSecretRef(reference);
replacements[`position${position}`] = position;
replacements[`project${position}`] = reference.projectId;
replacements[`name${position}`] = reference.name;
replacements[`version${position}`] = reference.version ?? null;
return `(:position${position}, :project${position}, :name${position}, :version${position})`;
});
const rows = await this.database.query<ResolvedSecretRow>(
`WITH requested(position, project_id, secret_name, requested_version) AS (
VALUES ${requestedValues.join(', ')}
)
SELECT requested.position,
envelope.project_id,
envelope.secret_name,
envelope.version,
envelope.mutation_id,
envelope.key_id,
envelope.algorithm,
envelope.nonce,
envelope.ciphertext,
envelope.auth_tag,
envelope.created_at_ms
FROM requested
LEFT JOIN "${LOCAL_SECRET_ENVELOPE_TABLE}" AS envelope
ON envelope.project_id = requested.project_id
AND envelope.secret_name = requested.secret_name
AND envelope.version = COALESCE(
requested.requested_version,
(SELECT MAX(current.version)
FROM "${LOCAL_SECRET_ENVELOPE_TABLE}" AS current
WHERE current.project_id = requested.project_id
AND current.secret_name = requested.secret_name)
)
ORDER BY requested.position ASC`,
{ type: QueryTypes.SELECT, replacements },
);
if (rows.length !== references.length) {
throw new LocalSecretUnavailableError();
}
return Object.freeze(rows.map(resolvedRowToEnvelope));
}
private values(envelope: LocalSecretEnvelope): LocalSecretEnvelopeRow {
return {
projectId: envelope.projectId,
name: envelope.name,
version: envelope.version,
mutationId: envelope.mutationId,
keyId: envelope.keyId,
algorithm: envelope.algorithm,
nonce: Buffer.from(envelope.nonce, 'base64url'),
ciphertext: Buffer.from(envelope.ciphertext, 'base64url'),
authTag: Buffer.from(envelope.authTag, 'base64url'),
createdAtMs: envelope.createdAtMs,
};
}
}
@@ -0,0 +1,280 @@
import {
DataTypes,
Model,
ModelStatic,
Op,
Sequelize,
type WhereOptions,
} from 'sequelize';
import {
RUN_ATTEMPT_TABLE,
RUN_TABLE,
} from '../../../migrations/0002-run-schema';
import type { ExecutionStopKind, ExecutorType } from '../../domain/execution';
import type {
PrimaryCancellationAttemptReference,
PrimaryCancellationCandidate,
PrimaryCancellationCursor,
PrimaryCancellationPage,
PrimaryCancellationSource,
} from '../../ports/primaryCancellationSource';
import { MAX_PRIMARY_CANCELLATION_BATCH_SIZE } from '../../ports/primaryCancellationSource';
interface CancellationRunRow {
id: string;
executionOwner: string;
status: string;
cancelRequestedAtMs: number | null;
cancelReason: string | null;
}
interface CancellationRequestedRunRow extends CancellationRunRow {
cancelRequestedAtMs: number;
cancelReason: string;
}
interface CancellationAttemptRow {
id: string;
runId: string;
attempt: number;
status: string;
executorType: string;
executorHandle: string | null;
pid: number | null;
createdAtMs: number;
}
interface CancellationRunInstance
extends Model<CancellationRunRow, CancellationRunRow>,
CancellationRunRow {}
interface CancellationAttemptInstance
extends Model<CancellationAttemptRow, CancellationAttemptRow>,
CancellationAttemptRow {}
function defineCancellationRunModel(
database: Sequelize,
): ModelStatic<CancellationRunInstance> {
return database.define<CancellationRunInstance>(
'Ql3PrimaryCancellationRun',
{
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
executionOwner: {
field: 'execution_owner',
type: DataTypes.STRING(16),
allowNull: false,
},
status: { type: DataTypes.STRING(32), allowNull: false },
cancelRequestedAtMs: {
field: 'cancel_requested_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
cancelReason: {
field: 'cancel_reason',
type: DataTypes.STRING(32),
allowNull: false,
},
},
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
);
}
function defineCancellationAttemptModel(
database: Sequelize,
): ModelStatic<CancellationAttemptInstance> {
return database.define<CancellationAttemptInstance>(
'Ql3PrimaryCancellationAttempt',
{
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
runId: {
field: 'run_id',
type: DataTypes.STRING(36),
allowNull: false,
},
attempt: { type: DataTypes.INTEGER, allowNull: false },
status: { type: DataTypes.STRING(32), allowNull: false },
executorType: {
field: 'executor_type',
type: DataTypes.STRING(64),
allowNull: false,
},
executorHandle: {
field: 'executor_handle',
type: DataTypes.TEXT,
allowNull: true,
},
pid: { type: DataTypes.INTEGER, allowNull: true },
createdAtMs: {
field: 'created_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: RUN_ATTEMPT_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function assertCursor(cursor: PrimaryCancellationCursor): void {
if (!Number.isSafeInteger(cursor.requestedAtMs) || cursor.requestedAtMs < 0) {
throw new RangeError('cursor.requestedAtMs must be a non-negative integer');
}
if (!cursor.runId || cursor.runId.length > 36) {
throw new RangeError('cursor.runId must be between 1 and 36 characters');
}
}
export class LegacySequelizePrimaryCancellationSource
implements PrimaryCancellationSource
{
private readonly run: ModelStatic<CancellationRunInstance>;
private readonly attempt: ModelStatic<CancellationAttemptInstance>;
constructor(database: Sequelize) {
this.run = defineCancellationRunModel(database);
this.attempt = defineCancellationAttemptModel(database);
}
async listCandidates({
cursor,
limit = 32,
}: {
cursor?: PrimaryCancellationCursor;
limit?: number;
} = {}): Promise<PrimaryCancellationPage> {
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_PRIMARY_CANCELLATION_BATCH_SIZE
) {
throw new RangeError(
'limit must be between 1 and MAX_PRIMARY_CANCELLATION_BATCH_SIZE',
);
}
if (cursor) assertCursor(cursor);
const where: WhereOptions<CancellationRunRow> = {
executionOwner: 'runtime',
status: {
[Op.in]: [
'created',
'queued',
'dispatching',
'running',
'waiting_approval',
'retry_wait',
'lost',
],
},
cancelRequestedAtMs: {
[Op.ne]: null,
},
cancelReason: { [Op.ne]: null },
...(cursor === undefined
? {}
: {
[Op.or]: [
{ cancelRequestedAtMs: { [Op.gt]: cursor.requestedAtMs } },
{
cancelRequestedAtMs: cursor.requestedAtMs,
id: { [Op.gt]: cursor.runId },
},
],
}),
};
const runRows = (await this.run.findAll({
attributes: ['id', 'cancelRequestedAtMs', 'cancelReason'],
where,
order: [
['cancelRequestedAtMs', 'ASC'],
['id', 'ASC'],
],
limit: limit + 1,
raw: true,
})) as unknown as CancellationRunRow[];
const truncated = runRows.length > limit;
const boundedRuns = runRows
.filter(
(run): run is CancellationRequestedRunRow =>
run.cancelRequestedAtMs !== null && run.cancelReason !== null,
)
.slice(0, limit);
if (boundedRuns.length === 0) {
return {
candidates: [],
truncated: false,
unsafeAttemptOverflow: false,
};
}
const maxAttemptRows = limit * 2;
const attemptRows = (await this.attempt.findAll({
attributes: [
'id',
'runId',
'attempt',
'executorType',
'executorHandle',
'pid',
],
where: {
runId: { [Op.in]: boundedRuns.map((run) => run.id) },
status: { [Op.in]: ['claimed', 'starting', 'running'] },
},
order: [
['runId', 'ASC'],
['attempt', 'DESC'],
['createdAtMs', 'DESC'],
['id', 'DESC'],
],
limit: maxAttemptRows + 1,
raw: true,
})) as unknown as CancellationAttemptRow[];
if (attemptRows.length > maxAttemptRows) {
return {
candidates: [],
truncated,
unsafeAttemptOverflow: true,
};
}
const attemptsByRun = new Map<
string,
PrimaryCancellationAttemptReference[]
>();
for (const attempt of attemptRows) {
const references = attemptsByRun.get(attempt.runId) ?? [];
references.push({
attemptId: attempt.id,
executorType: attempt.executorType as ExecutorType,
...(attempt.executorHandle === null
? {}
: { executorHandle: attempt.executorHandle }),
...(attempt.pid === null ? {} : { pid: attempt.pid }),
});
attemptsByRun.set(attempt.runId, references);
}
const candidates: PrimaryCancellationCandidate[] = boundedRuns.map(
(run) => ({
runId: run.id,
requestedAtMs: run.cancelRequestedAtMs,
reason: run.cancelReason as ExecutionStopKind,
attempts: attemptsByRun.get(run.id) ?? [],
}),
);
const last = boundedRuns[boundedRuns.length - 1];
return {
candidates,
truncated,
unsafeAttemptOverflow: false,
nextCursor: {
requestedAtMs: last.cancelRequestedAtMs,
runId: last.id,
},
};
}
}
@@ -0,0 +1,398 @@
import {
DataTypes,
Model,
type ModelStatic,
Op,
type Sequelize,
type Transaction,
} from 'sequelize';
import {
RUN_ATTEMPT_TABLE,
RUN_TABLE,
} from '../../../migrations/0002-run-schema';
import { RUNNING_INSTANCE_TABLE } from '../../../migrations/0003-running-instance-run-reference';
import type { RunAttemptStatus, RunStatus } from '../../domain/run';
import { parseLegacyLogOutputRef } from '../../compatibility/legacyLogOutputRef';
import type {
SequelizeRunProjectionContext,
SequelizeRunProjectionParticipant,
} from './projectedRunRepository';
const CRONTAB_TABLE = 'Crontabs';
const CRONTAB_STATUS_RUNNING = 0;
const CRONTAB_STATUS_IDLE = 1;
const CRONTAB_STATUS_QUEUED = 3;
const INSTANCE_STATUS_RUNNING = 0;
const INSTANCE_STATUS_FINISHED = 1;
const INSTANCE_STATUS_STOPPED = 2;
const INSTANCE_STATUS_ERROR = 3;
const RUNNING_RUN_STATUSES: readonly RunStatus[] = [
'running',
'waiting_approval',
];
const QUEUED_RUN_STATUSES: readonly RunStatus[] = [
'created',
'queued',
'dispatching',
'retry_wait',
];
const TERMINAL_RUN_STATUSES: readonly RunStatus[] = [
'lost',
'succeeded',
'failed',
'cancelled',
'timed_out',
];
interface ProjectionRunRow {
id: string;
legacyCronId: number | null;
executionOwner: string;
status: RunStatus;
outputRef: string | null;
createdAtMs: number;
startedAtMs: number | null;
finishedAtMs: number | null;
}
interface ProjectionAttemptRow {
id: string;
runId: string;
attempt: number;
status: RunAttemptStatus;
pid: number | null;
createdAtMs: number;
startedAtMs: number | null;
finishedAtMs: number | null;
exitCode: number | null;
}
interface ProjectionCrontabRow {
id: number;
status: number | null;
pid: number | null;
logPath: string | null;
lastRunningTime: number | null;
lastExecutionTime: number | null;
}
interface ProjectionInstanceRow {
id?: number;
cronId: number;
runId: string | null;
attemptId: string | null;
pid: number | null;
logPath: string | null;
startedAt: number;
finishedAt: number | null;
status: number;
exitCode: number | null;
}
interface ProjectionRunInstance
extends Model<ProjectionRunRow, ProjectionRunRow>,
ProjectionRunRow {}
interface ProjectionAttemptInstance
extends Model<ProjectionAttemptRow, ProjectionAttemptRow>,
ProjectionAttemptRow {}
interface ProjectionCrontabInstance
extends Model<ProjectionCrontabRow, ProjectionCrontabRow>,
ProjectionCrontabRow {}
interface ProjectionInstanceInstance
extends Model<ProjectionInstanceRow, ProjectionInstanceRow>,
ProjectionInstanceRow {}
interface ProjectionModels {
run: ModelStatic<ProjectionRunInstance>;
attempt: ModelStatic<ProjectionAttemptInstance>;
crontab: ModelStatic<ProjectionCrontabInstance>;
instance: ModelStatic<ProjectionInstanceInstance>;
}
interface SelectedRun {
run: ProjectionRunRow;
attempt: ProjectionAttemptRow | null;
}
function defineProjectionModels(database: Sequelize): ProjectionModels {
const common = { timestamps: false, freezeTableName: true } as const;
const run = database.define<ProjectionRunInstance>(
'Ql3PrimaryCronProjectionRun',
{
id: { type: DataTypes.STRING(36), primaryKey: true },
legacyCronId: { field: 'legacy_cron_id', type: DataTypes.INTEGER },
executionOwner: {
field: 'execution_owner',
type: DataTypes.STRING(16),
},
status: { type: DataTypes.STRING(32) },
outputRef: { field: 'output_ref', type: DataTypes.STRING(512) },
createdAtMs: { field: 'created_at_ms', type: DataTypes.BIGINT },
startedAtMs: { field: 'started_at_ms', type: DataTypes.BIGINT },
finishedAtMs: { field: 'finished_at_ms', type: DataTypes.BIGINT },
},
{ ...common, tableName: RUN_TABLE },
);
const attempt = database.define<ProjectionAttemptInstance>(
'Ql3PrimaryCronProjectionAttempt',
{
id: { type: DataTypes.STRING(36), primaryKey: true },
runId: { field: 'run_id', type: DataTypes.STRING(36) },
attempt: { type: DataTypes.INTEGER },
status: { type: DataTypes.STRING(32) },
pid: { type: DataTypes.INTEGER },
createdAtMs: { field: 'created_at_ms', type: DataTypes.BIGINT },
startedAtMs: { field: 'started_at_ms', type: DataTypes.BIGINT },
finishedAtMs: { field: 'finished_at_ms', type: DataTypes.BIGINT },
exitCode: { field: 'exit_code', type: DataTypes.INTEGER },
},
{ ...common, tableName: RUN_ATTEMPT_TABLE },
);
const crontab = database.define<ProjectionCrontabInstance>(
'Ql3PrimaryCronProjectionCrontab',
{
id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true },
status: { type: DataTypes.INTEGER },
pid: { type: DataTypes.INTEGER },
logPath: { field: 'log_path', type: DataTypes.STRING },
lastRunningTime: {
field: 'last_running_time',
type: DataTypes.INTEGER,
},
lastExecutionTime: {
field: 'last_execution_time',
type: DataTypes.INTEGER,
},
},
{ ...common, tableName: CRONTAB_TABLE },
);
const instance = database.define<ProjectionInstanceInstance>(
'Ql3PrimaryCronProjectionInstance',
{
id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true },
cronId: { field: 'cron_id', type: DataTypes.INTEGER },
runId: { field: 'run_id', type: DataTypes.STRING(36) },
attemptId: { field: 'attempt_id', type: DataTypes.STRING(36) },
pid: { type: DataTypes.INTEGER },
logPath: { field: 'log_path', type: DataTypes.STRING },
startedAt: { field: 'started_at', type: DataTypes.INTEGER },
finishedAt: { field: 'finished_at', type: DataTypes.INTEGER },
status: { type: DataTypes.INTEGER },
exitCode: { field: 'exit_code', type: DataTypes.INTEGER },
},
{ ...common, tableName: RUNNING_INSTANCE_TABLE },
);
return { run, attempt, crontab, instance };
}
function toUnixSeconds(milliseconds: number): number {
return Math.floor(milliseconds / 1000);
}
function instanceStatus(status: RunAttemptStatus): number | null {
switch (status) {
case 'claimed':
return null;
case 'starting':
case 'running':
return INSTANCE_STATUS_RUNNING;
case 'succeeded':
return INSTANCE_STATUS_FINISHED;
case 'cancelled':
return INSTANCE_STATUS_STOPPED;
case 'failed':
case 'timed_out':
case 'lost':
return INSTANCE_STATUS_ERROR;
}
}
function isAttemptActive(status: RunAttemptStatus): boolean {
return status === 'starting' || status === 'running';
}
/**
* Projects runtime-owned Run state into the legacy UI tables before the same
* SQLite transaction commits. It never projects legacy-owned Shadow Runs.
*/
export class PrimaryCronProjection
implements SequelizeRunProjectionParticipant
{
private readonly models: ProjectionModels;
constructor(database: Sequelize) {
this.models = defineProjectionModels(database);
}
async apply(context: SequelizeRunProjectionContext): Promise<void> {
const cronIds = new Set<number>();
for (const attemptId of context.changedAttemptIds) {
const cronId = await this.projectAttempt(attemptId, context.transaction);
if (cronId !== null) cronIds.add(cronId);
}
for (const runId of context.changedRunIds) {
const run = await context.runs.findRunById(runId);
if (run?.executionOwner === 'runtime' && run.legacyCronId !== undefined) {
cronIds.add(run.legacyCronId);
}
}
for (const cronId of cronIds) {
await this.projectCrontab(cronId, context.transaction);
}
}
private async projectAttempt(
attemptId: string,
transaction: Transaction,
): Promise<number | null> {
const attempt = await this.models.attempt.findByPk(attemptId, {
raw: true,
transaction,
});
if (!attempt) return null;
const run = await this.models.run.findByPk(attempt.runId, {
raw: true,
transaction,
});
if (!run || run.executionOwner !== 'runtime' || run.legacyCronId === null) {
return null;
}
const status = instanceStatus(attempt.status);
if (status === null) return run.legacyCronId;
const logPath = parseLegacyLogOutputRef(run.outputRef ?? undefined);
const values: ProjectionInstanceRow = {
cronId: run.legacyCronId,
runId: run.id,
attemptId: attempt.id,
pid: attempt.pid,
logPath,
startedAt: toUnixSeconds(
attempt.startedAtMs ?? run.startedAtMs ?? attempt.createdAtMs,
),
finishedAt:
attempt.finishedAtMs === null
? null
: toUnixSeconds(attempt.finishedAtMs),
status,
exitCode: attempt.exitCode,
};
const existing = await this.models.instance.findOne({
where: { attemptId: attempt.id },
transaction,
});
if (existing) {
await existing.update(values, { transaction });
} else {
await this.models.instance.create(values, { transaction });
}
return run.legacyCronId;
}
private async projectCrontab(
cronId: number,
transaction: Transaction,
): Promise<void> {
const running = await this.findSelectedRun(
cronId,
RUNNING_RUN_STATUSES,
transaction,
);
if (running) {
await this.updateCrontab(
cronId,
CRONTAB_STATUS_RUNNING,
running,
transaction,
);
return;
}
const queued = await this.findSelectedRun(
cronId,
QUEUED_RUN_STATUSES,
transaction,
);
if (queued) {
await this.updateCrontab(
cronId,
CRONTAB_STATUS_QUEUED,
queued,
transaction,
);
return;
}
const terminal = await this.findSelectedRun(
cronId,
TERMINAL_RUN_STATUSES,
transaction,
);
await this.updateCrontab(
cronId,
CRONTAB_STATUS_IDLE,
terminal,
transaction,
);
}
private async findSelectedRun(
cronId: number,
statuses: readonly RunStatus[],
transaction: Transaction,
): Promise<SelectedRun | null> {
const run = await this.models.run.findOne({
where: {
legacyCronId: cronId,
executionOwner: 'runtime',
status: { [Op.in]: [...statuses] },
},
order: [
['createdAtMs', 'DESC'],
['id', 'DESC'],
],
raw: true,
transaction,
});
if (!run) return null;
const attempt = await this.models.attempt.findOne({
where: { runId: run.id },
order: [
['attempt', 'DESC'],
['id', 'DESC'],
],
raw: true,
transaction,
});
return { run, attempt };
}
private async updateCrontab(
cronId: number,
status: number,
selected: SelectedRun | null,
transaction: Transaction,
): Promise<void> {
const run = selected?.run;
const attempt = selected?.attempt;
const startedAtMs = attempt?.startedAtMs ?? run?.startedAtMs ?? null;
const finishedAtMs = attempt?.finishedAtMs ?? run?.finishedAtMs ?? null;
const values: Partial<ProjectionCrontabRow> = {
status,
pid: attempt && isAttemptActive(attempt.status) ? attempt.pid : null,
logPath: parseLegacyLogOutputRef(run?.outputRef ?? undefined),
};
if (startedAtMs !== null) {
values.lastExecutionTime = toUnixSeconds(startedAtMs);
}
if (startedAtMs !== null && finishedAtMs !== null) {
values.lastRunningTime = Math.max(
0,
Math.floor((finishedAtMs - startedAtMs) / 1000),
);
}
await this.models.crontab.update(values, {
where: { id: cronId },
transaction,
});
}
}
@@ -0,0 +1,69 @@
import { DataTypes, Model, ModelStatic, Sequelize } from 'sequelize';
import { RUN_TABLE } from '../../../migrations/0002-run-schema';
import type { PrimaryRunIdempotencyLookup } from '../../ports/primaryRunIdempotencyLookup';
interface IdempotentRunRow {
id: string;
projectId: string;
idempotencyKey: string | null;
}
interface IdempotentRunInstance
extends Model<IdempotentRunRow, IdempotentRunRow>,
IdempotentRunRow {}
function defineIdempotentRunModel(
database: Sequelize,
): ModelStatic<IdempotentRunInstance> {
return database.define<IdempotentRunInstance>(
'Ql3PrimaryIdempotentRun',
{
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
projectId: {
field: 'project_id',
type: DataTypes.STRING(128),
allowNull: false,
},
idempotencyKey: {
field: 'idempotency_key',
type: DataTypes.STRING(255),
allowNull: true,
},
},
{
tableName: RUN_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
export class LegacySequelizePrimaryRunIdempotencyLookup
implements PrimaryRunIdempotencyLookup
{
private readonly run: ModelStatic<IdempotentRunInstance>;
constructor(database: Sequelize) {
this.run = defineIdempotentRunModel(database);
}
async findRunId(
projectId: string,
idempotencyKey: string,
): Promise<string | null> {
if (!projectId || projectId.length > 128) {
throw new RangeError('projectId must be between 1 and 128 characters');
}
if (!idempotencyKey || idempotencyKey.length > 255) {
throw new RangeError(
'idempotencyKey must be between 1 and 255 characters',
);
}
const row = await this.run.findOne({
attributes: ['id'],
where: { projectId, idempotencyKey },
raw: true,
});
return row?.id ?? null;
}
}
@@ -0,0 +1,225 @@
import {
DataTypes,
Model,
ModelStatic,
Op,
Sequelize,
type WhereOptions,
} from 'sequelize';
import {
RUN_ATTEMPT_TABLE,
RUN_TABLE,
} from '../../../migrations/0002-run-schema';
import type { ExecutorType } from '../../domain/execution';
import type {
PrimaryRunRecoveryAttemptReference,
PrimaryRunRecoveryCandidate,
PrimaryRunRecoveryCursor,
PrimaryRunRecoveryPage,
PrimaryRunRecoverySource,
} from '../../ports/primaryRunRecoverySource';
import { MAX_PRIMARY_RECOVERY_BATCH_SIZE } from '../../ports/primaryRunRecoverySource';
interface RecoveryRunRow {
id: string;
executionOwner: string;
status: string;
createdAtMs: number;
}
interface RecoveryAttemptRow {
id: string;
runId: string;
attempt: number;
status: string;
executorType: string;
createdAtMs: number;
}
interface RecoveryRunInstance
extends Model<RecoveryRunRow, RecoveryRunRow>,
RecoveryRunRow {}
interface RecoveryAttemptInstance
extends Model<RecoveryAttemptRow, RecoveryAttemptRow>,
RecoveryAttemptRow {}
function defineRecoveryRunModel(
database: Sequelize,
): ModelStatic<RecoveryRunInstance> {
return database.define<RecoveryRunInstance>(
'Ql3PrimaryRecoveryRun',
{
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
executionOwner: {
field: 'execution_owner',
type: DataTypes.STRING(16),
allowNull: false,
},
status: { type: DataTypes.STRING(32), allowNull: false },
createdAtMs: {
field: 'created_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
);
}
function defineRecoveryAttemptModel(
database: Sequelize,
): ModelStatic<RecoveryAttemptInstance> {
return database.define<RecoveryAttemptInstance>(
'Ql3PrimaryRecoveryAttempt',
{
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
runId: {
field: 'run_id',
type: DataTypes.STRING(36),
allowNull: false,
},
attempt: { type: DataTypes.INTEGER, allowNull: false },
status: { type: DataTypes.STRING(32), allowNull: false },
executorType: {
field: 'executor_type',
type: DataTypes.STRING(64),
allowNull: false,
},
createdAtMs: {
field: 'created_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: RUN_ATTEMPT_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function assertCursor(cursor: PrimaryRunRecoveryCursor): void {
if (!Number.isSafeInteger(cursor.createdAtMs) || cursor.createdAtMs < 0) {
throw new RangeError('cursor.createdAtMs must be a non-negative integer');
}
if (!cursor.runId || cursor.runId.length > 36) {
throw new RangeError('cursor.runId must be between 1 and 36 characters');
}
}
export class LegacySequelizePrimaryRunRecoverySource
implements PrimaryRunRecoverySource
{
private readonly run: ModelStatic<RecoveryRunInstance>;
private readonly attempt: ModelStatic<RecoveryAttemptInstance>;
constructor(database: Sequelize) {
this.run = defineRecoveryRunModel(database);
this.attempt = defineRecoveryAttemptModel(database);
}
async listCandidates({
cursor,
limit = 32,
}: {
cursor?: PrimaryRunRecoveryCursor;
limit?: number;
} = {}): Promise<PrimaryRunRecoveryPage> {
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_PRIMARY_RECOVERY_BATCH_SIZE
) {
throw new RangeError(
'limit must be between 1 and MAX_PRIMARY_RECOVERY_BATCH_SIZE',
);
}
if (cursor) assertCursor(cursor);
const where: WhereOptions<RecoveryRunRow> = {
executionOwner: 'runtime',
status: { [Op.in]: ['dispatching', 'running'] },
...(cursor === undefined
? {}
: {
[Op.or]: [
{ createdAtMs: { [Op.gt]: cursor.createdAtMs } },
{
createdAtMs: cursor.createdAtMs,
id: { [Op.gt]: cursor.runId },
},
],
}),
};
const runRows = (await this.run.findAll({
attributes: ['id', 'createdAtMs'],
where,
order: [
['createdAtMs', 'ASC'],
['id', 'ASC'],
],
limit: limit + 1,
raw: true,
})) as unknown as RecoveryRunRow[];
const truncated = runRows.length > limit;
const boundedRuns = runRows.slice(0, limit);
if (boundedRuns.length === 0) {
return {
candidates: [],
truncated: false,
unsafeAttemptOverflow: false,
};
}
const maxAttemptRows = limit * 2;
const attemptRows = (await this.attempt.findAll({
attributes: ['id', 'runId', 'attempt', 'executorType'],
where: {
runId: { [Op.in]: boundedRuns.map((run) => run.id) },
status: { [Op.in]: ['claimed', 'starting', 'running'] },
},
order: [
['runId', 'ASC'],
['attempt', 'DESC'],
['createdAtMs', 'DESC'],
['id', 'DESC'],
],
limit: maxAttemptRows + 1,
raw: true,
})) as unknown as RecoveryAttemptRow[];
if (attemptRows.length > maxAttemptRows) {
return {
candidates: [],
truncated,
unsafeAttemptOverflow: true,
};
}
const attemptsByRun = new Map<
string,
PrimaryRunRecoveryAttemptReference[]
>();
for (const attempt of attemptRows) {
const references = attemptsByRun.get(attempt.runId) ?? [];
references.push({
attemptId: attempt.id,
executorType: attempt.executorType as ExecutorType,
});
attemptsByRun.set(attempt.runId, references);
}
const candidates: PrimaryRunRecoveryCandidate[] = boundedRuns.map(
(run) => ({
runId: run.id,
attempts: attemptsByRun.get(run.id) ?? [],
}),
);
const last = boundedRuns[boundedRuns.length - 1];
return {
candidates,
truncated,
unsafeAttemptOverflow: false,
nextCursor: { createdAtMs: last.createdAtMs, runId: last.id },
};
}
}
@@ -0,0 +1,195 @@
import {
DataTypes,
Model,
ModelStatic,
Op,
Sequelize,
type WhereOptions,
} from 'sequelize';
import {
RUN_ATTEMPT_TABLE,
RUN_TABLE,
} from '../../../migrations/0002-run-schema';
import {
MAX_PRIMARY_TIMEOUT_BATCH_SIZE,
type PrimaryTimeoutCursor,
type PrimaryTimeoutPage,
type PrimaryTimeoutSource,
} from '../../ports/primaryTimeoutSource';
interface TimeoutAttemptRow {
id: string;
runId: string;
status: string;
deadlineAtMs: number | null;
}
interface TimeoutRunRow {
id: string;
executionOwner: string;
status: string;
cancelRequestedAtMs: number | null;
}
interface TimeoutAttemptInstance
extends Model<TimeoutAttemptRow, TimeoutAttemptRow>,
TimeoutAttemptRow {}
interface TimeoutRunInstance
extends Model<TimeoutRunRow, TimeoutRunRow>,
TimeoutRunRow {}
function defineTimeoutAttemptModel(
database: Sequelize,
): ModelStatic<TimeoutAttemptInstance> {
return database.define<TimeoutAttemptInstance>(
'Ql3PrimaryTimeoutAttempt',
{
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
runId: {
field: 'run_id',
type: DataTypes.STRING(36),
allowNull: false,
},
status: { type: DataTypes.STRING(32), allowNull: false },
deadlineAtMs: {
field: 'deadline_at_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
},
{ tableName: RUN_ATTEMPT_TABLE, timestamps: false, freezeTableName: true },
);
}
function defineTimeoutRunModel(
database: Sequelize,
): ModelStatic<TimeoutRunInstance> {
return database.define<TimeoutRunInstance>(
'Ql3PrimaryTimeoutRun',
{
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
executionOwner: {
field: 'execution_owner',
type: DataTypes.STRING(16),
allowNull: false,
},
status: { type: DataTypes.STRING(32), allowNull: false },
cancelRequestedAtMs: {
field: 'cancel_requested_at_ms',
type: DataTypes.BIGINT,
allowNull: true,
},
},
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
);
}
function assertCursor(cursor: PrimaryTimeoutCursor): void {
if (!Number.isSafeInteger(cursor.deadlineAtMs) || cursor.deadlineAtMs < 0) {
throw new RangeError('cursor.deadlineAtMs must be a non-negative integer');
}
if (!cursor.attemptId || cursor.attemptId.length > 36) {
throw new RangeError(
'cursor.attemptId must be between 1 and 36 characters',
);
}
}
export class LegacySequelizePrimaryTimeoutSource
implements PrimaryTimeoutSource
{
private readonly attempt: ModelStatic<TimeoutAttemptInstance>;
private readonly run: ModelStatic<TimeoutRunInstance>;
constructor(database: Sequelize) {
this.attempt = defineTimeoutAttemptModel(database);
this.run = defineTimeoutRunModel(database);
this.attempt.belongsTo(this.run, {
as: 'timeoutRun',
foreignKey: 'runId',
targetKey: 'id',
constraints: false,
});
}
async listOverdue(options: {
nowMs: number;
cursor?: PrimaryTimeoutCursor;
limit?: number;
}): Promise<PrimaryTimeoutPage> {
if (!Number.isSafeInteger(options.nowMs) || options.nowMs < 0) {
throw new RangeError('nowMs must be a non-negative safe integer');
}
const limit = options.limit ?? 32;
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_PRIMARY_TIMEOUT_BATCH_SIZE
) {
throw new RangeError(
'limit must be between 1 and MAX_PRIMARY_TIMEOUT_BATCH_SIZE',
);
}
if (options.cursor) assertCursor(options.cursor);
const cursorWhere: WhereOptions<TimeoutAttemptRow> = options.cursor
? {
[Op.or]: [
{ deadlineAtMs: { [Op.gt]: options.cursor.deadlineAtMs } },
{
deadlineAtMs: options.cursor.deadlineAtMs,
id: { [Op.gt]: options.cursor.attemptId },
},
],
}
: {};
const rows = (await this.attempt.findAll({
attributes: ['id', 'runId', 'status', 'deadlineAtMs'],
where: {
status: { [Op.in]: ['starting', 'running'] },
deadlineAtMs: { [Op.ne]: null, [Op.lte]: options.nowMs },
...cursorWhere,
},
include: [
{
model: this.run,
as: 'timeoutRun',
attributes: [],
required: true,
where: {
executionOwner: 'runtime',
status: { [Op.in]: ['dispatching', 'running'] },
cancelRequestedAtMs: null,
},
},
],
order: [
['deadlineAtMs', 'ASC'],
['id', 'ASC'],
],
limit: limit + 1,
raw: true,
})) as unknown as TimeoutAttemptRow[];
const truncated = rows.length > limit;
const selected = rows.slice(0, limit);
const candidates = selected.map((row) => ({
runId: row.runId,
attemptId: row.id,
deadlineAtMs: Number(row.deadlineAtMs),
}));
const last = candidates[candidates.length - 1];
return {
candidates,
truncated,
...(truncated && last
? {
nextCursor: {
deadlineAtMs: last.deadlineAtMs,
attemptId: last.attemptId,
},
}
: {}),
};
}
}
@@ -0,0 +1,522 @@
import { timingSafeEqual } from 'crypto';
import {
QueryTypes,
Sequelize,
Transaction,
UniqueConstraintError,
} from 'sequelize';
import {
PROJECT_ROLE_BINDING_TABLE,
PROJECT_TABLE,
} from '../../../migrations/0017-project-policy';
import { PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE } from '../../../migrations/0018-project-owner-bootstrap';
import {
assertProjectPolicyProjectId,
normalizePolicySubject,
normalizeProjectRoleBindingRecord,
type ProjectRoleBindingRecord,
} from '../../domain/projectPolicy';
import {
OWNER_BOOTSTRAP_MAX_VERSION,
OWNER_BOOTSTRAP_SYSTEM_SUBJECT,
ProjectOwnerBootstrapChallengeActiveError,
ProjectOwnerBootstrapClaimRejectedError,
ProjectOwnerBootstrapProjectInactiveError,
ProjectOwnerBootstrapProjectNotFoundError,
ProjectOwnerBootstrapProjectNotPristineError,
ProjectOwnerBootstrapUnavailableError,
assertProjectOwnerBootstrapChallengeId,
assertProjectOwnerBootstrapTokenDigest,
normalizeProjectOwnerBootstrapChallengeRecord,
type ProjectOwnerBootstrapChallengeRecord,
} from '../../domain/projectOwnerBootstrap';
import type {
ClaimProjectOwnerBootstrapChallengeCommand,
ClaimProjectOwnerBootstrapChallengeResult,
IssueProjectOwnerBootstrapChallengeCommand,
ProjectOwnerBootstrapRepository,
} from '../../ports/projectOwnerBootstrapRepository';
const RETRY_ATTEMPTS = 5;
interface ProjectStatusRow {
status: string;
}
interface BootstrapChallengeRow {
project_id: string;
version: number;
challenge_id: string;
token_digest: string;
issued_at_ms: number | string;
expires_at_ms: number | string;
consumed_at_ms: number | string | null;
claimed_subject_type: string | null;
claimed_subject_id: string | null;
}
interface BootstrapBindingRow {
project_id: string;
subject_type: string;
subject_id: string;
version: number;
state: string;
role: string | null;
mutation_id: string;
changed_by_type: string;
changed_by_id: string;
created_at_ms: number | string;
}
function assertExactKeys(value: object, expected: readonly string[]): void {
const keys = Object.keys(value).sort();
const canonical = [...expected].sort();
if (
keys.length !== canonical.length ||
keys.some((key, index) => key !== canonical[index])
) {
throw new TypeError('Project owner bootstrap command shape is invalid');
}
}
function assertTimestamp(name: string, value: number): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new TypeError(`Project owner bootstrap ${name} is invalid`);
}
}
function normalizeIssueCommand(
command: IssueProjectOwnerBootstrapChallengeCommand,
): Readonly<IssueProjectOwnerBootstrapChallengeCommand> {
if (!command || typeof command !== 'object' || Array.isArray(command)) {
throw new TypeError('Project owner bootstrap issue command is invalid');
}
assertExactKeys(command, [
'projectId',
'challengeId',
'tokenDigest',
'issuedAtMs',
'expiresAtMs',
]);
assertProjectPolicyProjectId(command.projectId);
assertProjectOwnerBootstrapChallengeId(command.challengeId);
assertProjectOwnerBootstrapTokenDigest(command.tokenDigest);
assertTimestamp('issuedAtMs', command.issuedAtMs);
assertTimestamp('expiresAtMs', command.expiresAtMs);
if (command.expiresAtMs <= command.issuedAtMs) {
throw new TypeError('Project owner bootstrap lifetime is invalid');
}
return Object.freeze({ ...command });
}
function normalizeClaimCommand(
command: ClaimProjectOwnerBootstrapChallengeCommand,
): Readonly<ClaimProjectOwnerBootstrapChallengeCommand> {
if (!command || typeof command !== 'object' || Array.isArray(command)) {
throw new TypeError('Project owner bootstrap claim command is invalid');
}
assertExactKeys(command, [
'projectId',
'challengeId',
'tokenDigest',
'subject',
'claimedAtMs',
]);
assertProjectPolicyProjectId(command.projectId);
assertProjectOwnerBootstrapChallengeId(command.challengeId);
assertProjectOwnerBootstrapTokenDigest(command.tokenDigest);
const subject = normalizePolicySubject(command.subject);
if (subject.type !== 'user') {
throw new ProjectOwnerBootstrapClaimRejectedError();
}
assertTimestamp('claimedAtMs', command.claimedAtMs);
return Object.freeze({ ...command, subject });
}
function rowToChallenge(
row: BootstrapChallengeRow,
): Readonly<ProjectOwnerBootstrapChallengeRecord> {
return normalizeProjectOwnerBootstrapChallengeRecord({
projectId: row.project_id,
version: Number(row.version),
challengeId: row.challenge_id,
tokenDigest: row.token_digest,
issuedAtMs: Number(row.issued_at_ms),
expiresAtMs: Number(row.expires_at_ms),
...(row.consumed_at_ms === null
? {}
: {
consumedAtMs: Number(row.consumed_at_ms),
claimedSubject: {
type: row.claimed_subject_type as NonNullable<
ProjectOwnerBootstrapChallengeRecord['claimedSubject']
>['type'],
id: row.claimed_subject_id!,
},
}),
});
}
function rowToBinding(
row: BootstrapBindingRow,
): Readonly<ProjectRoleBindingRecord> {
return normalizeProjectRoleBindingRecord({
projectId: row.project_id,
subject: {
type: row.subject_type as ProjectRoleBindingRecord['subject']['type'],
id: row.subject_id,
},
version: Number(row.version),
state: row.state as ProjectRoleBindingRecord['state'],
...(row.role === null
? {}
: { role: row.role as NonNullable<ProjectRoleBindingRecord['role']> }),
mutationId: row.mutation_id,
changedBy: {
type: row.changed_by_type as ProjectRoleBindingRecord['changedBy']['type'],
id: row.changed_by_id,
},
createdAtMs: Number(row.created_at_ms),
});
}
function digestMatches(expected: string, actual: string): boolean {
assertProjectOwnerBootstrapTokenDigest(expected);
assertProjectOwnerBootstrapTokenDigest(actual);
return timingSafeEqual(
Buffer.from(expected, 'hex'),
Buffer.from(actual, 'hex'),
);
}
function mutationId(challengeId: string): string {
return `owner-bootstrap:${challengeId}`;
}
function errorCode(error: unknown): string | undefined {
if (!error || typeof error !== 'object') return undefined;
for (const candidate of [
error,
'original' in error ? error.original : undefined,
'parent' in error ? error.parent : undefined,
]) {
if (
candidate &&
typeof candidate === 'object' &&
'code' in candidate &&
typeof candidate.code === 'string'
) {
return candidate.code;
}
}
return undefined;
}
function retryDelay(attempt: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
}
function isExpectedError(error: unknown): boolean {
return (
error instanceof ProjectOwnerBootstrapChallengeActiveError ||
error instanceof ProjectOwnerBootstrapClaimRejectedError ||
error instanceof ProjectOwnerBootstrapProjectInactiveError ||
error instanceof ProjectOwnerBootstrapProjectNotFoundError ||
error instanceof ProjectOwnerBootstrapProjectNotPristineError ||
error instanceof ProjectOwnerBootstrapUnavailableError
);
}
export class LegacySequelizeProjectOwnerBootstrapRepository
implements ProjectOwnerBootstrapRepository
{
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Project owner bootstrap repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
}
async issue(
rawCommand: IssueProjectOwnerBootstrapChallengeCommand,
): Promise<Readonly<ProjectOwnerBootstrapChallengeRecord>> {
const command = normalizeIssueCommand(rawCommand);
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
try {
return await this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
await this.assertActiveProject(command.projectId, transaction);
if ((await this.bindingCount(command.projectId, transaction)) > 0) {
throw new ProjectOwnerBootstrapProjectNotPristineError();
}
const latest = await this.latestChallenge(
command.projectId,
transaction,
);
if (
latest?.consumedAtMs !== undefined ||
(latest && latest.expiresAtMs > command.issuedAtMs)
) {
if (latest?.consumedAtMs !== undefined) {
throw new ProjectOwnerBootstrapProjectNotPristineError();
}
throw new ProjectOwnerBootstrapChallengeActiveError();
}
const version = (latest?.version ?? 0) + 1;
if (version > OWNER_BOOTSTRAP_MAX_VERSION) {
throw new ProjectOwnerBootstrapUnavailableError();
}
await this.database.query(
`INSERT INTO "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}"
(project_id, version, challenge_id, token_digest,
issued_at_ms, expires_at_ms, consumed_at_ms,
claimed_subject_type, claimed_subject_id)
VALUES
(:projectId, :version, :challengeId, :tokenDigest,
:issuedAtMs, :expiresAtMs, NULL, NULL, NULL)`,
{
type: QueryTypes.INSERT,
replacements: { ...command, version },
transaction,
},
);
return normalizeProjectOwnerBootstrapChallengeRecord({
...command,
version,
});
},
);
} catch (error) {
if (isExpectedError(error)) throw error;
if (
errorCode(error) === 'SQLITE_BUSY' &&
attempt < RETRY_ATTEMPTS - 1
) {
await retryDelay(attempt);
continue;
}
if (error instanceof UniqueConstraintError) {
throw new ProjectOwnerBootstrapUnavailableError();
}
throw new ProjectOwnerBootstrapUnavailableError();
}
}
throw new ProjectOwnerBootstrapUnavailableError();
}
async claim(
rawCommand: ClaimProjectOwnerBootstrapChallengeCommand,
): Promise<ClaimProjectOwnerBootstrapChallengeResult> {
const command = normalizeClaimCommand(rawCommand);
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
try {
return await this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
await this.assertActiveProject(command.projectId, transaction);
const latest = await this.latestChallenge(
command.projectId,
transaction,
);
if (
!latest ||
latest.challengeId !== command.challengeId ||
!digestMatches(latest.tokenDigest, command.tokenDigest)
) {
throw new ProjectOwnerBootstrapClaimRejectedError();
}
if (latest.consumedAtMs !== undefined) {
if (
latest.claimedSubject?.type !== command.subject.type ||
latest.claimedSubject.id !== command.subject.id
) {
throw new ProjectOwnerBootstrapClaimRejectedError();
}
const binding = await this.bootstrapBinding(
command.projectId,
latest.challengeId,
transaction,
);
if (
!binding ||
binding.subject.type !== command.subject.type ||
binding.subject.id !== command.subject.id ||
binding.role !== 'owner' ||
binding.state !== 'active'
) {
throw new ProjectOwnerBootstrapUnavailableError();
}
return { status: 'existing', binding };
}
if (
command.claimedAtMs < latest.issuedAtMs ||
command.claimedAtMs >= latest.expiresAtMs
) {
throw new ProjectOwnerBootstrapClaimRejectedError();
}
if ((await this.bindingCount(command.projectId, transaction)) > 0) {
throw new ProjectOwnerBootstrapProjectNotPristineError();
}
const binding = normalizeProjectRoleBindingRecord({
projectId: command.projectId,
subject: command.subject,
version: 1,
state: 'active',
role: 'owner',
mutationId: mutationId(latest.challengeId),
changedBy: OWNER_BOOTSTRAP_SYSTEM_SUBJECT,
createdAtMs: command.claimedAtMs,
});
const [, consumedCount] = await this.database.query(
`UPDATE "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}"
SET consumed_at_ms = :claimedAtMs,
claimed_subject_type = :subjectType,
claimed_subject_id = :subjectId
WHERE project_id = :projectId
AND version = :version
AND challenge_id = :challengeId
AND consumed_at_ms IS NULL`,
{
type: QueryTypes.UPDATE,
replacements: {
projectId: command.projectId,
version: latest.version,
challengeId: latest.challengeId,
claimedAtMs: command.claimedAtMs,
subjectType: command.subject.type,
subjectId: command.subject.id,
},
transaction,
},
);
if (consumedCount !== 1) {
throw new ProjectOwnerBootstrapUnavailableError();
}
await this.database.query(
`INSERT INTO "${PROJECT_ROLE_BINDING_TABLE}"
(project_id, subject_type, subject_id, version, state, role,
mutation_id, changed_by_type, changed_by_id, created_at_ms)
VALUES
(:projectId, :subjectType, :subjectId, 1, 'active', 'owner',
:mutationId, :changedByType, :changedById, :createdAtMs)`,
{
type: QueryTypes.INSERT,
replacements: {
projectId: binding.projectId,
subjectType: binding.subject.type,
subjectId: binding.subject.id,
mutationId: binding.mutationId,
changedByType: binding.changedBy.type,
changedById: binding.changedBy.id,
createdAtMs: binding.createdAtMs,
},
transaction,
},
);
return { status: 'claimed', binding };
},
);
} catch (error) {
if (isExpectedError(error)) throw error;
if (
errorCode(error) === 'SQLITE_BUSY' &&
attempt < RETRY_ATTEMPTS - 1
) {
await retryDelay(attempt);
continue;
}
throw new ProjectOwnerBootstrapUnavailableError();
}
}
throw new ProjectOwnerBootstrapUnavailableError();
}
private async assertActiveProject(
projectId: string,
transaction: Transaction,
): Promise<void> {
const projects = await this.database.query<ProjectStatusRow>(
`SELECT status FROM "${PROJECT_TABLE}" WHERE id = :projectId LIMIT 2`,
{
type: QueryTypes.SELECT,
replacements: { projectId },
transaction,
},
);
if (projects.length === 0) {
throw new ProjectOwnerBootstrapProjectNotFoundError();
}
if (projects.length !== 1 || projects[0].status !== 'active') {
throw new ProjectOwnerBootstrapProjectInactiveError();
}
}
private async bindingCount(
projectId: string,
transaction: Transaction,
): Promise<number> {
const rows = await this.database.query<{ count: number | string }>(
`SELECT COUNT(*) AS count
FROM "${PROJECT_ROLE_BINDING_TABLE}"
WHERE project_id = :projectId`,
{
type: QueryTypes.SELECT,
replacements: { projectId },
transaction,
},
);
const count = Number(rows[0]?.count);
if (!Number.isSafeInteger(count) || count < 0) {
throw new ProjectOwnerBootstrapUnavailableError();
}
return count;
}
private async latestChallenge(
projectId: string,
transaction: Transaction,
): Promise<Readonly<ProjectOwnerBootstrapChallengeRecord> | null> {
const rows = await this.database.query<BootstrapChallengeRow>(
`SELECT project_id, version, challenge_id, token_digest,
issued_at_ms, expires_at_ms, consumed_at_ms,
claimed_subject_type, claimed_subject_id
FROM "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}"
WHERE project_id = :projectId
ORDER BY version DESC
LIMIT 1`,
{
type: QueryTypes.SELECT,
replacements: { projectId },
transaction,
},
);
if (rows.length === 0) return null;
if (rows.length !== 1) throw new ProjectOwnerBootstrapUnavailableError();
return rowToChallenge(rows[0]);
}
private async bootstrapBinding(
projectId: string,
challengeId: string,
transaction: Transaction,
): Promise<Readonly<ProjectRoleBindingRecord> | null> {
const rows = await this.database.query<BootstrapBindingRow>(
`SELECT project_id, subject_type, subject_id, version, state, role,
mutation_id, changed_by_type, changed_by_id, created_at_ms
FROM "${PROJECT_ROLE_BINDING_TABLE}"
WHERE project_id = :projectId
AND mutation_id = :mutationId
LIMIT 2`,
{
type: QueryTypes.SELECT,
replacements: { projectId, mutationId: mutationId(challengeId) },
transaction,
},
);
if (rows.length === 0) return null;
if (rows.length !== 1) throw new ProjectOwnerBootstrapUnavailableError();
return rowToBinding(rows[0]);
}
}
@@ -0,0 +1,433 @@
import {
DataTypes,
Model,
ModelStatic,
QueryTypes,
Sequelize,
Transaction,
UniqueConstraintError,
} from 'sequelize';
import {
PROJECT_ROLE_BINDING_TABLE,
PROJECT_TABLE,
} from '../../../migrations/0017-project-policy';
import {
MAX_PROJECT_ROLE_BINDING_VERSION,
ProjectPolicyProjectNotFoundError,
ProjectPolicyUnavailableError,
ProjectRoleBindingMutationConflictError,
ProjectRoleBindingVersionConflictError,
assertProjectPolicyProjectId,
normalizePolicySubject,
normalizeProjectPolicySnapshot,
normalizeProjectRoleBindingRecord,
type ProjectPolicySnapshot,
type ProjectRoleBindingRecord,
} from '../../domain/projectPolicy';
import type {
AppendProjectRoleBindingCommand,
AppendProjectRoleBindingResult,
ProjectPolicyRepository,
} from '../../ports/projectPolicyRepository';
const RETRY_ATTEMPTS = 5;
interface ProjectRoleBindingRow {
projectId: string;
subjectType: string;
subjectId: string;
version: number;
state: string;
role: string | null;
mutationId: string;
changedByType: string;
changedById: string;
createdAtMs: number | string;
}
interface ProjectRoleBindingInstance
extends Model<ProjectRoleBindingRow, ProjectRoleBindingRow>,
ProjectRoleBindingRow {}
interface ProjectPolicySnapshotRow {
project_id: string;
project_name: string;
project_slug: string;
project_status: string;
project_version: number;
project_created_at_ms: number | string;
project_updated_at_ms: number | string;
binding_project_id: string | null;
binding_subject_type: string | null;
binding_subject_id: string | null;
binding_version: number | null;
binding_state: string | null;
binding_role: string | null;
binding_mutation_id: string | null;
binding_changed_by_type: string | null;
binding_changed_by_id: string | null;
binding_created_at_ms: number | string | null;
}
function defineBindingModel(
database: Sequelize,
): ModelStatic<ProjectRoleBindingInstance> {
return database.define<ProjectRoleBindingInstance>(
'Ql3ProjectRoleBinding',
{
projectId: {
field: 'project_id',
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
},
subjectType: {
field: 'subject_type',
type: DataTypes.STRING(32),
allowNull: false,
primaryKey: true,
},
subjectId: {
field: 'subject_id',
type: DataTypes.STRING(255),
allowNull: false,
primaryKey: true,
},
version: {
type: DataTypes.INTEGER,
allowNull: false,
primaryKey: true,
},
state: { type: DataTypes.STRING(16), allowNull: false },
role: { type: DataTypes.STRING(16), allowNull: true },
mutationId: {
field: 'mutation_id',
type: DataTypes.STRING(64),
allowNull: false,
},
changedByType: {
field: 'changed_by_type',
type: DataTypes.STRING(32),
allowNull: false,
},
changedById: {
field: 'changed_by_id',
type: DataTypes.STRING(255),
allowNull: false,
},
createdAtMs: {
field: 'created_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: PROJECT_ROLE_BINDING_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function rowToBinding(
row: ProjectRoleBindingRow,
): Readonly<ProjectRoleBindingRecord> {
try {
return normalizeProjectRoleBindingRecord({
projectId: row.projectId,
subject: {
type: row.subjectType as ProjectRoleBindingRecord['subject']['type'],
id: row.subjectId,
},
version: Number(row.version),
state: row.state as ProjectRoleBindingRecord['state'],
...(row.role === null
? {}
: { role: row.role as NonNullable<ProjectRoleBindingRecord['role']> }),
mutationId: row.mutationId,
changedBy: {
type: row.changedByType as ProjectRoleBindingRecord['changedBy']['type'],
id: row.changedById,
},
createdAtMs: Number(row.createdAtMs),
});
} catch {
throw new ProjectPolicyUnavailableError();
}
}
function snapshotRowToValue(
row: ProjectPolicySnapshotRow,
): Readonly<ProjectPolicySnapshot> {
const bindingFields = [
row.binding_project_id,
row.binding_subject_type,
row.binding_subject_id,
row.binding_version,
row.binding_state,
row.binding_mutation_id,
row.binding_changed_by_type,
row.binding_changed_by_id,
row.binding_created_at_ms,
];
const noBinding = bindingFields.every((value) => value === null);
if (!noBinding && bindingFields.some((value) => value === null)) {
throw new ProjectPolicyUnavailableError();
}
try {
return normalizeProjectPolicySnapshot({
project: {
id: row.project_id,
name: row.project_name,
slug: row.project_slug,
status:
row.project_status as ProjectPolicySnapshot['project']['status'],
version: Number(row.project_version),
createdAtMs: Number(row.project_created_at_ms),
updatedAtMs: Number(row.project_updated_at_ms),
},
...(noBinding
? {}
: {
binding: {
projectId: row.binding_project_id!,
subject: {
type: row.binding_subject_type as ProjectRoleBindingRecord['subject']['type'],
id: row.binding_subject_id!,
},
version: Number(row.binding_version),
state: row.binding_state as ProjectRoleBindingRecord['state'],
...(row.binding_role === null
? {}
: {
role: row.binding_role as NonNullable<
ProjectRoleBindingRecord['role']
>,
}),
mutationId: row.binding_mutation_id!,
changedBy: {
type: row.binding_changed_by_type as ProjectRoleBindingRecord['changedBy']['type'],
id: row.binding_changed_by_id!,
},
createdAtMs: Number(row.binding_created_at_ms),
},
}),
});
} catch (error) {
if (error instanceof ProjectPolicyUnavailableError) throw error;
throw new ProjectPolicyUnavailableError();
}
}
function sameBinding(
left: Readonly<ProjectRoleBindingRecord>,
right: Readonly<ProjectRoleBindingRecord>,
): boolean {
return (
left.projectId === right.projectId &&
left.subject.type === right.subject.type &&
left.subject.id === right.subject.id &&
left.version === right.version &&
left.state === right.state &&
left.role === right.role &&
left.mutationId === right.mutationId &&
left.changedBy.type === right.changedBy.type &&
left.changedBy.id === right.changedBy.id &&
left.createdAtMs === right.createdAtMs
);
}
function assertExpectedVersion(value: number): void {
if (
!Number.isSafeInteger(value) ||
value < 0 ||
value >= MAX_PROJECT_ROLE_BINDING_VERSION
) {
throw new TypeError('Project role binding expected version is invalid');
}
}
function errorCode(error: unknown): string | undefined {
if (!error || typeof error !== 'object') return undefined;
for (const candidate of [
error,
'original' in error ? error.original : undefined,
'parent' in error ? error.parent : undefined,
]) {
if (
candidate &&
typeof candidate === 'object' &&
'code' in candidate &&
typeof candidate.code === 'string'
) {
return candidate.code;
}
}
return undefined;
}
function retryDelay(attempt: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
}
export class LegacySequelizeProjectPolicyRepository
implements ProjectPolicyRepository
{
private readonly bindings: ModelStatic<ProjectRoleBindingInstance>;
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Project policy repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
this.bindings = defineBindingModel(database);
}
async resolve(
projectId: string,
requestedSubject: Parameters<ProjectPolicyRepository['resolve']>[1],
): Promise<Readonly<ProjectPolicySnapshot> | null> {
assertProjectPolicyProjectId(projectId);
const subject = normalizePolicySubject(requestedSubject);
const rows = await this.database.query<ProjectPolicySnapshotRow>(
`SELECT project.id AS project_id,
project.name AS project_name,
project.slug AS project_slug,
project.status AS project_status,
project.version AS project_version,
project.created_at_ms AS project_created_at_ms,
project.updated_at_ms AS project_updated_at_ms,
binding.project_id AS binding_project_id,
binding.subject_type AS binding_subject_type,
binding.subject_id AS binding_subject_id,
binding.version AS binding_version,
binding.state AS binding_state,
binding.role AS binding_role,
binding.mutation_id AS binding_mutation_id,
binding.changed_by_type AS binding_changed_by_type,
binding.changed_by_id AS binding_changed_by_id,
binding.created_at_ms AS binding_created_at_ms
FROM "${PROJECT_TABLE}" AS project
LEFT JOIN "${PROJECT_ROLE_BINDING_TABLE}" AS binding
ON binding.project_id = project.id
AND binding.subject_type = :subjectType
AND binding.subject_id = :subjectId
AND binding.version = (
SELECT MAX(current.version)
FROM "${PROJECT_ROLE_BINDING_TABLE}" AS current
WHERE current.project_id = project.id
AND current.subject_type = :subjectType
AND current.subject_id = :subjectId
)
WHERE project.id = :projectId
LIMIT 2`,
{
type: QueryTypes.SELECT,
replacements: {
projectId,
subjectType: subject.type,
subjectId: subject.id,
},
},
);
if (rows.length === 0) return null;
if (rows.length !== 1) throw new ProjectPolicyUnavailableError();
return snapshotRowToValue(rows[0]);
}
async append(
command: AppendProjectRoleBindingCommand,
): Promise<AppendProjectRoleBindingResult> {
if (!command || typeof command !== 'object' || Array.isArray(command)) {
throw new TypeError('Project role binding command must be an object');
}
assertExpectedVersion(command.expectedCurrentVersion);
const binding = normalizeProjectRoleBindingRecord(command.binding);
if (binding.version !== command.expectedCurrentVersion + 1) {
throw new ProjectRoleBindingVersionConflictError();
}
const values: ProjectRoleBindingRow = {
projectId: binding.projectId,
subjectType: binding.subject.type,
subjectId: binding.subject.id,
version: binding.version,
state: binding.state,
role: binding.role ?? null,
mutationId: binding.mutationId,
changedByType: binding.changedBy.type,
changedById: binding.changedBy.id,
createdAtMs: binding.createdAtMs,
};
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
try {
return await this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const replay = await this.bindings.findOne({
where: {
projectId: binding.projectId,
mutationId: binding.mutationId,
},
raw: true,
transaction,
});
if (replay) {
const previous = rowToBinding(replay);
if (!sameBinding(previous, binding)) {
throw new ProjectRoleBindingMutationConflictError();
}
return { status: 'existing', binding: previous };
}
const projects = await this.database.query<{ id: string }>(
`SELECT id FROM "${PROJECT_TABLE}" WHERE id = :projectId LIMIT 1`,
{
type: QueryTypes.SELECT,
replacements: { projectId: binding.projectId },
transaction,
},
);
if (projects.length !== 1) {
throw new ProjectPolicyProjectNotFoundError();
}
const current = await this.bindings.findOne({
where: {
projectId: binding.projectId,
subjectType: binding.subject.type,
subjectId: binding.subject.id,
},
order: [['version', 'DESC']],
raw: true,
transaction,
});
const currentVersion = current ? Number(current.version) : 0;
if (currentVersion !== command.expectedCurrentVersion) {
throw new ProjectRoleBindingVersionConflictError();
}
await this.bindings.create(values, { transaction });
return { status: 'inserted', binding };
},
);
} catch (error) {
if (
error instanceof ProjectRoleBindingVersionConflictError ||
error instanceof ProjectRoleBindingMutationConflictError ||
error instanceof ProjectPolicyProjectNotFoundError ||
error instanceof ProjectPolicyUnavailableError
) {
throw error;
}
if (
(error instanceof UniqueConstraintError ||
errorCode(error) === 'SQLITE_BUSY') &&
attempt < RETRY_ATTEMPTS - 1
) {
await retryDelay(attempt);
continue;
}
throw error;
}
}
throw new ProjectPolicyUnavailableError();
}
}
@@ -0,0 +1,155 @@
import { Sequelize, Transaction } from 'sequelize';
import type {
RunAttemptRecord,
RunAttemptStatus,
RunEventRecord,
RunRecord,
} from '../../domain/run';
import type { RunRetryPolicyRecord } from '../../domain/runRetryPolicy';
import type { RunRepositoryTransaction } from '../../ports/runRepository';
import {
LegacySequelizeRunRepository,
LegacySequelizeRunTransaction,
} from './runRepository';
export interface SequelizeRunProjectionContext {
transaction: Transaction;
runs: RunRepositoryTransaction;
changedRunIds: readonly string[];
changedAttemptIds: readonly string[];
}
export interface SequelizeRunProjectionParticipant {
apply(context: SequelizeRunProjectionContext): Promise<void>;
}
class TrackingRunRepositoryTransaction implements RunRepositoryTransaction {
readonly changedRunIds = new Set<string>();
readonly changedAttemptIds = new Set<string>();
constructor(private readonly delegate: RunRepositoryTransaction) {}
findRunById(runId: string): Promise<RunRecord | null> {
return this.delegate.findRunById(runId);
}
findAttemptById(attemptId: string): Promise<RunAttemptRecord | null> {
return this.delegate.findAttemptById(attemptId);
}
findLatestAttemptByRunId(runId: string): Promise<RunAttemptRecord | null> {
return this.delegate.findLatestAttemptByRunId(runId);
}
findRetryPolicyByRunId(runId: string): Promise<RunRetryPolicyRecord | null> {
return this.delegate.findRetryPolicyByRunId(runId);
}
listEvents(
runId: string,
options?: { afterSequence?: number; limit?: number },
): Promise<RunEventRecord[]> {
return this.delegate.listEvents(runId, options);
}
listCancellationRequested(options?: {
beforeMs?: number;
limit?: number;
}): Promise<RunRecord[]> {
return this.delegate.listCancellationRequested(options);
}
async insertRun(run: RunRecord): Promise<void> {
await this.delegate.insertRun(run);
this.changedRunIds.add(run.id);
}
async insertAttempt(attempt: RunAttemptRecord): Promise<void> {
await this.delegate.insertAttempt(attempt);
this.changedRunIds.add(attempt.runId);
this.changedAttemptIds.add(attempt.id);
}
insertRetryPolicy(policy: RunRetryPolicyRecord): Promise<void> {
return this.delegate.insertRetryPolicy(policy);
}
async compareAndSetRun(
run: RunRecord,
expectedVersion: number,
): Promise<boolean> {
const updated = await this.delegate.compareAndSetRun(run, expectedVersion);
if (updated) this.changedRunIds.add(run.id);
return updated;
}
async compareAndSetAttempt(
attempt: RunAttemptRecord,
expected: { status: RunAttemptStatus; callbackSequence: number },
): Promise<boolean> {
const updated = await this.delegate.compareAndSetAttempt(attempt, expected);
if (updated) {
this.changedRunIds.add(attempt.runId);
this.changedAttemptIds.add(attempt.id);
}
return updated;
}
compareAndSetRetryPolicy(
policy: RunRetryPolicyRecord,
expectedVersion: number,
): Promise<boolean> {
return this.delegate.compareAndSetRetryPolicy(policy, expectedVersion);
}
appendEvent(event: RunEventRecord): Promise<void> {
return this.delegate.appendEvent(event);
}
}
/**
* Primary-only repository. Existing Shadow repositories keep their original
* transaction implementation and never execute these projection participants.
*/
export class LegacySequelizeProjectedRunRepository extends LegacySequelizeRunRepository {
private readonly participants: readonly SequelizeRunProjectionParticipant[];
constructor(
private readonly projectedDatabase: Sequelize,
participants: readonly SequelizeRunProjectionParticipant[],
) {
super(projectedDatabase);
this.participants = [...participants];
}
override async transaction<T>(
work: (transaction: RunRepositoryTransaction) => Promise<T>,
): Promise<T> {
return this.projectedDatabase.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const runs = new LegacySequelizeRunTransaction(
this.models,
transaction,
);
const tracked = new TrackingRunRepositoryTransaction(runs);
const result = await work(tracked);
if (
tracked.changedRunIds.size > 0 ||
tracked.changedAttemptIds.size > 0
) {
const context: SequelizeRunProjectionContext = {
transaction,
runs,
changedRunIds: [...tracked.changedRunIds],
changedAttemptIds: [...tracked.changedAttemptIds],
};
for (const participant of this.participants) {
await participant.apply(context);
}
}
return result;
},
);
}
}
@@ -0,0 +1,136 @@
import { QueryTypes, Sequelize } from 'sequelize';
import {
RUN_ATTEMPT_TABLE,
RUN_TABLE,
} from '../../../migrations/0002-run-schema';
import { RUN_DISPATCH_LEASE_TABLE } from '../../../migrations/0009-run-dispatch-lease';
import { RUN_DISPATCH_CANDIDATE_RUN_INDEX } from '../../../migrations/0010-run-dispatch-candidates';
import {
MAX_RUN_DISPATCH_CANDIDATE_PAGE_SIZE,
assertRunDispatchCandidate,
assertRunDispatchCandidateCursor,
assertRunDispatchCandidatePageSize,
type RunDispatchCandidate,
} from '../../domain/runDispatchCandidate';
import { assertRunDispatchLeaseVersion } from '../../domain/runDispatchLease';
import type {
ListRunDispatchCandidatesOptions,
RunDispatchCandidateSource,
} from '../../ports/runDispatchCandidateSource';
interface CandidateRow {
runId: string;
attemptId: string;
projectId: string;
taskId: string;
taskRevision: string;
priority: number | string;
queuedAtMs: number | string;
attemptCreatedAtMs: number | string;
executorType: string;
}
const CANDIDATE_ORDER = `
r.priority DESC,
r.queued_at_ms ASC,
a.created_at_ms ASC,
a.id ASC
`;
export class LegacySequelizeRunDispatchCandidateSource
implements RunDispatchCandidateSource
{
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Legacy Run dispatch candidate source is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
}
async listCandidates({
observedAtMs,
after,
limit = MAX_RUN_DISPATCH_CANDIDATE_PAGE_SIZE,
}: ListRunDispatchCandidatesOptions): Promise<RunDispatchCandidate[]> {
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
assertRunDispatchCandidatePageSize(limit);
if (after) assertRunDispatchCandidateCursor(after);
const cursorPredicate = after
? `AND (
r.priority < :afterPriority
OR (r.priority = :afterPriority AND r.queued_at_ms > :afterQueuedAtMs)
OR (
r.priority = :afterPriority
AND r.queued_at_ms = :afterQueuedAtMs
AND a.created_at_ms > :afterAttemptCreatedAtMs
)
OR (
r.priority = :afterPriority
AND r.queued_at_ms = :afterQueuedAtMs
AND a.created_at_ms = :afterAttemptCreatedAtMs
AND a.id > :afterAttemptId
)
)`
: '';
const rows = await this.database.query<CandidateRow>(
`SELECT
r.id AS runId,
a.id AS attemptId,
r.project_id AS projectId,
r.task_id AS taskId,
r.task_revision AS taskRevision,
r.priority AS priority,
r.queued_at_ms AS queuedAtMs,
a.created_at_ms AS attemptCreatedAtMs,
a.executor_type AS executorType
FROM ${RUN_TABLE} r INDEXED BY ${RUN_DISPATCH_CANDIDATE_RUN_INDEX}
INNER JOIN ${RUN_ATTEMPT_TABLE} a ON a.run_id = r.id
LEFT JOIN ${RUN_DISPATCH_LEASE_TABLE} l ON l.attempt_id = a.id
WHERE r.execution_owner = 'runtime'
AND r.status IN ('queued', 'dispatching')
AND r.queued_at_ms IS NOT NULL
AND r.cancel_requested_at_ms IS NULL
AND a.status = 'claimed'
AND (
l.attempt_id IS NULL
OR l.status = 'released'
OR (l.status = 'leased' AND l.expires_at_ms <= :observedAtMs)
)
${cursorPredicate}
ORDER BY ${CANDIDATE_ORDER}
LIMIT :limit`,
{
type: QueryTypes.SELECT,
replacements: {
observedAtMs,
limit,
...(after
? {
afterPriority: after.priority,
afterQueuedAtMs: after.queuedAtMs,
afterAttemptCreatedAtMs: after.attemptCreatedAtMs,
afterAttemptId: after.attemptId,
}
: {}),
},
},
);
return rows.map((row) => {
const candidate: RunDispatchCandidate = {
runId: row.runId,
attemptId: row.attemptId,
projectId: row.projectId,
taskId: row.taskId,
taskRevision: row.taskRevision,
priority: Number(row.priority),
queuedAtMs: Number(row.queuedAtMs),
attemptCreatedAtMs: Number(row.attemptCreatedAtMs),
executorType: row.executorType,
};
assertRunDispatchCandidate(candidate);
return candidate;
});
}
}
@@ -0,0 +1,118 @@
import { QueryTypes, Sequelize } from 'sequelize';
import {
RUN_ATTEMPT_TABLE,
RUN_TABLE,
} from '../../../migrations/0002-run-schema';
import {
RUN_DISPATCH_LEASE_EXPIRY_INDEX,
RUN_DISPATCH_LEASE_TABLE,
} from '../../../migrations/0009-run-dispatch-lease';
import {
assertRunDispatchId,
assertRunDispatchLeaseVersion,
} from '../../domain/runDispatchLease';
import {
MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE,
type ExpiredRunDispatchLeaseCandidate,
type ListExpiredRunDispatchLeasesOptions,
type RunDispatchLeaseExpirySource,
} from '../../ports/runDispatchLeaseExpirySource';
interface ExpiredLeaseRow {
runId: string;
attemptId: string;
expiresAtMs: number | string;
}
function assertLimit(limit: number): void {
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE
) {
throw new RangeError(
`limit must be between 1 and ${MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE}`,
);
}
}
export class LegacySequelizeRunDispatchLeaseExpirySource
implements RunDispatchLeaseExpirySource
{
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Legacy Run dispatch lease expiry source is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
}
async listExpired({
observedAtMs,
after,
limit = 16,
}: ListExpiredRunDispatchLeasesOptions): Promise<
readonly ExpiredRunDispatchLeaseCandidate[]
> {
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
assertLimit(limit);
if (after) {
assertRunDispatchLeaseVersion('after.expiresAtMs', after.expiresAtMs);
assertRunDispatchId('after.attemptId', after.attemptId);
}
const cursorPredicate = after
? `AND (
l.expires_at_ms > :afterExpiresAtMs
OR (
l.expires_at_ms = :afterExpiresAtMs
AND l.attempt_id > :afterAttemptId
)
)`
: '';
const rows = await this.database.query<ExpiredLeaseRow>(
`SELECT
l.run_id AS runId,
l.attempt_id AS attemptId,
l.expires_at_ms AS expiresAtMs
FROM ${RUN_DISPATCH_LEASE_TABLE} l INDEXED BY ${RUN_DISPATCH_LEASE_EXPIRY_INDEX}
INNER JOIN ${RUN_TABLE} r ON r.id = l.run_id
INNER JOIN ${RUN_ATTEMPT_TABLE} a ON a.id = l.attempt_id
WHERE l.status = 'leased'
AND l.expires_at_ms <= :observedAtMs
AND r.execution_owner = 'runtime'
AND r.status IN ('dispatching', 'running')
AND a.run_id = r.id
AND a.status IN ('claimed', 'starting', 'running')
${cursorPredicate}
ORDER BY l.expires_at_ms ASC, l.attempt_id ASC
LIMIT :limit`,
{
type: QueryTypes.SELECT,
replacements: {
observedAtMs,
limit,
...(after
? {
afterExpiresAtMs: after.expiresAtMs,
afterAttemptId: after.attemptId,
}
: {}),
},
},
);
return rows.map((row) => {
const candidate = {
runId: row.runId,
attemptId: row.attemptId,
expiresAtMs: Number(row.expiresAtMs),
};
assertRunDispatchId('runId', candidate.runId);
assertRunDispatchId('attemptId', candidate.attemptId);
assertRunDispatchLeaseVersion('expiresAtMs', candidate.expiresAtMs);
if (candidate.expiresAtMs > observedAtMs) {
throw new TypeError('Expiry source returned a live Run lease');
}
return candidate;
});
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,164 @@
import { QueryTypes, Sequelize } from 'sequelize';
import {
RUN_ATTEMPT_TABLE,
RUN_TABLE,
} from '../../../migrations/0002-run-schema';
import {
RUN_DISPATCH_LEASE_EXPIRY_INDEX,
RUN_DISPATCH_LEASE_TABLE,
} from '../../../migrations/0009-run-dispatch-lease';
import { WORKER_REGISTRY_TABLE } from '../../../migrations/0008-worker-registry';
import type { RunDispatchCandidate } from '../../domain/runDispatchCandidate';
import {
MAX_RUN_DISPATCH_RECOVERY_PAGE_SIZE,
assertRecoverableRunDispatch,
assertRunDispatchRecoveryCursor,
assertRunDispatchRecoveryPageSize,
type RecoverableRunDispatch,
} from '../../domain/runDispatchRecovery';
import {
assertRunDispatchLeaseVersion,
type RunDispatchLeaseRecord,
} from '../../domain/runDispatchLease';
import type {
ListRecoverableRunDispatchesOptions,
RunDispatchRecoverySource,
} from '../../ports/runDispatchRecoverySource';
interface RecoveryRow {
runId: string;
attemptId: string;
projectId: string;
taskId: string;
taskRevision: string;
priority: number | string;
queuedAtMs: number | string;
attemptCreatedAtMs: number | string;
executorType: string;
version: number | string;
leaseGeneration: number | string;
workerId: string;
workerSessionId: string;
workerGeneration: number | string;
leaseToken: string;
acquiredAtMs: number | string;
renewedAtMs: number | string;
expiresAtMs: number | string;
updatedAtMs: number | string;
}
export class LegacySequelizeRunDispatchRecoverySource
implements RunDispatchRecoverySource
{
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Legacy Run dispatch recovery source is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
}
async listRecoverable({
observedAtMs,
after,
limit = MAX_RUN_DISPATCH_RECOVERY_PAGE_SIZE,
}: ListRecoverableRunDispatchesOptions): Promise<RecoverableRunDispatch[]> {
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
assertRunDispatchRecoveryPageSize(limit);
if (after) assertRunDispatchRecoveryCursor(after);
const cursorPredicate = after
? `AND (
l.expires_at_ms > :afterExpiresAtMs
OR (
l.expires_at_ms = :afterExpiresAtMs
AND l.attempt_id > :afterAttemptId
)
)`
: '';
const rows = await this.database.query<RecoveryRow>(
`SELECT
r.id AS runId,
a.id AS attemptId,
r.project_id AS projectId,
r.task_id AS taskId,
r.task_revision AS taskRevision,
r.priority AS priority,
r.queued_at_ms AS queuedAtMs,
a.created_at_ms AS attemptCreatedAtMs,
a.executor_type AS executorType,
l.version AS version,
l.lease_generation AS leaseGeneration,
l.worker_id AS workerId,
l.worker_session_id AS workerSessionId,
l.worker_generation AS workerGeneration,
l.lease_token AS leaseToken,
l.acquired_at_ms AS acquiredAtMs,
l.renewed_at_ms AS renewedAtMs,
l.expires_at_ms AS expiresAtMs,
l.updated_at_ms AS updatedAtMs
FROM ${RUN_DISPATCH_LEASE_TABLE} l INDEXED BY ${RUN_DISPATCH_LEASE_EXPIRY_INDEX}
INNER JOIN ${RUN_TABLE} r ON r.id = l.run_id
INNER JOIN ${RUN_ATTEMPT_TABLE} a ON a.id = l.attempt_id
INNER JOIN ${WORKER_REGISTRY_TABLE} w ON w.id = l.worker_id
WHERE l.status = 'leased'
AND l.expires_at_ms > :observedAtMs
AND r.execution_owner = 'runtime'
AND r.status = 'dispatching'
AND r.cancel_requested_at_ms IS NULL
AND r.queued_at_ms IS NOT NULL
AND a.run_id = r.id
AND a.status = 'claimed'
AND w.session_id = l.worker_session_id
AND w.generation = l.worker_generation
AND w.status IN ('online', 'draining')
AND w.lease_expires_at_ms > :observedAtMs
${cursorPredicate}
ORDER BY l.expires_at_ms ASC, l.attempt_id ASC
LIMIT :limit`,
{
type: QueryTypes.SELECT,
replacements: {
observedAtMs,
limit,
...(after
? {
afterExpiresAtMs: after.expiresAtMs,
afterAttemptId: after.attemptId,
}
: {}),
},
},
);
return rows.map((row) => {
const candidate: RunDispatchCandidate = {
runId: row.runId,
attemptId: row.attemptId,
projectId: row.projectId,
taskId: row.taskId,
taskRevision: row.taskRevision,
priority: Number(row.priority),
queuedAtMs: Number(row.queuedAtMs),
attemptCreatedAtMs: Number(row.attemptCreatedAtMs),
executorType: row.executorType,
};
const lease: RunDispatchLeaseRecord = {
attemptId: row.attemptId,
runId: row.runId,
status: 'leased',
version: Number(row.version),
leaseGeneration: Number(row.leaseGeneration),
workerId: row.workerId,
workerSessionId: row.workerSessionId,
workerGeneration: Number(row.workerGeneration),
leaseToken: row.leaseToken,
acquiredAtMs: Number(row.acquiredAtMs),
renewedAtMs: Number(row.renewedAtMs),
expiresAtMs: Number(row.expiresAtMs),
updatedAtMs: Number(row.updatedAtMs),
};
const recovery = { candidate, lease };
assertRecoverableRunDispatch(recovery);
return recovery;
});
}
}
@@ -0,0 +1,85 @@
import { QueryTypes, Sequelize } from 'sequelize';
import { RUN_TABLE } from '../../../migrations/0002-run-schema';
import {
RUN_LOST_RETRY_INDEX,
RUN_RETRY_POLICY_DUE_INDEX,
RUN_RETRY_POLICY_TABLE,
} from '../../../migrations/0011-run-retry-policy';
import {
MAX_RUN_LOST_RETRY_PAGE_SIZE,
type ListRunLostRetryCandidatesOptions,
type RunLostRetryCandidate,
type RunLostRetrySource,
} from '../../ports/runLostRetrySource';
interface RunLostRetryCandidateRow {
runId: string;
phase: 'lost' | 'retry_wait';
availableAtMs: number | string;
}
export class LegacySequelizeRunLostRetrySource implements RunLostRetrySource {
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Legacy lost retry source is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
}
async listCandidates({
observedAtMs,
limit = 16,
}: ListRunLostRetryCandidatesOptions): Promise<
readonly RunLostRetryCandidate[]
> {
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
throw new RangeError('observedAtMs must be a non-negative safe integer');
}
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_RUN_LOST_RETRY_PAGE_SIZE
) {
throw new RangeError(
`limit must be between 1 and ${MAX_RUN_LOST_RETRY_PAGE_SIZE}`,
);
}
const rows = await this.database.query<RunLostRetryCandidateRow>(
`SELECT runId, phase, availableAtMs
FROM (
SELECT
r.id AS runId,
'lost' AS phase,
0 AS availableAtMs
FROM ${RUN_TABLE} r INDEXED BY ${RUN_LOST_RETRY_INDEX}
WHERE r.execution_owner = 'runtime'
AND r.status = 'lost'
UNION ALL
SELECT
r.id AS runId,
'retry_wait' AS phase,
p.next_attempt_at_ms AS availableAtMs
FROM ${RUN_RETRY_POLICY_TABLE} p INDEXED BY ${RUN_RETRY_POLICY_DUE_INDEX}
INNER JOIN ${RUN_TABLE} r ON r.id = p.run_id
WHERE p.next_attempt_at_ms IS NOT NULL
AND p.next_attempt_at_ms <= :observedAtMs
AND r.execution_owner = 'runtime'
AND r.status = 'retry_wait'
) candidates
ORDER BY availableAtMs ASC, runId ASC
LIMIT :limit`,
{
type: QueryTypes.SELECT,
replacements: { observedAtMs, limit },
},
);
return rows.map((row) => ({
runId: row.runId,
phase: row.phase,
availableAtMs: Number(row.availableAtMs),
}));
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,278 @@
import {
DataTypes,
Model,
ModelStatic,
Sequelize,
UniqueConstraintError,
} from 'sequelize';
import { TASK_EXECUTION_REVISION_TABLE } from '../../../migrations/0012-task-execution-revisions';
import { EXECUTOR_TYPES, type ExecutorType } from '../../domain/execution';
import type { PinnedTaskExecutionRevision } from '../../domain/taskExecutionRevision';
import {
createPinnedTaskExecutionRevisionRecord,
normalizePinnedTaskExecutionRevision,
taskExecutionRevisionDigest,
TaskExecutionRevisionCorruptError,
} from '../../domain/taskExecutionRevisionRecord';
import type {
InsertTaskExecutionRevisionResult,
TaskExecutionRevisionRepository,
} from '../../ports/taskExecutionRevisionRepository';
import type { TaskExecutionRevisionRequest } from '../../ports/taskExecutionRevisionSource';
interface TaskExecutionRevisionRow {
projectId: string;
taskId: string;
taskRevision: string;
executorType: string;
executionTemplate: string;
contextRef: string;
contentDigest: string;
createdAtMs: number | string;
}
interface TaskExecutionRevisionInstance
extends Model<TaskExecutionRevisionRow, TaskExecutionRevisionRow>,
TaskExecutionRevisionRow {}
export class TaskExecutionRevisionConflictError extends Error {
constructor(
readonly projectId: string,
readonly taskId: string,
readonly taskRevision: string,
) {
super(
`Task execution revision ${projectId}/${taskId}@${taskRevision} is immutable`,
);
this.name = 'TaskExecutionRevisionConflictError';
}
}
function defineTaskExecutionRevisionModel(
database: Sequelize,
): ModelStatic<TaskExecutionRevisionInstance> {
return database.define<TaskExecutionRevisionInstance>(
'Ql3TaskExecutionRevision',
{
projectId: {
field: 'project_id',
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
},
taskId: {
field: 'task_id',
type: DataTypes.STRING(255),
allowNull: false,
primaryKey: true,
},
taskRevision: {
field: 'task_revision',
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
},
executorType: {
field: 'executor_type',
type: DataTypes.STRING(64),
allowNull: false,
},
executionTemplate: {
field: 'execution_template',
type: DataTypes.TEXT,
allowNull: false,
},
contextRef: {
field: 'context_ref',
type: DataTypes.STRING(512),
allowNull: false,
},
contentDigest: {
field: 'content_digest',
type: DataTypes.STRING(64),
allowNull: false,
},
createdAtMs: {
field: 'created_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: TASK_EXECUTION_REVISION_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function assertIdentity(name: string, value: string, maximum: number): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new TypeError(`${name} is invalid`);
}
}
function assertRequest(request: Readonly<TaskExecutionRevisionRequest>): void {
if (!request || typeof request !== 'object' || Array.isArray(request)) {
throw new TypeError('Task execution revision request must be an object');
}
assertIdentity('projectId', request.projectId, 128);
assertIdentity('taskId', request.taskId, 255);
assertIdentity('taskRevision', request.taskRevision, 128);
}
function errorCode(error: unknown): string | undefined {
if (!error || typeof error !== 'object') return undefined;
for (const candidate of [
error,
'original' in error ? error.original : undefined,
'parent' in error ? error.parent : undefined,
]) {
if (
candidate &&
typeof candidate === 'object' &&
'code' in candidate &&
typeof candidate.code === 'string'
) {
return candidate.code;
}
}
return undefined;
}
function retryDelay(attempt: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
}
function rowToRevision(
row: TaskExecutionRevisionRow,
): PinnedTaskExecutionRevision {
if (!EXECUTOR_TYPES.includes(row.executorType as ExecutorType)) {
throw new TaskExecutionRevisionCorruptError(
'Stored Task execution revision has an invalid executor type',
);
}
let execution: unknown;
try {
execution = JSON.parse(row.executionTemplate);
} catch {
throw new TaskExecutionRevisionCorruptError(
'Stored Task execution revision template is not valid JSON',
);
}
try {
const normalized = normalizePinnedTaskExecutionRevision({
projectId: row.projectId,
taskId: row.taskId,
taskRevision: row.taskRevision,
executorType: row.executorType as ExecutorType,
execution: execution as PinnedTaskExecutionRevision['execution'],
contextRef: row.contextRef,
});
if (JSON.stringify(normalized.execution) !== row.executionTemplate) {
throw new TaskExecutionRevisionCorruptError(
'Stored Task execution revision template is not canonical',
);
}
if (
!/^[0-9a-f]{64}$/.test(row.contentDigest) ||
taskExecutionRevisionDigest(normalized) !== row.contentDigest
) {
throw new TaskExecutionRevisionCorruptError(
'Stored Task execution revision digest does not match its content',
);
}
const createdAtMs = Number(row.createdAtMs);
return createPinnedTaskExecutionRevisionRecord(normalized, createdAtMs);
} catch (error) {
if (error instanceof TaskExecutionRevisionCorruptError) throw error;
throw new TaskExecutionRevisionCorruptError(
`Stored Task execution revision is invalid: ${
error instanceof Error ? error.message : 'unknown validation error'
}`,
);
}
}
export class LegacySequelizeTaskExecutionRevisionRepository
implements TaskExecutionRevisionRepository
{
private readonly revision: ModelStatic<TaskExecutionRevisionInstance>;
constructor(private readonly database: Sequelize) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'Legacy Task execution revision repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
);
}
this.revision = defineTaskExecutionRevisionModel(database);
}
async resolve(
request: Readonly<TaskExecutionRevisionRequest>,
): Promise<PinnedTaskExecutionRevision | null> {
assertRequest(request);
const row = (await this.revision.findOne({
where: {
projectId: request.projectId,
taskId: request.taskId,
taskRevision: request.taskRevision,
},
raw: true,
})) as unknown as TaskExecutionRevisionRow | null;
return row ? rowToRevision(row) : null;
}
async insert(
revision: PinnedTaskExecutionRevision,
createdAtMs: number,
): Promise<InsertTaskExecutionRevisionResult> {
const record = createPinnedTaskExecutionRevisionRecord(
revision,
createdAtMs,
);
const values: TaskExecutionRevisionRow = {
projectId: record.projectId,
taskId: record.taskId,
taskRevision: record.taskRevision,
executorType: record.executorType,
executionTemplate: JSON.stringify(record.execution),
contextRef: record.contextRef,
contentDigest: record.contentDigest,
createdAtMs: record.createdAtMs,
};
for (let attempt = 0; attempt < 5; attempt += 1) {
try {
await this.revision.create(values);
return 'inserted';
} catch (error) {
if (error instanceof UniqueConstraintError) {
const existing = await this.resolve(record);
if (
existing &&
taskExecutionRevisionDigest(existing) === record.contentDigest
) {
return 'idempotent';
}
throw new TaskExecutionRevisionConflictError(
record.projectId,
record.taskId,
record.taskRevision,
);
}
if (errorCode(error) === 'SQLITE_BUSY' && attempt < 4) {
await retryDelay(attempt);
continue;
}
throw error;
}
}
throw new Error('Task execution revision insert retry budget exhausted');
}
}
@@ -0,0 +1,504 @@
import {
DataTypes,
Model,
ModelStatic,
Op,
Sequelize,
Transaction,
UniqueConstraintError,
} from 'sequelize';
import { WORKER_REGISTRY_TABLE } from '../../../migrations/0008-worker-registry';
import {
WORKER_STATUSES,
WorkerFenceRejectedError,
WorkerSessionConflictError,
assertWorkerConcurrency,
assertWorkerId,
assertWorkerSessionId,
hashWorkerCapabilities,
parseWorkerCapabilities,
type WorkerRecord,
type WorkerStatus,
} from '../../domain/worker';
import {
MAX_AVAILABLE_WORKER_PAGE_SIZE,
type AvailableWorkerPage,
type HeartbeatWorkerSessionCommand,
type RegisterWorkerSessionCommand,
type RegisterWorkerSessionResult,
type TransitionWorkerSessionCommand,
type WorkerRegistryRepository,
} from '../../ports/workerRegistryRepository';
interface WorkerRow {
id: string;
sessionId: string;
generation: number;
status: string;
version: number;
capabilitiesJson: string;
capabilitiesHash: string;
maxConcurrentRuns: number;
availableSlots: number;
registeredAtMs: number;
lastHeartbeatAtMs: number;
leaseExpiresAtMs: number;
updatedAtMs: number;
}
interface WorkerInstance extends Model<WorkerRow, WorkerRow>, WorkerRow {}
function defineWorkerModel(database: Sequelize): ModelStatic<WorkerInstance> {
return database.define<WorkerInstance>(
'Ql3WorkerRegistry',
{
id: { type: DataTypes.STRING(128), primaryKey: true },
sessionId: {
field: 'session_id',
type: DataTypes.STRING(36),
allowNull: false,
},
generation: { type: DataTypes.INTEGER, allowNull: false },
status: { type: DataTypes.STRING(16), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false },
capabilitiesJson: {
field: 'capabilities_json',
type: DataTypes.TEXT,
allowNull: false,
},
capabilitiesHash: {
field: 'capabilities_hash',
type: DataTypes.STRING(64),
allowNull: false,
},
maxConcurrentRuns: {
field: 'max_concurrent_runs',
type: DataTypes.INTEGER,
allowNull: false,
},
availableSlots: {
field: 'available_slots',
type: DataTypes.INTEGER,
allowNull: false,
},
registeredAtMs: {
field: 'registered_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
lastHeartbeatAtMs: {
field: 'last_heartbeat_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
leaseExpiresAtMs: {
field: 'lease_expires_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
updatedAtMs: {
field: 'updated_at_ms',
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: WORKER_REGISTRY_TABLE,
timestamps: false,
freezeTableName: true,
},
);
}
function nonNegativeTimestamp(value: number, name: string): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new RangeError(`${name} must be a non-negative safe integer`);
}
}
function positiveInteger(value: number, name: string): void {
if (!Number.isSafeInteger(value) || value < 1) {
throw new RangeError(`${name} must be a positive safe integer`);
}
}
function assertCapabilities(
capabilitiesJson: string,
capabilitiesHash: string,
): void {
parseWorkerCapabilities(capabilitiesJson);
if (
!/^[0-9a-f]{64}$/.test(capabilitiesHash) ||
hashWorkerCapabilities(capabilitiesJson) !== capabilitiesHash
) {
throw new TypeError('capabilitiesHash does not match capabilitiesJson');
}
}
function toRecord(row: WorkerRow): WorkerRecord {
if (!WORKER_STATUSES.includes(row.status as WorkerStatus)) {
throw new Error(`Worker ${row.id} has an invalid status`);
}
assertWorkerId(row.id);
assertWorkerSessionId(row.sessionId);
positiveInteger(Number(row.generation), 'generation');
nonNegativeTimestamp(Number(row.version), 'version');
assertCapabilities(row.capabilitiesJson, row.capabilitiesHash);
assertWorkerConcurrency(
Number(row.maxConcurrentRuns),
Number(row.availableSlots),
);
for (const [name, value] of [
['registeredAtMs', row.registeredAtMs],
['lastHeartbeatAtMs', row.lastHeartbeatAtMs],
['leaseExpiresAtMs', row.leaseExpiresAtMs],
['updatedAtMs', row.updatedAtMs],
] as const) {
nonNegativeTimestamp(Number(value), name);
}
if (
Number(row.lastHeartbeatAtMs) < Number(row.registeredAtMs) ||
Number(row.leaseExpiresAtMs) <= Number(row.lastHeartbeatAtMs) ||
Number(row.updatedAtMs) < Number(row.lastHeartbeatAtMs)
) {
throw new Error(`Worker ${row.id} timestamps are corrupt`);
}
return {
id: row.id,
sessionId: row.sessionId,
generation: Number(row.generation),
status: row.status as WorkerStatus,
version: Number(row.version),
capabilities: parseWorkerCapabilities(row.capabilitiesJson),
capabilitiesHash: row.capabilitiesHash,
maxConcurrentRuns: Number(row.maxConcurrentRuns),
availableSlots: Number(row.availableSlots),
registeredAtMs: Number(row.registeredAtMs),
lastHeartbeatAtMs: Number(row.lastHeartbeatAtMs),
leaseExpiresAtMs: Number(row.leaseExpiresAtMs),
updatedAtMs: Number(row.updatedAtMs),
};
}
function assertRegister(command: RegisterWorkerSessionCommand): void {
assertWorkerId(command.workerId);
assertWorkerSessionId(command.sessionId);
assertCapabilities(command.capabilitiesJson, command.capabilitiesHash);
assertWorkerConcurrency(command.maxConcurrentRuns, command.availableSlots);
nonNegativeTimestamp(command.registeredAtMs, 'registeredAtMs');
nonNegativeTimestamp(command.leaseExpiresAtMs, 'leaseExpiresAtMs');
if (command.leaseExpiresAtMs <= command.registeredAtMs) {
throw new RangeError('leaseExpiresAtMs must be after registeredAtMs');
}
}
function assertHeartbeat(command: HeartbeatWorkerSessionCommand): void {
assertWorkerId(command.workerId);
assertWorkerSessionId(command.sessionId);
positiveInteger(command.generation, 'generation');
nonNegativeTimestamp(command.expectedVersion, 'expectedVersion');
nonNegativeTimestamp(command.availableSlots, 'availableSlots');
nonNegativeTimestamp(command.heartbeatAtMs, 'heartbeatAtMs');
nonNegativeTimestamp(command.leaseExpiresAtMs, 'leaseExpiresAtMs');
if (command.leaseExpiresAtMs <= command.heartbeatAtMs) {
throw new RangeError('leaseExpiresAtMs must be after heartbeatAtMs');
}
}
function assertTransition(command: TransitionWorkerSessionCommand): void {
assertWorkerId(command.workerId);
assertWorkerSessionId(command.sessionId);
positiveInteger(command.generation, 'generation');
nonNegativeTimestamp(command.expectedVersion, 'expectedVersion');
nonNegativeTimestamp(command.transitionedAtMs, 'transitionedAtMs');
if (command.status !== 'draining' && command.status !== 'offline') {
throw new TypeError('Worker transition status is invalid');
}
}
function fenceReason(
row: WorkerRow | null,
command: {
workerId: string;
sessionId: string;
generation: number;
expectedVersion: number;
},
): WorkerFenceRejectedError['reason'] | undefined {
if (!row) return 'missing';
if (row.sessionId !== command.sessionId) return 'session_mismatch';
if (Number(row.generation) !== command.generation) {
return 'generation_mismatch';
}
if (Number(row.version) !== command.expectedVersion) {
return 'version_mismatch';
}
if (row.status === 'offline') return 'offline';
return undefined;
}
function errorCode(error: unknown): string | undefined {
if (!error || typeof error !== 'object') return undefined;
for (const candidate of [
error,
'original' in error ? error.original : undefined,
'parent' in error ? error.parent : undefined,
]) {
if (
candidate &&
typeof candidate === 'object' &&
'code' in candidate &&
typeof candidate.code === 'string'
) {
return candidate.code;
}
}
return undefined;
}
function retryDelay(attempt: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
}
export class LegacySequelizeWorkerRegistryRepository
implements WorkerRegistryRepository
{
private readonly worker: ModelStatic<WorkerInstance>;
constructor(private readonly database: Sequelize) {
this.worker = defineWorkerModel(database);
}
async findById(workerId: string): Promise<WorkerRecord | null> {
assertWorkerId(workerId);
const row = (await this.worker.findByPk(workerId, {
raw: true,
})) as unknown as WorkerRow | null;
return row ? toRecord(row) : null;
}
async register(
command: RegisterWorkerSessionCommand,
): Promise<RegisterWorkerSessionResult> {
assertRegister(command);
for (let attempt = 0; attempt < 5; attempt += 1) {
try {
return await this.database.transaction(
this.database.getDialect() === 'sqlite'
? { type: Transaction.TYPES.IMMEDIATE }
: {},
async (transaction) => {
const current = await this.worker.findByPk(command.workerId, {
transaction,
lock: transaction.LOCK.UPDATE,
});
if (!current) {
const created = await this.worker.create(
{
id: command.workerId,
sessionId: command.sessionId,
generation: 1,
status: 'online',
version: 0,
capabilitiesJson: command.capabilitiesJson,
capabilitiesHash: command.capabilitiesHash,
maxConcurrentRuns: command.maxConcurrentRuns,
availableSlots: command.availableSlots,
registeredAtMs: command.registeredAtMs,
lastHeartbeatAtMs: command.registeredAtMs,
leaseExpiresAtMs: command.leaseExpiresAtMs,
updatedAtMs: command.registeredAtMs,
},
{ transaction },
);
return {
worker: toRecord(created.get()),
replacedSession: false,
};
}
const row = current.get();
if (row.sessionId === command.sessionId) {
if (
row.capabilitiesHash !== command.capabilitiesHash ||
Number(row.maxConcurrentRuns) !== command.maxConcurrentRuns ||
Number(row.availableSlots) !== command.availableSlots
) {
throw new WorkerSessionConflictError(command.workerId);
}
if (Number(row.leaseExpiresAtMs) <= command.registeredAtMs) {
throw new WorkerFenceRejectedError(
command.workerId,
'lease_expired',
);
}
return { worker: toRecord(row), replacedSession: false };
}
const next: Partial<WorkerRow> = {
sessionId: command.sessionId,
generation: Number(row.generation) + 1,
status: 'online',
version: Number(row.version) + 1,
capabilitiesJson: command.capabilitiesJson,
capabilitiesHash: command.capabilitiesHash,
maxConcurrentRuns: command.maxConcurrentRuns,
availableSlots: command.availableSlots,
registeredAtMs: command.registeredAtMs,
lastHeartbeatAtMs: command.registeredAtMs,
leaseExpiresAtMs: command.leaseExpiresAtMs,
updatedAtMs: command.registeredAtMs,
};
await current.update(next, { transaction });
return {
worker: toRecord(current.get()),
replacedSession: true,
};
},
);
} catch (error) {
if (
(error instanceof UniqueConstraintError ||
errorCode(error) === 'SQLITE_BUSY') &&
attempt < 4
) {
await retryDelay(attempt);
continue;
}
throw error;
}
}
throw new Error('Worker registration retry budget exhausted');
}
async heartbeat(
command: HeartbeatWorkerSessionCommand,
): Promise<WorkerRecord> {
assertHeartbeat(command);
return this.database.transaction(async (transaction) => {
const current = await this.worker.findByPk(command.workerId, {
transaction,
lock: transaction.LOCK.UPDATE,
});
const row = current?.get() ?? null;
const reason = fenceReason(row, command);
if (reason) throw new WorkerFenceRejectedError(command.workerId, reason);
if (!current || !row) {
throw new WorkerFenceRejectedError(command.workerId, 'missing');
}
if (Number(row.leaseExpiresAtMs) <= command.heartbeatAtMs) {
throw new WorkerFenceRejectedError(command.workerId, 'lease_expired');
}
if (command.heartbeatAtMs < Number(row.lastHeartbeatAtMs)) {
throw new RangeError('heartbeatAtMs must not move backwards');
}
assertWorkerConcurrency(
Number(row.maxConcurrentRuns),
command.availableSlots,
);
await current.update(
{
version: Number(row.version) + 1,
availableSlots:
row.status === 'draining' ? 0 : command.availableSlots,
lastHeartbeatAtMs: command.heartbeatAtMs,
leaseExpiresAtMs: command.leaseExpiresAtMs,
updatedAtMs: command.heartbeatAtMs,
},
{ transaction },
);
return toRecord(current.get());
});
}
async transition(
command: TransitionWorkerSessionCommand,
): Promise<WorkerRecord> {
assertTransition(command);
return this.database.transaction(async (transaction) => {
const current = await this.worker.findByPk(command.workerId, {
transaction,
lock: transaction.LOCK.UPDATE,
});
const row = current?.get() ?? null;
if (
row &&
row.sessionId === command.sessionId &&
Number(row.generation) === command.generation &&
row.status === command.status &&
Number(row.version) === command.expectedVersion + 1 &&
Number(row.updatedAtMs) === command.transitionedAtMs
) {
return toRecord(row);
}
const reason = fenceReason(row, command);
if (reason) throw new WorkerFenceRejectedError(command.workerId, reason);
if (!current || !row) {
throw new WorkerFenceRejectedError(command.workerId, 'missing');
}
if (
command.status === 'draining' &&
Number(row.leaseExpiresAtMs) <= command.transitionedAtMs
) {
throw new WorkerFenceRejectedError(command.workerId, 'lease_expired');
}
if (command.transitionedAtMs < Number(row.lastHeartbeatAtMs)) {
throw new RangeError('transitionedAtMs must not move backwards');
}
await current.update(
{
status: command.status,
version: Number(row.version) + 1,
availableSlots: 0,
updatedAtMs: command.transitionedAtMs,
},
{ transaction },
);
return toRecord(current.get());
});
}
async listAvailable({
observedAtMs,
afterWorkerId,
limit = 32,
}: {
observedAtMs: number;
afterWorkerId?: string;
limit?: number;
}): Promise<AvailableWorkerPage> {
nonNegativeTimestamp(observedAtMs, 'observedAtMs');
if (afterWorkerId !== undefined) assertWorkerId(afterWorkerId);
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_AVAILABLE_WORKER_PAGE_SIZE
) {
throw new RangeError(
'limit must be between 1 and MAX_AVAILABLE_WORKER_PAGE_SIZE',
);
}
const rows = (await this.worker.findAll({
where: {
status: 'online',
availableSlots: { [Op.gt]: 0 },
leaseExpiresAtMs: { [Op.gt]: observedAtMs },
...(afterWorkerId === undefined
? {}
: { id: { [Op.gt]: afterWorkerId } }),
},
order: [['id', 'ASC']],
limit: limit + 1,
raw: true,
})) as unknown as WorkerRow[];
const truncated = rows.length > limit;
const bounded = rows.slice(0, limit).map(toRecord);
return {
workers: bounded,
truncated,
...(bounded.length === 0
? {}
: { nextCursor: bounded[bounded.length - 1].id }),
};
}
}
@@ -0,0 +1,98 @@
import path from 'path';
import config from '../../../config';
import Logger from '../../../loaders/logger';
import {
activateManualPrimaryRuntime,
type ManualPrimaryActivationAudit,
type ManualPrimaryActivationStack,
} from '../../application/manualPrimaryRuntimeActivation';
import { installManualPrimaryExecutionRouter } from '../../compatibility/manualPrimaryExecutionBridge';
import type { RuntimeRolloutPolicy } from '../../domain/runtimeRollout';
import { parseDeploymentProfile } from '../../domain/deploymentProfile';
import type { RuntimeRolloutLoadResult } from '../../ports/runtimeRolloutLoader';
import { loadRuntimeRolloutManifest } from '../fs/runtimeRolloutManifestLoader';
import type { DefaultManualPrimaryActivationOptions } from './defaultManualPrimaryActivation';
export const DEFAULT_RUNTIME_ROLLOUT_MANIFEST_FILE = 'qinglong3-rollout.json';
interface DefaultManualPrimaryStackModule {
createDefaultManualPrimaryActivationStack(
rollout: RuntimeRolloutPolicy,
options?: DefaultManualPrimaryActivationOptions,
): ManualPrimaryActivationStack;
}
export interface BootstrapDefaultManualPrimaryRuntimeOptions
extends DefaultManualPrimaryActivationOptions {
load?: () => Promise<RuntimeRolloutLoadResult>;
loadStack?: () => Promise<DefaultManualPrimaryStackModule>;
install?: typeof installManualPrimaryExecutionRouter;
audit?: (record: ManualPrimaryActivationAudit) => void | Promise<void>;
}
/**
* Lightweight HTTP-worker bootstrap. Heavy Runtime adapters are imported only
* after an accepted manifest explicitly selects manual Primary ownership.
*/
export async function bootstrapDefaultManualPrimaryRuntime(
options: BootstrapDefaultManualPrimaryRuntimeOptions = {},
) {
const sourcePath = path.join(
config.configPath,
DEFAULT_RUNTIME_ROLLOUT_MANIFEST_FILE,
);
const load = await (
options.load ?? (() => loadRuntimeRolloutManifest(sourcePath))
)();
const selected =
load.status === 'accepted' && load.policy.modeFor('manual') === 'primary';
const audit =
options.audit ??
((record: ManualPrimaryActivationAudit) => {
Logger.info(`[runtime-activation] ${JSON.stringify(record)}`);
});
let stackModule: DefaultManualPrimaryStackModule | undefined;
if (selected) {
try {
stackModule = await (
options.loadStack ?? (() => import('./defaultManualPrimaryActivation'))
)();
} catch (error) {
try {
await audit({ ...load.audit, activation: 'failed' });
} catch {
// Preserve the module load error without exposing manifest contents.
}
throw error;
}
}
const activationOptions: DefaultManualPrimaryActivationOptions = {
...(options.database === undefined ? {} : { database: options.database }),
...(options.owner === undefined ? {} : { owner: options.owner }),
...(options.recovery === undefined ? {} : { recovery: options.recovery }),
...(options.completion === undefined
? {}
: { completion: options.completion }),
...(options.cancellation === undefined
? {}
: { cancellation: options.cancellation }),
...(options.timeout === undefined ? {} : { timeout: options.timeout }),
};
return activateManualPrimaryRuntime({
load: async () => load,
create(rollout) {
if (!stackModule) {
throw new Error('Primary stack was not loaded for the selected policy');
}
return stackModule.createDefaultManualPrimaryActivationStack(rollout, {
...activationOptions,
deploymentProfile:
options.deploymentProfile ??
parseDeploymentProfile(process.env.QL_DEPLOYMENT_PROFILE),
});
},
install: options.install ?? installManualPrimaryExecutionRouter,
audit,
});
}
@@ -0,0 +1,271 @@
import type { Sequelize } from 'sequelize';
import { sequelize } from '../../../data';
import Logger from '../../../loaders/logger';
import {
PrimaryCompletionReceiptLifecycle,
type PrimaryCompletionReceiptLifecycleOptions,
} from '../../application/primaryCompletionReceiptLifecycle';
import { PrimaryCompletionReceiptJournalScanner } from '../../application/primaryCompletionReceiptJournalScanner';
import { PrimaryCompletionReceiptSupervisor } from '../../application/primaryCompletionReceiptSupervisor';
import { PrimaryCompletionReceiptConsumer } from '../../application/primaryCompletionReceiptConsumer';
import { PrimaryRunCompletionService } from '../../application/primaryRunCompletionService';
import {
PrimaryCancellationLifecycle,
type PrimaryCancellationLifecycleOptions,
} from '../../application/primaryCancellationLifecycle';
import { PrimaryCancellationDispatcher } from '../../application/primaryCancellationDispatcher';
import { PrimaryCancellationSupervisor } from '../../application/primaryCancellationSupervisor';
import { PrimaryTimeoutRequester } from '../../application/primaryTimeoutRequester';
import { PrimaryTimeoutSupervisor } from '../../application/primaryTimeoutSupervisor';
import {
PrimaryTimeoutLifecycle,
type PrimaryTimeoutLifecycleOptions,
} from '../../application/primaryTimeoutLifecycle';
import { RunCommandService } from '../../application/runCommandService';
import { PrimaryRunStartupReconciler } from '../../application/primaryRunStartupReconciler';
import {
PrimaryRunStartupSupervisor,
type PrimaryRunStartupOptions,
} from '../../application/primaryRunStartupSupervisor';
import type { RuntimeRolloutPolicy } from '../../domain/runtimeRollout';
import {
localPrimaryResourcePolicy,
type DeploymentProfile,
} from '../../domain/deploymentProfile';
import { LegacySequelizeCancellationDispatchRepository } from '../legacy-sequelize/cancellationDispatchRepository';
import { LegacySequelizePrimaryCancellationSource } from '../legacy-sequelize/primaryCancellationSource';
import { LegacySequelizePrimaryTimeoutSource } from '../legacy-sequelize/primaryTimeoutSource';
import { PrimaryCronProjection } from '../legacy-sequelize/primaryCronProjection';
import { LegacySequelizePrimaryRunRecoverySource } from '../legacy-sequelize/primaryRunRecoverySource';
import { LegacySequelizeCompletionReceiptJournal } from '../legacy-sequelize/completionReceiptJournal';
import { LegacySequelizeProjectedRunRepository } from '../legacy-sequelize/projectedRunRepository';
import { LocalProcessPersistedExecutionInspector } from '../local-process/localProcessIdentity';
import { LocalProcessPersistedExecutionController } from '../local-process/persistedLocalProcessController';
import { LocalProcessExecutor } from '../local-process/localProcessExecutor';
import { CompletionReceiptFileStore } from '../fs/completionReceiptFileStore';
import {
DEFAULT_COMPLETION_RECEIPT_ROOT,
DEFAULT_LOCAL_PROCESS_LAUNCHER_PATH,
LegacyManualPrimaryLogFiles,
} from './defaultManualPrimaryRuntime';
import { ManualPrimaryRuntime } from '../../application/manualPrimaryRuntime';
export interface DefaultManualPrimaryActivationOptions {
database?: Sequelize;
owner?: string;
deploymentProfile?: DeploymentProfile;
recovery?: PrimaryRunStartupOptions;
completion?: Pick<
PrimaryCompletionReceiptLifecycleOptions,
'intervalMs' | 'initialDelayMs' | 'stopTimeoutMs' | 'cycle'
>;
cancellation?: Pick<
PrimaryCancellationLifecycleOptions,
'intervalMs' | 'initialDelayMs' | 'stopTimeoutMs' | 'cycle'
>;
timeout?: Pick<
PrimaryTimeoutLifecycleOptions,
'intervalMs' | 'initialDelayMs' | 'stopTimeoutMs' | 'cycle'
>;
}
function boundedOwner(value: string): string {
if (!value || value.length > 128) {
throw new RangeError(
'Primary activation owner must be 1 to 128 characters',
);
}
return value;
}
export function createDefaultManualPrimaryActivationStack(
rollout: RuntimeRolloutPolicy,
options: DefaultManualPrimaryActivationOptions = {},
) {
const database = options.database ?? sequelize;
const resources = localPrimaryResourcePolicy(
options.deploymentProfile ?? 'standalone',
);
const repository = new LegacySequelizeProjectedRunRepository(database, [
new PrimaryCronProjection(database),
]);
const recoverySource = new LegacySequelizePrimaryRunRecoverySource(database);
const completionReceiptJournal = new LegacySequelizeCompletionReceiptJournal(
database,
);
const completionReceiptStore = new CompletionReceiptFileStore(
DEFAULT_COMPLETION_RECEIPT_ROOT,
);
const completionReceipts = new PrimaryCompletionReceiptConsumer(
completionReceiptStore,
new PrimaryRunCompletionService(repository),
{
journal: completionReceiptJournal,
quarantineRetentionMs: resources.receiptQuarantineRetentionMs,
},
);
const startup = new PrimaryRunStartupSupervisor(
new PrimaryRunStartupReconciler(
repository,
recoverySource,
[new LocalProcessPersistedExecutionInspector()],
{
completionReceipts,
completionReceiptJournal,
receiptPublishGraceMs: resources.receiptPublishGraceMs,
},
),
);
const completion = new PrimaryCompletionReceiptLifecycle(
new PrimaryCompletionReceiptSupervisor(
new PrimaryCompletionReceiptJournalScanner(
completionReceiptJournal,
completionReceiptStore,
completionReceipts,
{
terminalMissingRetentionMs:
resources.receiptTerminalMissingRetentionMs,
},
),
),
{
intervalMs:
options.completion?.intervalMs ?? resources.completion.intervalMs,
initialDelayMs:
options.completion?.initialDelayMs ??
resources.completion.initialDelayMs,
stopTimeoutMs:
options.completion?.stopTimeoutMs ?? resources.completion.stopTimeoutMs,
cycle: options.completion?.cycle ?? {
pageSize: resources.completion.pageSize,
maxPages: resources.completion.maxPages,
},
onCycle(summary) {
Logger.info(
`[runtime-completion] ${JSON.stringify({
profile: resources.profile,
pages: summary.pages,
scanned: summary.scanned,
applied: summary.applied,
alreadyTerminal: summary.alreadyTerminal,
quarantined: summary.quarantined,
purgedQuarantines: summary.purgedQuarantines,
expiredMissing: summary.expiredMissing,
missing: summary.missing,
cleanupPending: summary.cleanupPending,
skipped: summary.skipped,
ambiguous: summary.ambiguous,
failed: summary.failed,
stopReason: summary.stopReason,
remaining: summary.remaining,
})}`,
);
},
onError() {
Logger.error('[runtime-completion] cycle failed');
},
},
);
const cancellation = new PrimaryCancellationLifecycle(
new PrimaryCancellationSupervisor(
new PrimaryCancellationDispatcher(
new LegacySequelizePrimaryCancellationSource(database),
new LegacySequelizeCancellationDispatchRepository(database),
[new LocalProcessPersistedExecutionController()],
{ owner: boundedOwner(options.owner ?? `http:${process.pid}`) },
),
),
{
intervalMs:
options.cancellation?.intervalMs ?? resources.cancellation.intervalMs,
initialDelayMs:
options.cancellation?.initialDelayMs ??
resources.cancellation.initialDelayMs,
stopTimeoutMs:
options.cancellation?.stopTimeoutMs ??
resources.cancellation.stopTimeoutMs,
cycle: options.cancellation?.cycle ?? {
pageSize: resources.cancellation.pageSize,
maxPages: resources.cancellation.maxPages,
},
onCycle(summary) {
Logger.info(
`[runtime-cancellation] ${JSON.stringify({
pages: summary.pages,
scanned: summary.scanned,
claimed: summary.claimed,
pending: summary.pending,
failed: summary.failed,
stopReason: summary.stopReason,
remaining: summary.remaining,
})}`,
);
},
onError() {
Logger.error('[runtime-cancellation] cycle failed');
},
},
);
const timeout = new PrimaryTimeoutLifecycle(
new PrimaryTimeoutSupervisor(
new PrimaryTimeoutRequester(
new LegacySequelizePrimaryTimeoutSource(database),
new RunCommandService(repository),
),
),
{
intervalMs: options.timeout?.intervalMs ?? resources.timeout.intervalMs,
initialDelayMs:
options.timeout?.initialDelayMs ?? resources.timeout.initialDelayMs,
stopTimeoutMs:
options.timeout?.stopTimeoutMs ?? resources.timeout.stopTimeoutMs,
cycle: options.timeout?.cycle ?? {
pageSize: resources.timeout.pageSize,
maxPages: resources.timeout.maxPages,
},
onCycle(summary) {
Logger.info(
`[runtime-timeout] ${JSON.stringify({
profile: resources.profile,
pages: summary.pages,
scanned: summary.scanned,
accepted: summary.accepted,
alreadyRequested: summary.alreadyRequested,
alreadyTerminal: summary.alreadyTerminal,
failed: summary.failed,
stopReason: summary.stopReason,
remaining: summary.remaining,
})}`,
);
},
onError() {
Logger.error('[runtime-timeout] cycle failed');
},
},
);
return {
router: new ManualPrimaryRuntime(
repository,
new LocalProcessExecutor({
durableLauncherPath: DEFAULT_LOCAL_PROCESS_LAUNCHER_PATH,
}),
rollout,
new LegacyManualPrimaryLogFiles(
undefined,
undefined,
completionReceiptJournal,
),
{
orchestrator: { completionReceiptJournal },
},
),
reconcile: () => startup.run(options.recovery),
startCompletion: () => completion.start(),
stopCompletion: () => completion.stop(),
startTimeout: () => timeout.start(),
stopTimeout: () => timeout.stop(),
startCancellation: () => cancellation.start(),
stopCancellation: () => cancellation.stop(),
};
}
@@ -0,0 +1,132 @@
import fs from 'fs/promises';
import path from 'path';
import dayjs from 'dayjs';
import config from '../../../config';
import { getUniqPath } from '../../../config/util';
import { sequelize } from '../../../data';
import { logStreamManager } from '../../../shared/logStreamManager';
import {
ManualPrimaryRuntime,
type ManualPrimaryLogFiles,
type PreparedManualPrimaryLog,
} from '../../application/manualPrimaryRuntime';
import type { ManualPrimaryStartInput } from '../../compatibility/manualPrimaryExecutionBridge';
import { createLegacyLogOutputRef } from '../../compatibility/legacyLogOutputRef';
import type { RuntimeRolloutPolicy } from '../../domain/runtimeRollout';
import { PrimaryCronProjection } from '../legacy-sequelize/primaryCronProjection';
import { LegacySequelizeProjectedRunRepository } from '../legacy-sequelize/projectedRunRepository';
import { LocalProcessExecutor } from '../local-process/localProcessExecutor';
import { enableDurableLocalProcessOutput } from '../local-process/durableLocalProcessOutput';
import { CompletionReceiptFileStore } from '../fs/completionReceiptFileStore';
import type { CompletionReceiptJournal } from '../../ports/completionReceiptJournal';
export const DEFAULT_COMPLETION_RECEIPT_ROOT = path.join(
config.dataPath,
'runtime',
'completion-receipts',
);
export const DEFAULT_LOCAL_PROCESS_LAUNCHER_PATH = path.join(
config.rootPath,
'shell',
'ql3-launcher.sh',
);
function isWithin(root: string, candidate: string): boolean {
return candidate === root || candidate.startsWith(root + path.sep);
}
function relativeLogDirectory(root: string, value: string): string {
const candidate = path.isAbsolute(value) ? value : path.resolve(root, value);
if (!isWithin(root, candidate)) {
throw new Error('Manual Primary log directory escapes the configured root');
}
const relative = path.relative(root, candidate).split(path.sep).join('/');
if (!relative || relative === '.') {
throw new Error('Manual Primary log directory must be below the log root');
}
return relative;
}
export class LegacyManualPrimaryLogFiles implements ManualPrimaryLogFiles {
private readonly completionReceipts: CompletionReceiptFileStore;
private readonly completionReceiptRoot: string;
constructor(
private readonly logRoot = path.resolve(config.logPath),
completionReceiptRoot = DEFAULT_COMPLETION_RECEIPT_ROOT,
private readonly completionReceiptJournal?: Pick<
CompletionReceiptJournal,
'resolve'
>,
) {
this.completionReceiptRoot = path.resolve(completionReceiptRoot);
this.completionReceipts = new CompletionReceiptFileStore(
this.completionReceiptRoot,
);
}
async prepare(
input: ManualPrimaryStartInput,
): Promise<PreparedManualPrimaryLog> {
const configured =
!input.cron.logName || input.cron.logName === '/dev/null'
? await getUniqPath(input.cron.command, String(input.cron.id))
: input.cron.logName;
const directory = relativeLogDirectory(this.logRoot, configured);
const logPath = path.posix.join(
directory,
dayjs(input.acceptedAtMs).format('YYYY-MM-DD-HH-mm-ss-SSS') + '.log',
);
createLegacyLogOutputRef(logPath);
const absolutePath = path.resolve(this.logRoot, ...logPath.split('/'));
if (!isWithin(this.logRoot, absolutePath)) {
throw new Error('Manual Primary log file escapes the configured root');
}
await fs.mkdir(path.dirname(absolutePath), { recursive: true });
const output = enableDurableLocalProcessOutput(
{
async write(output) {
await logStreamManager.write(
absolutePath,
Buffer.from(output.chunk).toString('utf8'),
);
},
},
{
outputFilePath: absolutePath,
completionReceiptRoot: this.completionReceiptRoot,
},
);
const completionReceipts = this.completionReceipts;
const completionReceiptJournal = this.completionReceiptJournal;
return {
logPath,
output,
async completionCommitted(attemptId) {
await completionReceipts.remove(attemptId);
await completionReceiptJournal?.resolve(attemptId);
},
async close() {
await logStreamManager.closeStream(absolutePath);
},
};
}
}
/** Legacy factory retained for focused tests; production activation uses the
* shared lifecycle stack in defaultManualPrimaryActivation.ts. */
export function createDefaultManualPrimaryRuntime(
rollout: RuntimeRolloutPolicy,
): ManualPrimaryRuntime {
const repository = new LegacySequelizeProjectedRunRepository(sequelize, [
new PrimaryCronProjection(sequelize),
]);
return new ManualPrimaryRuntime(
repository,
new LocalProcessExecutor({
durableLauncherPath: DEFAULT_LOCAL_PROCESS_LAUNCHER_PATH,
}),
rollout,
new LegacyManualPrimaryLogFiles(),
);
}
@@ -0,0 +1,110 @@
import type {
ExecutionResourcePolicy,
ExecutionSpec,
} from '../../domain/execution';
import { InvalidExecutionSpecError } from '../../domain/executorErrors';
export interface LegacyCronSnapshot {
id: number;
command: string;
taskBefore?: string;
taskAfter?: string;
workDirectory?: string;
logName?: string;
}
export interface LegacyCronExecutionInput {
runId: string;
attemptId: string;
projectId: string;
taskRevision: string;
cron: LegacyCronSnapshot;
realTime: boolean;
realLogPath?: string;
noDelay?: boolean;
timeoutMs?: number;
terminationGraceMs?: number;
resourcePolicy?: ExecutionResourcePolicy;
}
export const DEFAULT_LEGACY_TERMINATION_GRACE_MS = 10_000;
function quoteShellValue(value: string): string {
return `'${value.replace(/'/g, `'\\''`)}'`;
}
function normalizeHook(value: string): string {
return value.replace(/;? *\r?\n/g, ';').trim();
}
function assignment(name: string, value: string | number | boolean): string {
return `${name}=${quoteShellValue(String(value))}`;
}
function legacyTaskCommand(command: string): string {
const trimmed = command.trim();
if (!trimmed) {
throw new InvalidExecutionSpecError(
'Legacy Cron command must not be empty',
);
}
if (trimmed.startsWith('task ') || trimmed.startsWith('ql ')) return trimmed;
return `task ${trimmed}`;
}
export function buildLegacyCronExecutionSpec(
input: LegacyCronExecutionInput,
): ExecutionSpec {
if (!Number.isSafeInteger(input.cron.id) || input.cron.id < 1) {
throw new InvalidExecutionSpecError(
'Legacy Cron id must be a positive safe integer',
);
}
const variables: string[] = [];
if (input.realLogPath) {
variables.push(assignment('real_log_path', input.realLogPath));
}
if (input.noDelay) variables.push(assignment('no_delay', true));
variables.push(assignment('real_time', input.realTime));
variables.push(assignment('no_tee', true));
variables.push(assignment('ID', input.cron.id));
if (input.cron.logName) {
variables.push(assignment('log_name', input.cron.logName));
}
if (input.cron.taskBefore) {
variables.push(
assignment('task_before', normalizeHook(input.cron.taskBefore)),
);
}
if (input.cron.taskAfter) {
variables.push(
assignment('task_after', normalizeHook(input.cron.taskAfter)),
);
}
if (input.cron.workDirectory) {
variables.push(assignment('work_dir', input.cron.workDirectory));
}
return {
runId: input.runId,
attemptId: input.attemptId,
projectId: input.projectId,
taskId: `legacy-cron:${input.cron.id}`,
taskRevision: input.taskRevision,
command: {
kind: 'shell',
command: `${variables.join(' ')} ${legacyTaskCommand(
input.cron.command,
)}`,
shell: '/bin/bash',
},
environmentPolicy: 'inherit',
...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }),
terminationGraceMs:
input.terminationGraceMs ?? DEFAULT_LEGACY_TERMINATION_GRACE_MS,
...(input.resourcePolicy === undefined
? {}
: { resourcePolicy: input.resourcePolicy }),
};
}
@@ -0,0 +1,226 @@
import { randomBytes } from 'crypto';
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import type { ExecutionContext, ExecutionSpec } from '../../domain/execution';
import { assertCompletionReceiptId } from '../../domain/completionReceipt';
import {
ExecutorCapabilityUnavailableError,
InvalidExecutionSpecError,
} from '../../domain/executorErrors';
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
import type { DurableLocalProcessOutput } from './durableLocalProcessOutput';
const CALLBACK_TOKEN_PATTERN = /^[A-Za-z0-9_-]{32,128}$/;
export interface DurableLocalProcessLaunch {
file: string;
args: readonly string[];
environment: NodeJS.ProcessEnv;
outputDescriptor: number;
closeParentOutput(): Promise<void>;
}
function assertCallback(
callback: ExecutionContext['completionCallback'],
): asserts callback is NonNullable<ExecutionContext['completionCallback']> {
if (!callback || !CALLBACK_TOKEN_PATTERN.test(callback.token)) {
throw new InvalidExecutionSpecError(
'durable completion requires a bounded base64url callback token',
);
}
if (
!Number.isSafeInteger(callback.callbackSequence) ||
callback.callbackSequence < 1
) {
throw new InvalidExecutionSpecError(
'durable completion requires a positive callback sequence',
);
}
}
function launcherEnvironment(
environment: NodeJS.ProcessEnv,
spec: ExecutionSpec,
context: ExecutionContext,
capability: DurableLocalProcessOutput,
receiptTarget: string,
receiptTemporary: string,
startedAtMs: number,
quotaFifo: string | undefined,
quotaRemainingBytes: number | undefined,
truncationTarget: string | undefined,
truncationTemporary: string | undefined,
): NodeJS.ProcessEnv {
const callback = context.completionCallback!;
return {
...environment,
QL3_RECEIPT_RUN_ID: spec.runId,
QL3_RECEIPT_ATTEMPT_ID: spec.attemptId,
QL3_RECEIPT_CALLBACK_SEQUENCE: String(callback.callbackSequence),
QL3_RECEIPT_CALLBACK_TOKEN: callback.token,
QL3_RECEIPT_STARTED_AT_MS: String(startedAtMs),
QL3_RECEIPT_TARGET: receiptTarget,
QL3_RECEIPT_TEMPORARY: receiptTemporary,
...(quotaFifo === undefined
? {}
: {
QL3_OUTPUT_QUOTA_FIFO: quotaFifo,
QL3_OUTPUT_QUOTA_REMAINING_BYTES: String(quotaRemainingBytes),
QL3_OUTPUT_ARTIFACT_ID: capability.logArtifactId!,
QL3_OUTPUT_MAXIMUM_BYTES: String(capability.maximumBytes),
QL3_OUTPUT_TRUNCATION_TARGET: truncationTarget!,
QL3_OUTPUT_TRUNCATION_TEMPORARY: truncationTemporary!,
}),
...(spec.command.kind === 'shell'
? {
QL3_LAUNCH_SHELL: spec.command.shell ?? '/bin/bash',
QL3_LAUNCH_SHELL_COMMAND: spec.command.command,
}
: {}),
};
}
export async function prepareDurableLocalProcessLaunch(
spec: ExecutionSpec,
context: ExecutionContext,
environment: NodeJS.ProcessEnv,
capability: DurableLocalProcessOutput,
launcherPath: string | undefined,
startedAtMs: number,
): Promise<DurableLocalProcessLaunch> {
if (!launcherPath) {
throw new ExecutorCapabilityUnavailableError('durableLocalCompletion');
}
if (!path.isAbsolute(launcherPath) || launcherPath.includes('\0')) {
throw new InvalidExecutionSpecError(
'durable launcher path must be absolute and contain no NUL',
);
}
assertCallback(context.completionCallback);
assertCompletionReceiptId(spec.runId, 'runId');
assertCompletionReceiptId(spec.attemptId, 'attemptId');
if (!Number.isSafeInteger(startedAtMs) || startedAtMs < 0) {
throw new InvalidExecutionSpecError(
'durable completion start time must be a non-negative safe integer',
);
}
const launcher = await fs.lstat(launcherPath);
if (!launcher.isFile()) {
throw new InvalidExecutionSpecError(
'durable launcher must be a regular file',
);
}
const receiptDirectory = path.join(
capability.completionReceiptRoot,
spec.attemptId.slice(0, 2),
);
const receiptTarget = path.join(receiptDirectory, `${spec.attemptId}.json`);
const receiptTemporary = path.join(
receiptDirectory,
`.${spec.attemptId}.${randomBytes(16).toString('hex')}.tmp`,
);
await fs.mkdir(path.dirname(capability.outputFilePath), {
recursive: true,
mode: 0o700,
});
await fs.mkdir(receiptDirectory, { recursive: true, mode: 0o700 });
const output = await fs.open(
capability.outputFilePath,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_APPEND |
(constants.O_NOFOLLOW ?? 0),
0o600,
);
let quotaFifo: string | undefined;
let quotaRemainingBytes: number | undefined;
let truncationTarget: string | undefined;
let truncationTemporary: string | undefined;
try {
const stat = await output.stat();
if (!stat.isFile()) {
throw new InvalidExecutionSpecError(
'durable output target must be a regular file',
);
}
await output.chmod(0o600);
if (capability.maximumBytes !== undefined) {
if (
!Number.isSafeInteger(capability.maximumBytes) ||
capability.maximumBytes < 1 ||
!Number.isSafeInteger(stat.size) ||
stat.size < 0 ||
stat.size > capability.maximumBytes
) {
throw new InvalidExecutionSpecError(
'durable output quota or existing size is invalid',
);
}
quotaRemainingBytes = capability.maximumBytes - stat.size;
if (!capability.logArtifactId) {
throw new InvalidExecutionSpecError(
'durable output quota requires a Local Artifact identity',
);
}
try {
assertLocalExecutionArtifactId(capability.logArtifactId);
} catch {
throw new InvalidExecutionSpecError(
'durable output Local Artifact identity is invalid',
);
}
if (
path.basename(capability.outputFilePath) !==
`${capability.logArtifactId}.log`
) {
throw new InvalidExecutionSpecError(
'durable output path does not match its Local Artifact identity',
);
}
quotaFifo = path.join(
path.dirname(capability.outputFilePath),
`.${path.basename(capability.outputFilePath)}.fifo`,
);
truncationTarget = path.join(
path.dirname(capability.outputFilePath),
`.${path.basename(capability.outputFilePath)}.truncated.json`,
);
truncationTemporary = path.join(
path.dirname(capability.outputFilePath),
`.${path.basename(capability.outputFilePath)}.truncated.tmp`,
);
}
} catch (error) {
await output.close().catch(() => undefined);
throw error;
}
const launchMode = spec.command.kind;
const args =
spec.command.kind === 'argv'
? [launcherPath, launchMode, spec.command.file, ...spec.command.args]
: [launcherPath, launchMode];
return {
file: '/bin/sh',
args,
environment: launcherEnvironment(
environment,
spec,
context,
capability,
receiptTarget,
receiptTemporary,
startedAtMs,
quotaFifo,
quotaRemainingBytes,
truncationTarget,
truncationTemporary,
),
outputDescriptor: output.fd,
closeParentOutput: () => output.close(),
};
}
@@ -0,0 +1,66 @@
import path from 'path';
import type { ExecutionOutputSink } from '../../domain/execution';
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
const DURABLE_LOCAL_PROCESS_OUTPUT = Symbol('durable-local-process-output');
export interface DurableLocalProcessOutput {
outputFilePath: string;
completionReceiptRoot: string;
maximumBytes?: number;
logArtifactId?: string;
}
type CapableExecutionOutputSink = ExecutionOutputSink & {
[DURABLE_LOCAL_PROCESS_OUTPUT]?: DurableLocalProcessOutput;
};
function assertAbsolutePath(value: string, name: string): void {
if (!path.isAbsolute(value) || value.includes('\0')) {
throw new RangeError(`${name} must be an absolute path containing no NUL`);
}
}
/**
* Adds an adapter-local launch capability without widening ExecutionContext.
* The symbol is deliberately non-enumerable so paths cannot leak through
* routine context serialization or diagnostic logging.
*/
export function enableDurableLocalProcessOutput<T extends ExecutionOutputSink>(
output: T,
capability: DurableLocalProcessOutput,
): T {
assertAbsolutePath(capability.outputFilePath, 'outputFilePath');
assertAbsolutePath(capability.completionReceiptRoot, 'completionReceiptRoot');
if (
capability.maximumBytes !== undefined &&
(!Number.isSafeInteger(capability.maximumBytes) ||
capability.maximumBytes < 1)
) {
throw new RangeError('maximumBytes must be a positive safe integer');
}
if (
(capability.maximumBytes === undefined) !==
(capability.logArtifactId === undefined)
) {
throw new RangeError(
'maximumBytes and logArtifactId must be provided together',
);
}
if (capability.logArtifactId !== undefined) {
assertLocalExecutionArtifactId(capability.logArtifactId);
}
Object.defineProperty(output, DURABLE_LOCAL_PROCESS_OUTPUT, {
configurable: false,
enumerable: false,
writable: false,
value: Object.freeze({ ...capability }),
});
return output;
}
export function durableLocalProcessOutput(
output: ExecutionOutputSink,
): DurableLocalProcessOutput | undefined {
return (output as CapableExecutionOutputSink)[DURABLE_LOCAL_PROCESS_OUTPUT];
}
@@ -0,0 +1,580 @@
import { ChildProcess, SpawnOptions } from 'child_process';
import { Readable } from 'stream';
import { v7 as uuidV7 } from 'uuid';
import { spawn } from 'cross-spawn';
import type {
ExecutionContext,
ExecutionDiagnostic,
ExecutionHandle,
ExecutionInspection,
ExecutionOutputStream,
ExecutionResult,
ExecutionSpec,
ExecutionStopReason,
ExecutionStopResult,
ExecutorCapabilities,
} from '../../domain/execution';
import {
ExecutorCapabilityUnavailableError,
ExecutorHandleNotFoundError,
ExecutorStartError,
InvalidExecutionSpecError,
} from '../../domain/executorErrors';
import { assertExecutionSpec as assertDomainExecutionSpec } from '../../domain/executionSpec';
export {
MAX_EXECUTION_ARGUMENTS,
MAX_EXECUTION_COMMAND_BYTES,
MAX_EXECUTION_TIMEOUT_MS,
MAX_TERMINATION_GRACE_MS,
} from '../../domain/executionSpec';
import type { Executor } from '../../ports/executor';
import {
createLocalProcessDurableHandle,
LinuxProcProcessIdentityProvider,
type LocalProcessIdentityProvider,
} from './localProcessIdentity';
import {
PosixProcessTerminator,
type ProcessTerminator,
} from './processTerminator';
import { durableLocalProcessOutput } from './durableLocalProcessOutput';
import {
prepareDurableLocalProcessLaunch,
type DurableLocalProcessLaunch,
} from './durableLocalProcessLaunch';
export const MAX_EXECUTION_ENVIRONMENT_ENTRIES = 1024;
export const MAX_EXECUTION_ENVIRONMENT_BYTES = 512 * 1024;
const DEFAULT_POSIX_SHELL = '/bin/bash';
const ISOLATED_ENVIRONMENT_KEYS = [
'PATH',
'LANG',
'LC_ALL',
'LC_CTYPE',
'TZ',
'TMPDIR',
] as const;
const LOCAL_PROCESS_CAPABILITIES: ExecutorCapabilities = Object.freeze({
timeout: true,
processGroupTermination: process.platform !== 'win32',
workingDirectory: true,
isolatedEnvironment: true,
memoryLimit: 'none',
cpuLimit: 'none',
filesystemIsolation: 'none',
networkIsolation: 'none',
});
export interface ExecutorClock {
now(): number;
}
export interface LocalProcessExecutorOptions {
clock?: ExecutorClock;
createHandleId?: () => string;
terminator?: ProcessTerminator;
identityProvider?: LocalProcessIdentityProvider;
durableLauncherPath?: string;
}
interface LocalExecutionLifecycle {
startedAtMs: number;
closedObserved: boolean;
finished: boolean;
result?: ExecutionResult;
terminationReason?: ExecutionStopReason;
runtimeError: boolean;
diagnostics: ExecutionDiagnostic[];
timeout?: NodeJS.Timeout;
removeAbortListener?: () => void;
}
interface LocalExecutionState {
child: ChildProcess;
processGroup: boolean;
graceMs: number;
closed: Promise<void>;
lifecycle: LocalExecutionLifecycle;
stopPromise?: Promise<ExecutionStopResult>;
}
function assertHandleIdentifier(value: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > 255 ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new InvalidExecutionSpecError(
'handleId must be between 1 and 255 characters and contain no control characters',
);
}
}
function assertResourcePolicy(spec: ExecutionSpec): void {
const policy = spec.resourcePolicy;
if (!policy) return;
if (policy.memoryBytes?.enforcement === 'required') {
throw new ExecutorCapabilityUnavailableError('memoryLimit');
}
if (policy.cpuMillisPerSecond?.enforcement === 'required') {
throw new ExecutorCapabilityUnavailableError('cpuLimit');
}
if (policy.filesystemIsolation === 'required') {
throw new ExecutorCapabilityUnavailableError('filesystemIsolation');
}
if (policy.networkIsolation === 'required') {
throw new ExecutorCapabilityUnavailableError('networkIsolation');
}
}
function resourcePolicyDiagnostics(spec: ExecutionSpec): ExecutionDiagnostic[] {
const policy = spec.resourcePolicy;
if (!policy) return [];
const unavailable = [
policy.memoryBytes?.enforcement === 'best_effort' ? 'memoryLimit' : null,
policy.cpuMillisPerSecond?.enforcement === 'best_effort'
? 'cpuLimit'
: null,
policy.filesystemIsolation === 'best_effort' ? 'filesystemIsolation' : null,
policy.networkIsolation === 'best_effort' ? 'networkIsolation' : null,
].filter((value): value is string => value !== null);
return unavailable.length === 0
? []
: [
{
code: 'RESOURCE_POLICY_BEST_EFFORT_UNAVAILABLE',
summary: `Best-effort capabilities were unavailable: ${unavailable.join(
', ',
)}`,
},
];
}
function assertExecutionSpec(spec: ExecutionSpec): void {
assertDomainExecutionSpec(spec);
assertResourcePolicy(spec);
}
function environmentBytes(environment: NodeJS.ProcessEnv): number {
return Object.entries(environment).reduce(
(total, [key, value]) =>
total +
Buffer.byteLength(key, 'utf8') +
Buffer.byteLength(value ?? '', 'utf8'),
0,
);
}
function buildEnvironment(
policy: ExecutionSpec['environmentPolicy'],
supplied: Readonly<Record<string, string>>,
): NodeJS.ProcessEnv {
if (Object.keys(supplied).length > MAX_EXECUTION_ENVIRONMENT_ENTRIES) {
throw new InvalidExecutionSpecError(
'execution environment has too many entries',
);
}
const environment: NodeJS.ProcessEnv = {};
if (policy === 'inherit') {
Object.assign(environment, process.env);
} else {
for (const key of ISOLATED_ENVIRONMENT_KEYS) {
if (process.env[key] !== undefined) environment[key] = process.env[key];
}
}
for (const [key, value] of Object.entries(supplied)) {
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || value.includes('\0')) {
throw new InvalidExecutionSpecError(
'execution environment contains an invalid key or NUL value',
);
}
environment[key] = value;
}
if (environmentBytes(environment) > MAX_EXECUTION_ENVIRONMENT_BYTES) {
throw new InvalidExecutionSpecError('execution environment is too large');
}
return environment;
}
function diagnosticOnce(
lifecycle: LocalExecutionLifecycle,
diagnostic: ExecutionDiagnostic,
): void {
if (!lifecycle.diagnostics.some((item) => item.code === diagnostic.code)) {
lifecycle.diagnostics.push(diagnostic);
}
}
function createResult(
lifecycle: LocalExecutionLifecycle,
code: number | null,
signal: NodeJS.Signals | null,
finishedAtMs: number,
): ExecutionResult {
const base = {
startedAtMs: lifecycle.startedAtMs,
finishedAtMs: Math.max(finishedAtMs, lifecycle.startedAtMs),
...(code === null ? {} : { exitCode: code }),
...(signal === null ? {} : { signal }),
...(lifecycle.diagnostics.length === 0
? {}
: { diagnostics: [...lifecycle.diagnostics] }),
};
if (lifecycle.terminationReason?.kind === 'timeout') {
return {
...base,
outcome: 'timed_out',
errorCode: 'EXECUTION_TIMED_OUT',
errorSummary: 'Execution exceeded its configured timeout',
};
}
if (lifecycle.terminationReason) {
return {
...base,
outcome: 'cancelled',
errorCode: 'EXECUTION_CANCELLED',
errorSummary: 'Execution was cancelled',
};
}
if (lifecycle.runtimeError) {
return {
...base,
outcome: 'failed',
errorCode: 'PROCESS_RUNTIME_ERROR',
errorSummary: 'The child process reported a runtime error',
};
}
if (code === 0) return { ...base, outcome: 'succeeded' };
if (code !== null) {
return {
...base,
outcome: 'failed',
errorCode: 'PROCESS_EXIT_NON_ZERO',
errorSummary: `Process exited with code ${code}`,
};
}
if (signal !== null) {
return {
...base,
outcome: 'failed',
errorCode: 'PROCESS_SIGNALLED',
errorSummary: `Process exited after signal ${signal}`,
};
}
return {
...base,
outcome: 'failed',
errorCode: 'PROCESS_EXIT_UNKNOWN',
errorSummary: 'Process exited without an exit code or signal',
};
}
export class LocalProcessExecutor implements Executor {
readonly type = 'local_process' as const;
private readonly clock: ExecutorClock;
private readonly createHandleId: () => string;
private readonly terminator: ProcessTerminator;
private readonly identityProvider: LocalProcessIdentityProvider;
private readonly durableLauncherPath?: string;
private readonly states = new WeakMap<ExecutionHandle, LocalExecutionState>();
constructor(options: LocalProcessExecutorOptions = {}) {
this.clock = options.clock ?? { now: Date.now };
this.createHandleId = options.createHandleId ?? uuidV7;
this.terminator = options.terminator ?? new PosixProcessTerminator();
this.identityProvider =
options.identityProvider ?? new LinuxProcProcessIdentityProvider();
this.durableLauncherPath = options.durableLauncherPath;
}
capabilities(): ExecutorCapabilities {
return LOCAL_PROCESS_CAPABILITIES;
}
async start(
spec: ExecutionSpec,
context: ExecutionContext,
): Promise<ExecutionHandle> {
assertExecutionSpec(spec);
const environment = buildEnvironment(
spec.environmentPolicy,
context.environment,
);
const handleId = this.createHandleId();
assertHandleIdentifier(handleId);
if (context.signal?.aborted) {
throw new ExecutorStartError(
new Error('Execution was aborted before spawn'),
);
}
if (
context.signal &&
(!context.signal.addEventListener || !context.signal.removeEventListener)
) {
throw new InvalidExecutionSpecError(
'Execution abort signal must support event listeners',
);
}
const processGroup = process.platform !== 'win32';
const durableOutput = durableLocalProcessOutput(context.output);
let durableLaunch: DurableLocalProcessLaunch | undefined;
if (durableOutput) {
durableLaunch = await prepareDurableLocalProcessLaunch(
spec,
context,
environment,
durableOutput,
this.durableLauncherPath,
this.clock.now(),
);
}
const options: SpawnOptions = {
cwd: spec.workingDirectory,
env: durableLaunch?.environment ?? environment,
detached: processGroup,
stdio: durableLaunch
? [
'ignore',
durableLaunch.outputDescriptor,
durableLaunch.outputDescriptor,
]
: ['ignore', 'pipe', 'pipe'],
};
let child: ChildProcess;
try {
child = durableLaunch
? spawn(durableLaunch.file, [...durableLaunch.args], options)
: spec.command.kind === 'argv'
? spawn(spec.command.file, [...spec.command.args], options)
: spawn(spec.command.command, {
...options,
shell: spec.command.shell ?? DEFAULT_POSIX_SHELL,
});
} catch (error) {
await durableLaunch?.closeParentOutput().catch(() => undefined);
throw new ExecutorStartError(error);
}
const lifecycle: LocalExecutionLifecycle = {
startedAtMs: 0,
closedObserved: false,
finished: false,
runtimeError: false,
diagnostics: resourcePolicyDiagnostics(spec),
};
let resolveClosed: () => void = () => undefined;
const closed = new Promise<void>((resolve) => {
resolveClosed = resolve;
});
const outputPumps = durableLaunch
? []
: [
this.pumpOutput(child.stdout, 'stdout', context, lifecycle),
this.pumpOutput(child.stderr, 'stderr', context, lifecycle),
];
let spawnConfirmed = false;
const spawned = new Promise<void>((resolve, reject) => {
child.once('spawn', () => {
spawnConfirmed = true;
lifecycle.startedAtMs = this.clock.now();
resolve();
});
child.on('error', (error) => {
if (!spawnConfirmed) reject(error);
else lifecycle.runtimeError = true;
});
});
const completion = new Promise<ExecutionResult>((resolve) => {
child.once('close', (code, signal) => {
lifecycle.closedObserved = true;
resolveClosed();
void Promise.all(outputPumps).then(() => {
if (lifecycle.timeout) clearTimeout(lifecycle.timeout);
lifecycle.removeAbortListener?.();
const result = createResult(
lifecycle,
code,
signal,
this.clock.now(),
);
lifecycle.result = result;
lifecycle.finished = true;
resolve(result);
});
});
});
try {
await spawned;
} catch (error) {
await durableLaunch?.closeParentOutput().catch(() => undefined);
throw new ExecutorStartError(error);
}
await durableLaunch?.closeParentOutput().catch(() => undefined);
if (!child.pid) {
throw new ExecutorStartError(new Error('Spawn did not return a PID'));
}
let durableHandle: string | undefined;
try {
const identity = await this.identityProvider.capture(child.pid);
if (identity) {
durableHandle = createLocalProcessDurableHandle(handleId, identity);
}
} catch {
// Recovery identity is optional; the Reconciler will conservatively mark
// an unprovable execution lost and will never signal by PID alone.
}
const handle: ExecutionHandle = {
id: handleId,
...(durableHandle === undefined ? {} : { durableHandle }),
executorType: this.type,
runId: spec.runId,
attemptId: spec.attemptId,
startedAtMs: lifecycle.startedAtMs,
pid: child.pid,
completion,
};
const state: LocalExecutionState = {
child,
processGroup,
graceMs: spec.terminationGraceMs,
closed,
lifecycle,
};
this.states.set(handle, state);
if (!lifecycle.closedObserved && spec.timeoutMs !== undefined) {
lifecycle.timeout = setTimeout(() => {
void this.stop(handle, {
kind: 'timeout',
requestedAtMs: this.clock.now(),
}).catch(() => {
diagnosticOnce(lifecycle, {
code: 'TIMEOUT_STOP_FAILED',
summary: 'Executor could not stop the process after timeout',
});
});
}, spec.timeoutMs);
lifecycle.timeout.unref?.();
}
if (!lifecycle.closedObserved && context.signal) {
const onAbort = () => {
void this.stop(handle, {
kind: 'user',
requestedAtMs: this.clock.now(),
}).catch(() => {
diagnosticOnce(lifecycle, {
code: 'ABORT_STOP_FAILED',
summary: 'Executor could not stop the process after abort',
});
});
};
context.signal.addEventListener!('abort', onAbort, { once: true });
lifecycle.removeAbortListener = () =>
context.signal?.removeEventListener?.('abort', onAbort);
if (context.signal.aborted) onAbort();
}
return handle;
}
async stop(
handle: ExecutionHandle,
reason: ExecutionStopReason,
): Promise<ExecutionStopResult> {
const state = this.states.get(handle);
if (!state) throw new ExecutorHandleNotFoundError(handle.id);
if (state.lifecycle.closedObserved) {
return {
status: 'already_exited',
termSignalSent: false,
killSignalSent: false,
};
}
if (state.stopPromise) return state.stopPromise;
state.lifecycle.terminationReason = reason;
state.stopPromise = this.terminator
.terminate({
pid: state.child.pid!,
processGroup: state.processGroup,
graceMs: state.graceMs,
closed: state.closed,
})
.then((result) => ({
status: result.alreadyExited
? ('already_exited' as const)
: ('termination_requested' as const),
termSignalSent: result.termSignalSent,
killSignalSent: result.killSignalSent,
}));
return state.stopPromise;
}
async inspect(handle: ExecutionHandle): Promise<ExecutionInspection> {
const state = this.states.get(handle);
if (!state) throw new ExecutorHandleNotFoundError(handle.id);
if (state.lifecycle.closedObserved) {
return {
status: 'exited',
...(state.lifecycle.result === undefined
? {}
: { result: state.lifecycle.result }),
};
}
return {
status: state.stopPromise ? 'stopping' : 'running',
};
}
private async pumpOutput(
stream: Readable | null,
outputStream: ExecutionOutputStream,
context: ExecutionContext,
lifecycle: LocalExecutionLifecycle,
): Promise<void> {
if (!stream) return;
let sinkAvailable = true;
try {
for await (const value of stream) {
if (!sinkAvailable) continue;
try {
const chunk =
value instanceof Uint8Array ? value : Buffer.from(String(value));
await context.output.write({
stream: outputStream,
chunk,
observedAtMs: this.clock.now(),
});
} catch {
sinkAvailable = false;
diagnosticOnce(lifecycle, {
code: 'OUTPUT_SINK_FAILED',
summary: 'Execution output sink failed; output may be incomplete',
});
}
}
} catch {
diagnosticOnce(lifecycle, {
code: 'OUTPUT_STREAM_FAILED',
summary: 'Execution output stream failed; output may be incomplete',
});
}
}
}
@@ -0,0 +1,279 @@
import { readFile } from 'fs/promises';
import type {
PersistedExecutionInspection,
PersistedExecutionInspector,
} from '../../ports/persistedExecutionInspector';
export const LOCAL_PROCESS_DURABLE_HANDLE_PREFIX = 'ql3lp1.';
export const MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES = 512;
const LINUX_BOOT_ID_PATH = '/proc/sys/kernel/random/boot_id';
export interface LinuxProcessIdentity {
platform: 'linux';
bootId: string;
pid: number;
processGroupId: number;
startTimeTicks: string;
}
interface LinuxProcessSnapshot extends LinuxProcessIdentity {
state: string;
}
export interface LocalProcessIdentityProvider {
capture(pid: number): Promise<LinuxProcessIdentity | null>;
inspect(
identity: LinuxProcessIdentity,
): Promise<PersistedExecutionInspection>;
}
export interface LinuxProcProcessIdentityProviderOptions {
platform?: NodeJS.Platform;
readTextFile?: (path: string) => Promise<string>;
}
interface DurableHandlePayload {
v: 1;
h: string;
b: string;
p: number;
g: number;
s: string;
}
function isMissingFileError(error: unknown): boolean {
return (
error instanceof Error &&
'code' in error &&
['ENOENT', 'ESRCH'].includes((error as NodeJS.ErrnoException).code ?? '')
);
}
function normalizeBootId(value: string): string {
const bootId = value.trim();
if (!/^[A-Za-z0-9-]{1,64}$/.test(bootId)) {
throw new Error('Linux boot id has an invalid format');
}
return bootId;
}
function assertPositiveSafeInteger(value: number, name: string): void {
if (!Number.isSafeInteger(value) || value < 1) {
throw new Error(`${name} must be a positive safe integer`);
}
}
function assertPositiveStartTimeTicks(value: string): void {
if (!/^\d{1,32}$/.test(value) || BigInt(value) < BigInt(1)) {
throw new Error('Linux process start time has an invalid format');
}
}
function parseLinuxProcStat(pid: number, value: string): LinuxProcessSnapshot {
assertPositiveSafeInteger(pid, 'pid');
const open = value.indexOf('(');
const close = value.lastIndexOf(')');
if (open < 1 || close <= open) {
throw new Error('Linux process stat has an invalid command field');
}
const observedPid = Number(value.slice(0, open).trim());
if (observedPid !== pid) {
throw new Error('Linux process stat PID does not match the requested PID');
}
// Values after comm begin at field 3 (state); starttime is field 22.
const fields = value
.slice(close + 1)
.trim()
.split(/\s+/);
if (fields.length < 20) {
throw new Error('Linux process stat is missing identity fields');
}
const state = fields[0];
const processGroupId = Number(fields[2]);
const startTimeTicks = fields[19];
assertPositiveSafeInteger(processGroupId, 'processGroupId');
assertPositiveStartTimeTicks(startTimeTicks);
return {
platform: 'linux',
bootId: '',
pid,
processGroupId,
startTimeTicks,
state,
};
}
function validateIdentity(identity: LinuxProcessIdentity): void {
if (identity.platform !== 'linux') {
throw new Error('Local process identity has an unsupported platform');
}
normalizeBootId(identity.bootId);
assertPositiveSafeInteger(identity.pid, 'pid');
assertPositiveSafeInteger(identity.processGroupId, 'processGroupId');
assertPositiveStartTimeTicks(identity.startTimeTicks);
}
export function createLocalProcessDurableHandle(
handleId: string,
identity: LinuxProcessIdentity,
): string {
if (!handleId || handleId.length > 255 || handleId.includes('\0')) {
throw new Error('Local process handle id has an invalid format');
}
validateIdentity(identity);
const payload: DurableHandlePayload = {
v: 1,
h: handleId,
b: identity.bootId,
p: identity.pid,
g: identity.processGroupId,
s: identity.startTimeTicks,
};
const durableHandle = `${LOCAL_PROCESS_DURABLE_HANDLE_PREFIX}${Buffer.from(
JSON.stringify(payload),
'utf8',
).toString('base64url')}`;
if (
Buffer.byteLength(durableHandle, 'utf8') >
MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES
) {
throw new Error('Local process durable handle exceeds its size limit');
}
return durableHandle;
}
export function parseLocalProcessDurableHandle(
durableHandle: string,
): { handleId: string; identity: LinuxProcessIdentity } | null {
if (
!durableHandle.startsWith(LOCAL_PROCESS_DURABLE_HANDLE_PREFIX) ||
Buffer.byteLength(durableHandle, 'utf8') >
MAX_LOCAL_PROCESS_DURABLE_HANDLE_BYTES
) {
return null;
}
const encoded = durableHandle.slice(
LOCAL_PROCESS_DURABLE_HANDLE_PREFIX.length,
);
if (!encoded || !/^[A-Za-z0-9_-]+$/.test(encoded)) return null;
try {
const payload = JSON.parse(
Buffer.from(encoded, 'base64url').toString('utf8'),
) as Partial<DurableHandlePayload>;
if (
payload.v !== 1 ||
typeof payload.h !== 'string' ||
typeof payload.b !== 'string' ||
typeof payload.p !== 'number' ||
typeof payload.g !== 'number' ||
typeof payload.s !== 'string'
) {
return null;
}
const identity: LinuxProcessIdentity = {
platform: 'linux',
bootId: payload.b,
pid: payload.p,
processGroupId: payload.g,
startTimeTicks: payload.s,
};
if (!payload.h || payload.h.length > 255 || payload.h.includes('\0')) {
return null;
}
validateIdentity(identity);
return { handleId: payload.h, identity };
} catch {
return null;
}
}
export class LinuxProcProcessIdentityProvider
implements LocalProcessIdentityProvider
{
private readonly platform: NodeJS.Platform;
private readonly readTextFile: (path: string) => Promise<string>;
constructor(options: LinuxProcProcessIdentityProviderOptions = {}) {
this.platform = options.platform ?? process.platform;
this.readTextFile =
options.readTextFile ?? ((path) => readFile(path, { encoding: 'utf8' }));
}
async capture(pid: number): Promise<LinuxProcessIdentity | null> {
if (this.platform !== 'linux') return null;
try {
const [bootIdValue, statValue] = await Promise.all([
this.readTextFile(LINUX_BOOT_ID_PATH),
this.readTextFile(`/proc/${pid}/stat`),
]);
const snapshot = parseLinuxProcStat(pid, statValue);
return {
platform: 'linux',
bootId: normalizeBootId(bootIdValue),
pid,
processGroupId: snapshot.processGroupId,
startTimeTicks: snapshot.startTimeTicks,
};
} catch (error) {
if (isMissingFileError(error)) return null;
throw error;
}
}
async inspect(
identity: LinuxProcessIdentity,
): Promise<PersistedExecutionInspection> {
if (this.platform !== 'linux') return { status: 'unsupported' };
validateIdentity(identity);
let bootId: string;
try {
bootId = normalizeBootId(await this.readTextFile(LINUX_BOOT_ID_PATH));
} catch (error) {
if (isMissingFileError(error)) return { status: 'unsupported' };
throw error;
}
if (bootId !== identity.bootId) return { status: 'identity_mismatch' };
let snapshot: LinuxProcessSnapshot;
try {
snapshot = parseLinuxProcStat(
identity.pid,
await this.readTextFile(`/proc/${identity.pid}/stat`),
);
} catch (error) {
if (isMissingFileError(error)) return { status: 'exited' };
throw error;
}
if (
snapshot.processGroupId !== identity.processGroupId ||
snapshot.startTimeTicks !== identity.startTimeTicks
) {
return { status: 'identity_mismatch' };
}
if (['Z', 'X', 'x'].includes(snapshot.state)) return { status: 'exited' };
return { status: 'running' };
}
}
export class LocalProcessPersistedExecutionInspector
implements PersistedExecutionInspector
{
readonly executorType = 'local_process' as const;
constructor(
private readonly identityProvider: LocalProcessIdentityProvider = new LinuxProcProcessIdentityProvider(),
) {}
async inspect(durableHandle: string): Promise<PersistedExecutionInspection> {
const parsed = parseLocalProcessDurableHandle(durableHandle);
if (!parsed) return { status: 'invalid' };
return {
...(await this.identityProvider.inspect(parsed.identity)),
identityPid: parsed.identity.pid,
};
}
}
@@ -0,0 +1,164 @@
import { ExecutorStopError } from '../../domain/executorErrors';
import type {
PersistedExecutionController,
PersistedExecutionStopResult,
PersistedExecutionStopStatus,
} from '../../ports/persistedExecutionController';
import {
LinuxProcProcessIdentityProvider,
parseLocalProcessDurableHandle,
type LocalProcessIdentityProvider,
type LinuxProcessIdentity,
} from './localProcessIdentity';
export const MAX_PERSISTED_LOCAL_STOP_GRACE_MS = 60_000;
export type PersistedLocalProcessSignalSender = (
pid: number,
signal: NodeJS.Signals,
) => void;
export interface PersistedLocalProcessControllerOptions {
identityProvider?: LocalProcessIdentityProvider;
sendSignal?: PersistedLocalProcessSignalSender;
graceMs?: number;
pollIntervalMs?: number;
sleep?: (delayMs: number) => Promise<void>;
}
function isNoSuchProcessError(error: unknown): boolean {
return (
error instanceof Error &&
'code' in error &&
(error as NodeJS.ErrnoException).code === 'ESRCH'
);
}
function withoutSignal(status: PersistedExecutionStopStatus) {
return { status, termSignalSent: false, killSignalSent: false } as const;
}
function mappedInspectionStatus(
status: 'identity_mismatch' | 'unsupported' | 'invalid',
termSignalSent: boolean,
): PersistedExecutionStopResult {
return {
status,
termSignalSent,
killSignalSent: false,
};
}
export class LocalProcessPersistedExecutionController
implements PersistedExecutionController
{
readonly executorType = 'local_process' as const;
private readonly identityProvider: LocalProcessIdentityProvider;
private readonly sendSignal: PersistedLocalProcessSignalSender;
private readonly graceMs: number;
private readonly pollIntervalMs: number;
private readonly sleep: (delayMs: number) => Promise<void>;
constructor(options: PersistedLocalProcessControllerOptions = {}) {
this.identityProvider =
options.identityProvider ?? new LinuxProcProcessIdentityProvider();
this.sendSignal = options.sendSignal ?? process.kill;
this.graceMs = options.graceMs ?? 5_000;
this.pollIntervalMs = options.pollIntervalMs ?? 50;
this.sleep =
options.sleep ??
((delayMs) =>
new Promise<void>((resolve) => {
setTimeout(resolve, delayMs);
}));
if (
!Number.isSafeInteger(this.graceMs) ||
this.graceMs < 0 ||
this.graceMs > MAX_PERSISTED_LOCAL_STOP_GRACE_MS
) {
throw new RangeError('Persisted local stop graceMs is invalid');
}
if (!Number.isSafeInteger(this.pollIntervalMs) || this.pollIntervalMs < 1) {
throw new RangeError(
'Persisted local stop pollIntervalMs must be a positive integer',
);
}
}
async stop({
durableHandle,
expectedPid,
}: Parameters<
PersistedExecutionController['stop']
>[0]): Promise<PersistedExecutionStopResult> {
const parsed = parseLocalProcessDurableHandle(durableHandle);
if (!parsed) return withoutSignal('invalid');
const identity = parsed.identity;
if (expectedPid !== undefined && expectedPid !== identity.pid) {
return withoutSignal('pid_mismatch');
}
// LocalProcessExecutor uses a detached child as process-group leader.
if (identity.processGroupId !== identity.pid) {
return withoutSignal('identity_mismatch');
}
const initial = await this.identityProvider.inspect(identity);
if (initial.status === 'exited') return withoutSignal('already_exited');
if (initial.status !== 'running') {
return mappedInspectionStatus(initial.status, false);
}
if (!this.trySignal(identity, 'SIGTERM')) {
return withoutSignal('already_exited');
}
let waitedMs = 0;
while (waitedMs < this.graceMs) {
const delayMs = Math.min(this.pollIntervalMs, this.graceMs - waitedMs);
await this.sleep(delayMs);
waitedMs += delayMs;
const inspection = await this.identityProvider.inspect(identity);
if (inspection.status === 'exited') {
return {
status: 'termination_requested',
termSignalSent: true,
killSignalSent: false,
};
}
if (inspection.status !== 'running') {
return mappedInspectionStatus(inspection.status, true);
}
}
const finalInspection = await this.identityProvider.inspect(identity);
if (finalInspection.status === 'exited') {
return {
status: 'termination_requested',
termSignalSent: true,
killSignalSent: false,
};
}
if (finalInspection.status !== 'running') {
return mappedInspectionStatus(finalInspection.status, true);
}
const killSignalSent = this.trySignal(identity, 'SIGKILL');
return {
status: 'termination_requested',
termSignalSent: true,
killSignalSent,
};
}
private trySignal(
identity: LinuxProcessIdentity,
signal: NodeJS.Signals,
): boolean {
try {
this.sendSignal(-identity.processGroupId, signal);
return true;
} catch (error) {
if (isNoSuchProcessError(error)) return false;
throw new ExecutorStopError(error);
}
}
}
@@ -0,0 +1,96 @@
import { ExecutorStopError } from '../../domain/executorErrors';
export interface ProcessTerminationRequest {
pid: number;
processGroup: boolean;
graceMs: number;
closed: Promise<void>;
}
export interface ProcessTerminationResult {
alreadyExited: boolean;
termSignalSent: boolean;
killSignalSent: boolean;
}
export interface ProcessTerminator {
terminate(
request: ProcessTerminationRequest,
): Promise<ProcessTerminationResult>;
}
export type ProcessSignalSender = (pid: number, signal: NodeJS.Signals) => void;
function isNoSuchProcessError(error: unknown): boolean {
return (
error instanceof Error &&
'code' in error &&
(error as NodeJS.ErrnoException).code === 'ESRCH'
);
}
async function exitsWithin(closed: Promise<void>, timeoutMs: number) {
if (timeoutMs === 0) return false;
let timer: NodeJS.Timeout | undefined;
try {
return await Promise.race([
closed.then(() => true),
new Promise<boolean>((resolve) => {
timer = setTimeout(() => resolve(false), timeoutMs);
}),
]);
} finally {
if (timer) clearTimeout(timer);
}
}
export class PosixProcessTerminator implements ProcessTerminator {
constructor(
private readonly sendSignal: ProcessSignalSender = process.kill,
) {}
async terminate(
request: ProcessTerminationRequest,
): Promise<ProcessTerminationResult> {
const targetPid = request.processGroup ? -request.pid : request.pid;
try {
this.sendSignal(targetPid, 'SIGTERM');
} catch (error) {
if (isNoSuchProcessError(error)) {
return {
alreadyExited: true,
termSignalSent: false,
killSignalSent: false,
};
}
throw new ExecutorStopError(error);
}
if (await exitsWithin(request.closed, request.graceMs)) {
return {
alreadyExited: false,
termSignalSent: true,
killSignalSent: false,
};
}
try {
this.sendSignal(targetPid, 'SIGKILL');
return {
alreadyExited: false,
termSignalSent: true,
killSignalSent: true,
};
} catch (error) {
if (isNoSuchProcessError(error)) {
return {
alreadyExited: false,
termSignalSent: true,
killSignalSent: false,
};
}
throw new ExecutorStopError(error);
}
}
}
@@ -0,0 +1,23 @@
import type { ArtifactReadAuthorizer } from '../../ports/artifactReadAuthorizer';
import type { ArtifactReadAuthorizationEffect } from '../../ports/artifactReadAuthorizer';
import type { ProjectPolicyEngine } from '../../application/projectPolicyEngine';
export class ProjectPolicyArtifactReadAuthorizer
implements ArtifactReadAuthorizer
{
constructor(private readonly policy: Pick<ProjectPolicyEngine, 'decide'>) {}
async authorize(
request: Parameters<ArtifactReadAuthorizer['authorize']>[0],
): Promise<ArtifactReadAuthorizationEffect> {
if (request.action !== 'artifact.read') {
throw new TypeError('Artifact read authorization action is invalid');
}
const result = await this.policy.decide({
subject: request.subject,
projectId: request.projectId,
permission: 'artifact.read',
});
return result.effect;
}
}
@@ -0,0 +1,132 @@
import {
activateClusterControlRuntime,
type ClusterControlActivationAudit,
type ClusterControlActivationStack,
type ClusterControlReadinessEvidence,
type ClusterControlRuntimeActivationResult,
type ClusterControlStopResult,
} from '../../application/clusterControlRuntimeActivation';
import type { DeploymentProfile } from '../../domain/deploymentProfile';
import type {
PostgresDatabaseResource,
PostgresPool,
} from '@qinglong/runtime-core';
import {
assertPostgresSchemaReady,
type PostgresSchemaReadinessReport,
} from '../../../migrations/postgresql/schemaReadiness';
import { PostgresRunRepository } from './runRepository';
export type ClusterControlDatabasePool = PostgresPool;
export type ClusterControlDatabaseResource = PostgresDatabaseResource;
export interface ClusterControlAssemblyInput {
readonly evidence: ClusterControlReadinessEvidence;
readonly runs: PostgresRunRepository;
}
export interface ClusterControlRuntimeBootstrapOptions {
readonly enabled?: boolean;
readonly profile: DeploymentProfile;
readonly openDatabase: () => Promise<ClusterControlDatabaseResource>;
readonly create: (
input: ClusterControlAssemblyInput,
) => ClusterControlActivationStack;
readonly audit: (
record: ClusterControlActivationAudit,
) => void | Promise<void>;
}
function readinessEvidence(
report: PostgresSchemaReadinessReport,
): ClusterControlReadinessEvidence {
return Object.freeze({
contractName: report.contractName,
contractVersion: report.contractVersion,
serverMajor: report.serverMajor,
migrationIds: Object.freeze([...report.migrationIds]),
});
}
/**
* Lazily owns the cluster database around the readiness-first activation gate.
* A concrete cluster package supplies openDatabase() and the pg.Pool binding;
* disabled and wrong-profile paths never import or open the database driver.
*/
export async function bootstrapClusterControlRuntime(
options: ClusterControlRuntimeBootstrapOptions,
): Promise<ClusterControlRuntimeActivationResult> {
let database: ClusterControlDatabaseResource | undefined;
let closePromise: Promise<void> | undefined;
const closeDatabase = (): Promise<void> => {
if (!database) return Promise.resolve();
closePromise ??= Promise.resolve().then(() => database!.close());
return closePromise;
};
try {
const activation = await activateClusterControlRuntime({
enabled: options.enabled,
profile: options.profile,
readiness: {
async assertReady() {
if (database) {
throw new Error(
'Cluster-control database was opened more than once',
);
}
database = await options.openDatabase();
return readinessEvidence(
await assertPostgresSchemaReady(database.pool),
);
},
},
create(evidence) {
if (!database) {
throw new Error(
'Cluster-control database is unavailable after readiness',
);
}
return options.create({
evidence,
runs: new PostgresRunRepository(database.pool),
});
},
audit: options.audit,
});
if (activation.status === 'disabled') return activation;
let stopPromise: Promise<ClusterControlStopResult> | undefined;
return {
...activation,
stop() {
if (stopPromise) return stopPromise;
stopPromise = (async () => {
let result: ClusterControlStopResult | undefined;
let primaryError: unknown;
try {
result = await activation.stop();
} catch (error) {
primaryError = error;
}
try {
await closeDatabase();
} catch (error) {
primaryError ??= error;
}
if (primaryError) throw primaryError;
return result!;
})();
return stopPromise;
},
};
} catch (error) {
try {
await closeDatabase();
} catch {
// Preserve the readiness/assembly/activation failure.
}
throw error;
}
}
@@ -0,0 +1,831 @@
import type {
RunAttemptRecord,
RunAttemptStatus,
RunEventRecord,
RunRecord,
} from '../../domain/run';
import {
EXECUTION_ORIGINS,
RUN_ATTEMPT_STATUSES,
RUN_CANCELLATION_REASONS,
RUN_EVENT_ACTOR_TYPES,
RUN_STATUSES,
} from '../../domain/run';
import {
assertRunRetryPolicyRecord,
RUN_RETRY_SAFETIES,
type RunRetryPolicyRecord,
} from '../../domain/runRetryPolicy';
import {
DuplicateIdempotencyKeyError,
DuplicateRunAttemptError,
DuplicateRunEventError,
RunEventPayloadTooLargeError,
RunRepositoryBusyError,
RunRepositoryConstraintError,
RunRepositoryError,
RunRepositoryOperationError,
} from '../../domain/repositoryErrors';
import type {
RunRepository,
RunRepositoryReader,
RunRepositoryTransaction,
} from '../../ports/runRepository';
import type {
PostgresClient as PostgresRunClient,
PostgresPool as PostgresRunPool,
PostgresQueryable as PostgresRunQueryable,
PostgresQueryResult as PostgresRunQueryResult,
} from '@qinglong/runtime-core';
export type {
PostgresClient as PostgresRunClient,
PostgresPool as PostgresRunPool,
PostgresQueryable as PostgresRunQueryable,
PostgresQueryResult as PostgresRunQueryResult,
} from '@qinglong/runtime-core';
import {
MAX_CANCELLATION_RECOVERY_PAGE_SIZE,
MAX_RUN_EVENT_PAGE_SIZE,
MAX_RUN_EVENT_PAYLOAD_BYTES,
} from '../../ports/runRepository';
interface ColumnDefinition {
readonly column: string;
readonly property: string;
}
type QueryRow = Record<string, unknown>;
const POSTGRES_RUNTIME_STATEMENT_TIMEOUT_MS = 5_000;
const POSTGRES_RUNTIME_LOCK_TIMEOUT_MS = 1_000;
const POSTGRES_RUNTIME_IDLE_TRANSACTION_TIMEOUT_MS = 10_000;
const RUN_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
{ column: 'id', property: 'id' },
{ column: 'project_id', property: 'projectId' },
{ column: 'task_id', property: 'taskId' },
{ column: 'task_revision', property: 'taskRevision' },
{ column: 'task_name', property: 'taskName' },
{ column: 'task_snapshot_ref', property: 'taskSnapshotRef' },
{ column: 'legacy_cron_id', property: 'legacyCronId' },
{ column: 'parent_run_id', property: 'parentRunId' },
{ column: 'retry_of_run_id', property: 'retryOfRunId' },
{ column: 'trigger_id', property: 'triggerId' },
{ column: 'trigger_type', property: 'triggerType' },
{ column: 'execution_origin', property: 'executionOrigin' },
{ column: 'execution_owner', property: 'executionOwner' },
{ column: 'triggered_by', property: 'triggeredBy' },
{ column: 'request_id', property: 'requestId' },
{ column: 'scheduled_for_ms', property: 'scheduledForMs' },
{ column: 'status', property: 'status' },
{ column: 'version', property: 'version' },
{ column: 'event_sequence', property: 'eventSequence' },
{ column: 'priority', property: 'priority' },
{ column: 'idempotency_key', property: 'idempotencyKey' },
{ column: 'input_ref', property: 'inputRef' },
{ column: 'output_ref', property: 'outputRef' },
{ column: 'created_at_ms', property: 'createdAtMs' },
{ column: 'queued_at_ms', property: 'queuedAtMs' },
{ column: 'started_at_ms', property: 'startedAtMs' },
{ column: 'finished_at_ms', property: 'finishedAtMs' },
{ column: 'cancel_requested_at_ms', property: 'cancelRequestedAtMs' },
{ column: 'cancel_reason', property: 'cancelReason' },
{ column: 'error_code', property: 'errorCode' },
{ column: 'error_summary', property: 'errorSummary' },
]);
const ATTEMPT_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
{ column: 'id', property: 'id' },
{ column: 'run_id', property: 'runId' },
{ column: 'step_run_id', property: 'stepRunId' },
{ column: 'attempt', property: 'attempt' },
{ column: 'status', property: 'status' },
{ column: 'executor_type', property: 'executorType' },
{ column: 'worker_id', property: 'workerId' },
{ column: 'executor_handle', property: 'executorHandle' },
{ column: 'pid', property: 'pid' },
{ column: 'log_artifact_id', property: 'logArtifactId' },
{ column: 'lease_token', property: 'leaseToken' },
{ column: 'lease_expires_at_ms', property: 'leaseExpiresAtMs' },
{ column: 'deadline_at_ms', property: 'deadlineAtMs' },
{ column: 'callback_token_hash', property: 'callbackTokenHash' },
{ column: 'callback_sequence', property: 'callbackSequence' },
{ column: 'created_at_ms', property: 'createdAtMs' },
{ column: 'started_at_ms', property: 'startedAtMs' },
{ column: 'finished_at_ms', property: 'finishedAtMs' },
{ column: 'exit_code', property: 'exitCode' },
{ column: 'error_code', property: 'errorCode' },
{ column: 'error_summary', property: 'errorSummary' },
]);
const EVENT_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
{ column: 'id', property: 'id' },
{ column: 'run_id', property: 'runId' },
{ column: 'sequence', property: 'sequence' },
{ column: 'type', property: 'type' },
{ column: 'dedupe_key', property: 'dedupeKey' },
{ column: 'actor_type', property: 'actorType' },
{ column: 'actor_id', property: 'actorId' },
{ column: 'attempt_id', property: 'attemptId' },
{ column: 'step_run_id', property: 'stepRunId' },
{ column: 'payload', property: 'payload' },
{ column: 'created_at_ms', property: 'createdAtMs' },
]);
const RETRY_POLICY_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
{ column: 'run_id', property: 'runId' },
{ column: 'max_attempts', property: 'maxAttempts' },
{ column: 'retry_on_lost', property: 'retryOnLost' },
{ column: 'safety', property: 'safety' },
{ column: 'backoff_base_ms', property: 'backoffBaseMs' },
{ column: 'backoff_max_ms', property: 'backoffMaxMs' },
{ column: 'next_attempt_at_ms', property: 'nextAttemptAtMs' },
{ column: 'version', property: 'version' },
{ column: 'created_at_ms', property: 'createdAtMs' },
{ column: 'updated_at_ms', property: 'updatedAtMs' },
]);
const TERMINAL_RUN_STATUSES = Object.freeze([
'succeeded',
'failed',
'cancelled',
'timed_out',
]);
const BUSY_SQL_STATES = new Set([
'08000',
'08001',
'08003',
'08004',
'08006',
'08007',
'08P01',
'40001',
'40P01',
'55P03',
'57014',
'57P01',
'57P02',
'57P03',
]);
const RUN_IDEMPOTENCY_CONSTRAINT = 'ql3_runs_project_idempotency_uidx';
const ATTEMPT_NUMBER_CONSTRAINT = 'ql3_run_attempts_run_attempt_uidx';
const EVENT_SEQUENCE_CONSTRAINT = 'ql3_run_events_run_sequence_uidx';
const EVENT_DEDUPE_CONSTRAINT = 'ql3_run_events_run_dedupe_uidx';
function quoted(identifier: string): string {
return `"${identifier}"`;
}
function selectColumns(columns: readonly ColumnDefinition[]): string {
return columns
.map(({ column, property }) => `${quoted(column)} AS ${quoted(property)}`)
.join(', ');
}
function insertSql(
tableName: string,
columns: readonly ColumnDefinition[],
): string {
return `INSERT INTO "ql3".${quoted(tableName)} (${columns
.map(({ column }) => quoted(column))
.join(', ')}) VALUES (${columns
.map((_, index) => `$${index + 1}`)
.join(', ')})`;
}
function updateSql(
tableName: string,
columns: readonly ColumnDefinition[],
predicate: string,
): string {
const mutableColumns = columns.slice(1);
return `UPDATE "ql3".${quoted(tableName)} SET ${mutableColumns
.map(({ column }, index) => `${quoted(column)} = $${index + 2}`)
.join(', ')} WHERE ${predicate} RETURNING ${quoted(columns[0].column)}`;
}
function writeValues(
record: object,
columns: readonly ColumnDefinition[],
): unknown[] {
const values = record as Record<string, unknown>;
return columns.map(({ property }) => values[property] ?? null);
}
function requiredString(row: QueryRow, property: string): string {
const value = row[property];
if (typeof value !== 'string' || value.length === 0) {
throw new RunRepositoryConstraintError(
`PostgreSQL Run row has an invalid ${property}`,
);
}
return value;
}
function optionalString(row: QueryRow, property: string): string | undefined {
const value = row[property];
if (value === null || value === undefined) return undefined;
if (typeof value !== 'string') {
throw new RunRepositoryConstraintError(
`PostgreSQL Run row has an invalid ${property}`,
);
}
return value;
}
function requiredInteger(row: QueryRow, property: string): number {
const value = row[property];
if (typeof value === 'number' && Number.isSafeInteger(value)) return value;
if (typeof value === 'string' && /^-?(0|[1-9]\d*)$/.test(value)) {
const parsed = Number(value);
if (Number.isSafeInteger(parsed)) return parsed;
}
throw new RunRepositoryConstraintError(
`PostgreSQL Run row has an invalid ${property}`,
);
}
function optionalInteger(row: QueryRow, property: string): number | undefined {
if (row[property] === null || row[property] === undefined) return undefined;
return requiredInteger(row, property);
}
function requiredBoolean(row: QueryRow, property: string): boolean {
const value = row[property];
if (typeof value !== 'boolean') {
throw new RunRepositoryConstraintError(
`PostgreSQL Run row has an invalid ${property}`,
);
}
return value;
}
function requiredEnum<T extends string>(
row: QueryRow,
property: string,
allowed: readonly T[],
): T {
const value = requiredString(row, property);
if (!allowed.includes(value as T)) {
throw new RunRepositoryConstraintError(
`PostgreSQL Run row has an unsupported ${property}`,
);
}
return value as T;
}
function assignOptional<T extends object, K extends keyof T>(
record: T,
key: K,
value: T[K] | undefined,
): void {
if (value !== undefined) record[key] = value;
}
function rowToRun(row: QueryRow): RunRecord {
const run: RunRecord = {
id: requiredString(row, 'id'),
projectId: requiredString(row, 'projectId'),
taskId: requiredString(row, 'taskId'),
taskRevision: requiredString(row, 'taskRevision'),
triggerType: requiredString(row, 'triggerType'),
executionOrigin: requiredEnum(row, 'executionOrigin', EXECUTION_ORIGINS),
executionOwner: requiredEnum(row, 'executionOwner', [
'legacy',
'runtime',
] as const),
status: requiredEnum(row, 'status', RUN_STATUSES),
version: requiredInteger(row, 'version'),
eventSequence: requiredInteger(row, 'eventSequence'),
priority: requiredInteger(row, 'priority'),
createdAtMs: requiredInteger(row, 'createdAtMs'),
};
assignOptional(run, 'taskName', optionalString(row, 'taskName'));
assignOptional(
run,
'taskSnapshotRef',
optionalString(row, 'taskSnapshotRef'),
);
assignOptional(run, 'legacyCronId', optionalInteger(row, 'legacyCronId'));
assignOptional(run, 'parentRunId', optionalString(row, 'parentRunId'));
assignOptional(run, 'retryOfRunId', optionalString(row, 'retryOfRunId'));
assignOptional(run, 'triggerId', optionalString(row, 'triggerId'));
assignOptional(run, 'triggeredBy', optionalString(row, 'triggeredBy'));
assignOptional(run, 'requestId', optionalString(row, 'requestId'));
assignOptional(run, 'scheduledForMs', optionalInteger(row, 'scheduledForMs'));
assignOptional(run, 'idempotencyKey', optionalString(row, 'idempotencyKey'));
assignOptional(run, 'inputRef', optionalString(row, 'inputRef'));
assignOptional(run, 'outputRef', optionalString(row, 'outputRef'));
assignOptional(run, 'queuedAtMs', optionalInteger(row, 'queuedAtMs'));
assignOptional(run, 'startedAtMs', optionalInteger(row, 'startedAtMs'));
assignOptional(run, 'finishedAtMs', optionalInteger(row, 'finishedAtMs'));
assignOptional(
run,
'cancelRequestedAtMs',
optionalInteger(row, 'cancelRequestedAtMs'),
);
if (row.cancelReason !== null && row.cancelReason !== undefined) {
run.cancelReason = requiredEnum(
row,
'cancelReason',
RUN_CANCELLATION_REASONS,
);
}
assignOptional(run, 'errorCode', optionalString(row, 'errorCode'));
assignOptional(run, 'errorSummary', optionalString(row, 'errorSummary'));
return run;
}
function rowToAttempt(row: QueryRow): RunAttemptRecord {
const attempt: RunAttemptRecord = {
id: requiredString(row, 'id'),
runId: requiredString(row, 'runId'),
attempt: requiredInteger(row, 'attempt'),
status: requiredEnum(row, 'status', RUN_ATTEMPT_STATUSES),
executorType: requiredString(row, 'executorType'),
callbackSequence: requiredInteger(row, 'callbackSequence'),
createdAtMs: requiredInteger(row, 'createdAtMs'),
};
assignOptional(attempt, 'stepRunId', optionalString(row, 'stepRunId'));
assignOptional(attempt, 'workerId', optionalString(row, 'workerId'));
assignOptional(
attempt,
'executorHandle',
optionalString(row, 'executorHandle'),
);
assignOptional(attempt, 'pid', optionalInteger(row, 'pid'));
assignOptional(
attempt,
'logArtifactId',
optionalString(row, 'logArtifactId'),
);
assignOptional(attempt, 'leaseToken', optionalString(row, 'leaseToken'));
assignOptional(
attempt,
'leaseExpiresAtMs',
optionalInteger(row, 'leaseExpiresAtMs'),
);
assignOptional(attempt, 'deadlineAtMs', optionalInteger(row, 'deadlineAtMs'));
assignOptional(
attempt,
'callbackTokenHash',
optionalString(row, 'callbackTokenHash'),
);
assignOptional(attempt, 'startedAtMs', optionalInteger(row, 'startedAtMs'));
assignOptional(attempt, 'finishedAtMs', optionalInteger(row, 'finishedAtMs'));
assignOptional(attempt, 'exitCode', optionalInteger(row, 'exitCode'));
assignOptional(attempt, 'errorCode', optionalString(row, 'errorCode'));
assignOptional(attempt, 'errorSummary', optionalString(row, 'errorSummary'));
return attempt;
}
function normalizePayload(payload: unknown): Readonly<Record<string, unknown>> {
let value = payload;
if (typeof value === 'string') {
try {
value = JSON.parse(value);
} catch (error) {
throw new RunRepositoryConstraintError(
'PostgreSQL RunEvent payload is invalid JSON',
error,
);
}
}
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new RunRepositoryConstraintError(
'PostgreSQL RunEvent payload is not a JSON object',
);
}
return value as Readonly<Record<string, unknown>>;
}
function rowToEvent(row: QueryRow): RunEventRecord {
const event: RunEventRecord = {
id: requiredString(row, 'id'),
runId: requiredString(row, 'runId'),
sequence: requiredInteger(row, 'sequence'),
type: requiredString(row, 'type'),
actorType: requiredEnum(row, 'actorType', RUN_EVENT_ACTOR_TYPES),
payload: normalizePayload(row.payload),
createdAtMs: requiredInteger(row, 'createdAtMs'),
};
assignOptional(event, 'dedupeKey', optionalString(row, 'dedupeKey'));
assignOptional(event, 'actorId', optionalString(row, 'actorId'));
assignOptional(event, 'attemptId', optionalString(row, 'attemptId'));
assignOptional(event, 'stepRunId', optionalString(row, 'stepRunId'));
return event;
}
function rowToRetryPolicy(row: QueryRow): RunRetryPolicyRecord {
const policy: RunRetryPolicyRecord = {
runId: requiredString(row, 'runId'),
maxAttempts: requiredInteger(row, 'maxAttempts'),
retryOnLost: requiredBoolean(row, 'retryOnLost'),
safety: requiredEnum(row, 'safety', RUN_RETRY_SAFETIES),
backoffBaseMs: requiredInteger(row, 'backoffBaseMs'),
backoffMaxMs: requiredInteger(row, 'backoffMaxMs'),
version: requiredInteger(row, 'version'),
createdAtMs: requiredInteger(row, 'createdAtMs'),
updatedAtMs: requiredInteger(row, 'updatedAtMs'),
};
assignOptional(
policy,
'nextAttemptAtMs',
optionalInteger(row, 'nextAttemptAtMs'),
);
assertRunRetryPolicyRecord(policy);
return policy;
}
function sqlState(error: unknown): string | undefined {
if (!error || typeof error !== 'object') return undefined;
const value = (error as { code?: unknown }).code;
return typeof value === 'string' ? value : undefined;
}
function constraintName(error: unknown): string | undefined {
if (!error || typeof error !== 'object') return undefined;
const value = (error as { constraint?: unknown }).constraint;
return typeof value === 'string' ? value : undefined;
}
function mapPostgresError(error: unknown): RunRepositoryError {
if (error instanceof RunRepositoryError) return error;
const state = sqlState(error);
if (state && BUSY_SQL_STATES.has(state)) {
return new RunRepositoryBusyError(error);
}
if (state?.startsWith('23')) {
return new RunRepositoryConstraintError(
'PostgreSQL Run repository constraint violation',
error,
);
}
return new RunRepositoryOperationError(error);
}
function affectedOneOrNone(result: PostgresRunQueryResult): boolean {
const count = result.rowCount ?? result.rows.length;
if (count === 0) return false;
if (count === 1) return true;
throw new RunRepositoryConstraintError(
'PostgreSQL compare-and-set affected more than one row',
);
}
function assertEventPayloadSize(event: RunEventRecord): void {
let serialized: string;
try {
serialized = JSON.stringify(event.payload);
} catch (error) {
throw new RunRepositoryConstraintError(
'RunEvent payload is not JSON serializable',
error,
);
}
const bytes = Buffer.byteLength(serialized, 'utf8');
if (bytes > MAX_RUN_EVENT_PAYLOAD_BYTES) {
throw new RunEventPayloadTooLargeError(bytes, MAX_RUN_EVENT_PAYLOAD_BYTES);
}
}
async function queryMapped<TRow extends QueryRow = QueryRow>(
queryable: PostgresRunQueryable,
text: string,
values?: readonly unknown[],
): Promise<PostgresRunQueryResult<TRow>> {
try {
return await queryable.query<TRow>(text, values);
} catch (error) {
throw mapPostgresError(error);
}
}
function singleRow<TRow extends QueryRow>(
result: PostgresRunQueryResult<TRow>,
): TRow | null {
if (result.rows.length === 0) return null;
if (result.rows.length !== 1) {
throw new RunRepositoryConstraintError(
'PostgreSQL Run repository returned duplicate identity rows',
);
}
return result.rows[0];
}
const RUN_SELECT = selectColumns(RUN_COLUMNS);
const ATTEMPT_SELECT = selectColumns(ATTEMPT_COLUMNS);
const EVENT_SELECT = selectColumns(EVENT_COLUMNS);
const RETRY_POLICY_SELECT = selectColumns(RETRY_POLICY_COLUMNS);
const INSERT_RUN_SQL = insertSql('runs', RUN_COLUMNS);
const INSERT_ATTEMPT_SQL = insertSql('run_attempts', ATTEMPT_COLUMNS);
const INSERT_EVENT_SQL = insertSql('run_events', EVENT_COLUMNS);
const INSERT_RETRY_POLICY_SQL = insertSql(
'run_retry_policies',
RETRY_POLICY_COLUMNS,
);
const UPDATE_RUN_SQL = updateSql(
'runs',
RUN_COLUMNS,
`"id" = $1 AND "version" = $${RUN_COLUMNS.length + 1}`,
);
const UPDATE_ATTEMPT_SQL = updateSql(
'run_attempts',
ATTEMPT_COLUMNS,
`"id" = $1 AND "status" = $${
ATTEMPT_COLUMNS.length + 1
} AND "callback_sequence" = $${ATTEMPT_COLUMNS.length + 2}`,
);
const UPDATE_RETRY_POLICY_SQL = updateSql(
'run_retry_policies',
RETRY_POLICY_COLUMNS,
`"run_id" = $1 AND "version" = $${RETRY_POLICY_COLUMNS.length + 1}`,
);
class PostgresRunReader implements RunRepositoryReader {
constructor(protected readonly queryable: PostgresRunQueryable) {}
async findRunById(runId: string): Promise<RunRecord | null> {
const row = singleRow(
await queryMapped(
this.queryable,
`SELECT ${RUN_SELECT} FROM "ql3"."runs" WHERE "id" = $1`,
[runId],
),
);
return row ? rowToRun(row) : null;
}
async findAttemptById(attemptId: string): Promise<RunAttemptRecord | null> {
const row = singleRow(
await queryMapped(
this.queryable,
`SELECT ${ATTEMPT_SELECT} FROM "ql3"."run_attempts" WHERE "id" = $1`,
[attemptId],
),
);
return row ? rowToAttempt(row) : null;
}
async findLatestAttemptByRunId(
runId: string,
): Promise<RunAttemptRecord | null> {
const row = singleRow(
await queryMapped(
this.queryable,
`SELECT ${ATTEMPT_SELECT} FROM "ql3"."run_attempts" WHERE "run_id" = $1 ORDER BY "attempt" DESC, "id" DESC LIMIT 1`,
[runId],
),
);
return row ? rowToAttempt(row) : null;
}
async findRetryPolicyByRunId(
runId: string,
): Promise<RunRetryPolicyRecord | null> {
const row = singleRow(
await queryMapped(
this.queryable,
`SELECT ${RETRY_POLICY_SELECT} FROM "ql3"."run_retry_policies" WHERE "run_id" = $1`,
[runId],
),
);
return row ? rowToRetryPolicy(row) : null;
}
async listEvents(
runId: string,
options: { afterSequence?: number; limit?: number } = {},
): Promise<RunEventRecord[]> {
const afterSequence = options.afterSequence ?? 0;
const limit = options.limit ?? 100;
if (!Number.isInteger(afterSequence) || afterSequence < 0) {
throw new RangeError('afterSequence must be a non-negative integer');
}
if (
!Number.isInteger(limit) ||
limit < 1 ||
limit > MAX_RUN_EVENT_PAGE_SIZE
) {
throw new RangeError(
'limit must be between 1 and MAX_RUN_EVENT_PAGE_SIZE',
);
}
const result = await queryMapped(
this.queryable,
`SELECT ${EVENT_SELECT} FROM "ql3"."run_events" WHERE "run_id" = $1 AND "sequence" > $2 ORDER BY "sequence", "id" LIMIT $3`,
[runId, afterSequence, limit],
);
return result.rows.map(rowToEvent);
}
async listCancellationRequested(
options: { beforeMs?: number; limit?: number } = {},
): Promise<RunRecord[]> {
const beforeMs = options.beforeMs;
const limit = options.limit ?? 100;
if (
beforeMs !== undefined &&
(!Number.isSafeInteger(beforeMs) || beforeMs < 0)
) {
throw new RangeError('beforeMs must be a non-negative safe integer');
}
if (
!Number.isSafeInteger(limit) ||
limit < 1 ||
limit > MAX_CANCELLATION_RECOVERY_PAGE_SIZE
) {
throw new RangeError(
'limit must be between 1 and MAX_CANCELLATION_RECOVERY_PAGE_SIZE',
);
}
const result = await queryMapped(
this.queryable,
`SELECT ${RUN_SELECT} FROM "ql3"."runs" WHERE "status" <> ALL($1::text[]) AND "cancel_requested_at_ms" IS NOT NULL AND ($2::bigint IS NULL OR "cancel_requested_at_ms" <= $2) ORDER BY "cancel_requested_at_ms", "id" LIMIT $3`,
[TERMINAL_RUN_STATUSES, beforeMs ?? null, limit],
);
return result.rows.map(rowToRun);
}
}
export class PostgresRunTransaction
extends PostgresRunReader
implements RunRepositoryTransaction
{
async insertRun(run: RunRecord): Promise<void> {
try {
await this.queryable.query(INSERT_RUN_SQL, writeValues(run, RUN_COLUMNS));
} catch (error) {
if (
sqlState(error) === '23505' &&
constraintName(error) === RUN_IDEMPOTENCY_CONSTRAINT &&
run.idempotencyKey
) {
throw new DuplicateIdempotencyKeyError(
run.projectId,
run.idempotencyKey,
);
}
throw mapPostgresError(error);
}
}
async insertAttempt(attempt: RunAttemptRecord): Promise<void> {
try {
await this.queryable.query(
INSERT_ATTEMPT_SQL,
writeValues(attempt, ATTEMPT_COLUMNS),
);
} catch (error) {
if (
sqlState(error) === '23505' &&
constraintName(error) === ATTEMPT_NUMBER_CONSTRAINT
) {
throw new DuplicateRunAttemptError(attempt.runId, attempt.attempt);
}
throw mapPostgresError(error);
}
}
async insertRetryPolicy(policy: RunRetryPolicyRecord): Promise<void> {
assertRunRetryPolicyRecord(policy);
try {
await this.queryable.query(
INSERT_RETRY_POLICY_SQL,
writeValues(policy, RETRY_POLICY_COLUMNS),
);
} catch (error) {
throw mapPostgresError(error);
}
}
async compareAndSetRun(
run: RunRecord,
expectedVersion: number,
): Promise<boolean> {
if (run.version !== expectedVersion + 1) {
throw new RunRepositoryConstraintError(
'A compare-and-set Run write must increment version exactly once',
);
}
const result = await queryMapped(this.queryable, UPDATE_RUN_SQL, [
...writeValues(run, RUN_COLUMNS),
expectedVersion,
]);
return affectedOneOrNone(result);
}
async compareAndSetAttempt(
attempt: RunAttemptRecord,
expected: {
status: RunAttemptStatus;
callbackSequence: number;
},
): Promise<boolean> {
const result = await queryMapped(this.queryable, UPDATE_ATTEMPT_SQL, [
...writeValues(attempt, ATTEMPT_COLUMNS),
expected.status,
expected.callbackSequence,
]);
return affectedOneOrNone(result);
}
async compareAndSetRetryPolicy(
policy: RunRetryPolicyRecord,
expectedVersion: number,
): Promise<boolean> {
if (policy.version !== expectedVersion + 1) {
throw new RunRepositoryConstraintError(
'A compare-and-set retry policy write must increment version exactly once',
);
}
assertRunRetryPolicyRecord(policy);
const result = await queryMapped(this.queryable, UPDATE_RETRY_POLICY_SQL, [
...writeValues(policy, RETRY_POLICY_COLUMNS),
expectedVersion,
]);
return affectedOneOrNone(result);
}
async appendEvent(event: RunEventRecord): Promise<void> {
assertEventPayloadSize(event);
try {
await this.queryable.query(
INSERT_EVENT_SQL,
writeValues(event, EVENT_COLUMNS),
);
} catch (error) {
if (
sqlState(error) === '23505' &&
(constraintName(error) === EVENT_SEQUENCE_CONSTRAINT ||
constraintName(error) === EVENT_DEDUPE_CONSTRAINT)
) {
throw new DuplicateRunEventError(event.runId, event.dedupeKey);
}
throw mapPostgresError(error);
}
}
}
/**
* Driver-neutral PostgreSQL Run Repository. The cluster-only package owns the
* concrete pg.Pool binding; edge/standalone builds never import the driver.
*/
export class PostgresRunRepository
extends PostgresRunReader
implements RunRepository
{
constructor(private readonly pool: PostgresRunPool) {
super(pool);
}
async transaction<T>(
work: (transaction: RunRepositoryTransaction) => Promise<T>,
): Promise<T> {
let client: PostgresRunClient;
try {
client = await this.pool.connect();
} catch (error) {
throw mapPostgresError(error);
}
let began = false;
let phase: 'begin' | 'work' | 'commit' = 'begin';
try {
await client.query('BEGIN');
began = true;
await client.query('SET TRANSACTION ISOLATION LEVEL READ COMMITTED');
await client.query(`SELECT set_config('statement_timeout', $1, true)`, [
`${POSTGRES_RUNTIME_STATEMENT_TIMEOUT_MS}ms`,
]);
await client.query(`SELECT set_config('lock_timeout', $1, true)`, [
`${POSTGRES_RUNTIME_LOCK_TIMEOUT_MS}ms`,
]);
await client.query(
`SELECT set_config('idle_in_transaction_session_timeout', $1, true)`,
[`${POSTGRES_RUNTIME_IDLE_TRANSACTION_TIMEOUT_MS}ms`],
);
phase = 'work';
const result = await work(new PostgresRunTransaction(client));
phase = 'commit';
await client.query('COMMIT');
began = false;
return result;
} catch (error) {
if (began) {
try {
await client.query('ROLLBACK');
} catch {
// Preserve the work/commit failure; release discards broken clients.
}
}
if (phase === 'work') throw error;
throw mapPostgresError(error);
} finally {
client.release();
}
}
}