mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 18:08:20 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,925 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
APPROVAL_REQUEST_TABLE,
|
||||
APPROVED_ACTION_DISPATCH_TABLE,
|
||||
} from '../../../migrations/0020-approval-requests';
|
||||
import { APPROVED_ACTION_DISPATCH_EXECUTION_TABLE } from '../../../migrations/0021-approved-action-dispatch-executions';
|
||||
import { DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS } from '../../domain/approvedActionDispatchExecution';
|
||||
import {
|
||||
ApprovalMutationConflictError,
|
||||
ApprovalPolicyFenceConflictError,
|
||||
ApprovalRequestExpiredError,
|
||||
ApprovalRequestNotFoundError,
|
||||
ApprovalRequestStateConflictError,
|
||||
ApprovalRequestVersionConflictError,
|
||||
ApprovalUnavailableError,
|
||||
InvalidApprovalValueError,
|
||||
normalizeApprovalActionBinding,
|
||||
normalizeApprovalPolicyFence,
|
||||
normalizeApprovalRequestRecord,
|
||||
normalizeApprovedActionDispatchRecord,
|
||||
sameApprovalAction,
|
||||
sameApprovalSubject,
|
||||
type ApprovalRequestRecord,
|
||||
type ApprovedActionDispatchRecord,
|
||||
} from '../../domain/approvalRequest';
|
||||
import {
|
||||
normalizePolicySubject,
|
||||
type ProjectPolicyFence,
|
||||
} from '../../domain/projectPolicy';
|
||||
import type {
|
||||
ApprovalRequestRepository,
|
||||
ConsumeApprovalRequestCommand,
|
||||
ConsumeApprovalRequestResult,
|
||||
CreateApprovalRequestCommand,
|
||||
CreateApprovalRequestResult,
|
||||
DecideApprovalRequestCommand,
|
||||
DecideApprovalRequestResult,
|
||||
} from '../../ports/approvalRequestRepository';
|
||||
import {
|
||||
PROJECT_ROLE_BINDING_TABLE,
|
||||
PROJECT_TABLE,
|
||||
} from '../../../migrations/0017-project-policy';
|
||||
|
||||
const RETRY_ATTEMPTS = 5;
|
||||
|
||||
interface ApprovalRequestRow {
|
||||
id: string;
|
||||
projectId: string;
|
||||
version: number;
|
||||
state: string;
|
||||
permission: string;
|
||||
actionType: string;
|
||||
actionRef: string;
|
||||
actionDigest: string;
|
||||
previewDigest: string;
|
||||
risk: string;
|
||||
requestedByType: string;
|
||||
requestedById: string;
|
||||
requestedAtMs: number | string;
|
||||
expiresAtMs: number | string;
|
||||
decisionId: string | null;
|
||||
decision: string | null;
|
||||
decisionReasonCode: string | null;
|
||||
decidedByType: string | null;
|
||||
decidedById: string | null;
|
||||
decidedAtMs: number | string | null;
|
||||
consumptionId: string | null;
|
||||
dispatchId: string | null;
|
||||
consumedByType: string | null;
|
||||
consumedById: string | null;
|
||||
consumedAtMs: number | string | null;
|
||||
}
|
||||
|
||||
interface ApprovalRequestInstance
|
||||
extends Model<ApprovalRequestRow, ApprovalRequestRow>,
|
||||
ApprovalRequestRow {}
|
||||
|
||||
interface ApprovedActionDispatchRow {
|
||||
id: string;
|
||||
approvalRequestId: string;
|
||||
approvalRequestVersion: number;
|
||||
projectId: string;
|
||||
state: string;
|
||||
permission: string;
|
||||
actionType: string;
|
||||
actionRef: string;
|
||||
actionDigest: string;
|
||||
previewDigest: string;
|
||||
requestedByType: string;
|
||||
requestedById: string;
|
||||
consumedByType: string;
|
||||
consumedById: string;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface ApprovedActionDispatchInstance
|
||||
extends Model<ApprovedActionDispatchRow, ApprovedActionDispatchRow>,
|
||||
ApprovedActionDispatchRow {}
|
||||
|
||||
interface PolicyFenceRow {
|
||||
project_version: number | string;
|
||||
binding_version: number | string | null;
|
||||
}
|
||||
|
||||
function defineApprovalRequestModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<ApprovalRequestInstance> {
|
||||
return database.define<ApprovalRequestInstance>(
|
||||
'Ql3ApprovalRequest',
|
||||
{
|
||||
id: { type: DataTypes.STRING(64), allowNull: false, primaryKey: true },
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
},
|
||||
version: { type: DataTypes.INTEGER, allowNull: false },
|
||||
state: { type: DataTypes.STRING(16), allowNull: false },
|
||||
permission: { type: DataTypes.STRING(255), allowNull: false },
|
||||
actionType: {
|
||||
field: 'action_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
actionRef: {
|
||||
field: 'action_ref',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
actionDigest: {
|
||||
field: 'action_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
previewDigest: {
|
||||
field: 'preview_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
risk: { type: DataTypes.STRING(16), allowNull: false },
|
||||
requestedByType: {
|
||||
field: 'requested_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
requestedById: {
|
||||
field: 'requested_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
requestedAtMs: {
|
||||
field: 'requested_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
expiresAtMs: {
|
||||
field: 'expires_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
decisionId: {
|
||||
field: 'decision_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: true,
|
||||
},
|
||||
decision: { type: DataTypes.STRING(16), allowNull: true },
|
||||
decisionReasonCode: {
|
||||
field: 'decision_reason_code',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: true,
|
||||
},
|
||||
decidedByType: {
|
||||
field: 'decided_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: true,
|
||||
},
|
||||
decidedById: {
|
||||
field: 'decided_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: true,
|
||||
},
|
||||
decidedAtMs: {
|
||||
field: 'decided_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
consumptionId: {
|
||||
field: 'consumption_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: true,
|
||||
},
|
||||
dispatchId: {
|
||||
field: 'dispatch_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: true,
|
||||
},
|
||||
consumedByType: {
|
||||
field: 'consumed_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: true,
|
||||
},
|
||||
consumedById: {
|
||||
field: 'consumed_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: true,
|
||||
},
|
||||
consumedAtMs: {
|
||||
field: 'consumed_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: APPROVAL_REQUEST_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function defineApprovedActionDispatchModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<ApprovedActionDispatchInstance> {
|
||||
return database.define<ApprovedActionDispatchInstance>(
|
||||
'Ql3ApprovedActionDispatch',
|
||||
{
|
||||
id: { type: DataTypes.STRING(64), allowNull: false, primaryKey: true },
|
||||
approvalRequestId: {
|
||||
field: 'approval_request_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
approvalRequestVersion: {
|
||||
field: 'approval_request_version',
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
},
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
},
|
||||
state: { type: DataTypes.STRING(16), allowNull: false },
|
||||
permission: { type: DataTypes.STRING(255), allowNull: false },
|
||||
actionType: {
|
||||
field: 'action_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
actionRef: {
|
||||
field: 'action_ref',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
actionDigest: {
|
||||
field: 'action_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
previewDigest: {
|
||||
field: 'preview_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
requestedByType: {
|
||||
field: 'requested_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
requestedById: {
|
||||
field: 'requested_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
consumedByType: {
|
||||
field: 'consumed_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
consumedById: {
|
||||
field: 'consumed_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: APPROVED_ACTION_DISPATCH_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function rowToRequest(
|
||||
row: ApprovalRequestRow,
|
||||
): Readonly<ApprovalRequestRecord> {
|
||||
try {
|
||||
return normalizeApprovalRequestRecord({
|
||||
id: row.id,
|
||||
projectId: row.projectId,
|
||||
version: Number(row.version),
|
||||
state: row.state as ApprovalRequestRecord['state'],
|
||||
action: {
|
||||
permission:
|
||||
row.permission as ApprovalRequestRecord['action']['permission'],
|
||||
actionType: row.actionType,
|
||||
actionRef: row.actionRef,
|
||||
actionDigest: row.actionDigest,
|
||||
previewDigest: row.previewDigest,
|
||||
},
|
||||
risk: row.risk as ApprovalRequestRecord['risk'],
|
||||
requestedBy: {
|
||||
type: row.requestedByType as ApprovalRequestRecord['requestedBy']['type'],
|
||||
id: row.requestedById,
|
||||
},
|
||||
requestedAtMs: Number(row.requestedAtMs),
|
||||
expiresAtMs: Number(row.expiresAtMs),
|
||||
decisionId: row.decisionId,
|
||||
decision: row.decision as ApprovalRequestRecord['decision'],
|
||||
decisionReasonCode: row.decisionReasonCode,
|
||||
decidedBy:
|
||||
row.decidedByType === null || row.decidedById === null
|
||||
? null
|
||||
: {
|
||||
type: row.decidedByType as NonNullable<
|
||||
ApprovalRequestRecord['decidedBy']
|
||||
>['type'],
|
||||
id: row.decidedById,
|
||||
},
|
||||
decidedAtMs: row.decidedAtMs === null ? null : Number(row.decidedAtMs),
|
||||
consumptionId: row.consumptionId,
|
||||
dispatchId: row.dispatchId,
|
||||
consumedBy:
|
||||
row.consumedByType === null || row.consumedById === null
|
||||
? null
|
||||
: {
|
||||
type: row.consumedByType as NonNullable<
|
||||
ApprovalRequestRecord['consumedBy']
|
||||
>['type'],
|
||||
id: row.consumedById,
|
||||
},
|
||||
consumedAtMs: row.consumedAtMs === null ? null : Number(row.consumedAtMs),
|
||||
});
|
||||
} catch {
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function requestToRow(
|
||||
request: Readonly<ApprovalRequestRecord>,
|
||||
): ApprovalRequestRow {
|
||||
return {
|
||||
id: request.id,
|
||||
projectId: request.projectId,
|
||||
version: request.version,
|
||||
state: request.state,
|
||||
permission: request.action.permission,
|
||||
actionType: request.action.actionType,
|
||||
actionRef: request.action.actionRef,
|
||||
actionDigest: request.action.actionDigest,
|
||||
previewDigest: request.action.previewDigest,
|
||||
risk: request.risk,
|
||||
requestedByType: request.requestedBy.type,
|
||||
requestedById: request.requestedBy.id,
|
||||
requestedAtMs: request.requestedAtMs,
|
||||
expiresAtMs: request.expiresAtMs,
|
||||
decisionId: request.decisionId,
|
||||
decision: request.decision,
|
||||
decisionReasonCode: request.decisionReasonCode,
|
||||
decidedByType: request.decidedBy?.type ?? null,
|
||||
decidedById: request.decidedBy?.id ?? null,
|
||||
decidedAtMs: request.decidedAtMs,
|
||||
consumptionId: request.consumptionId,
|
||||
dispatchId: request.dispatchId,
|
||||
consumedByType: request.consumedBy?.type ?? null,
|
||||
consumedById: request.consumedBy?.id ?? null,
|
||||
consumedAtMs: request.consumedAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function rowToDispatch(
|
||||
row: ApprovedActionDispatchRow,
|
||||
): Readonly<ApprovedActionDispatchRecord> {
|
||||
try {
|
||||
return normalizeApprovedActionDispatchRecord({
|
||||
id: row.id,
|
||||
approvalRequestId: row.approvalRequestId,
|
||||
approvalRequestVersion: Number(row.approvalRequestVersion),
|
||||
projectId: row.projectId,
|
||||
state: row.state as ApprovedActionDispatchRecord['state'],
|
||||
action: {
|
||||
permission:
|
||||
row.permission as ApprovedActionDispatchRecord['action']['permission'],
|
||||
actionType: row.actionType,
|
||||
actionRef: row.actionRef,
|
||||
actionDigest: row.actionDigest,
|
||||
previewDigest: row.previewDigest,
|
||||
},
|
||||
requestedBy: {
|
||||
type: row.requestedByType as ApprovedActionDispatchRecord['requestedBy']['type'],
|
||||
id: row.requestedById,
|
||||
},
|
||||
consumedBy: {
|
||||
type: row.consumedByType as ApprovedActionDispatchRecord['consumedBy']['type'],
|
||||
id: row.consumedById,
|
||||
},
|
||||
createdAtMs: Number(row.createdAtMs),
|
||||
});
|
||||
} catch {
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function dispatchToRow(
|
||||
dispatch: Readonly<ApprovedActionDispatchRecord>,
|
||||
): ApprovedActionDispatchRow {
|
||||
return {
|
||||
id: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
approvalRequestVersion: dispatch.approvalRequestVersion,
|
||||
projectId: dispatch.projectId,
|
||||
state: dispatch.state,
|
||||
permission: dispatch.action.permission,
|
||||
actionType: dispatch.action.actionType,
|
||||
actionRef: dispatch.action.actionRef,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
previewDigest: dispatch.action.previewDigest,
|
||||
requestedByType: dispatch.requestedBy.type,
|
||||
requestedById: dispatch.requestedBy.id,
|
||||
consumedByType: dispatch.consumedBy.type,
|
||||
consumedById: dispatch.consumedBy.id,
|
||||
createdAtMs: dispatch.createdAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function sameRequestCreation(
|
||||
left: Readonly<ApprovalRequestRecord>,
|
||||
right: Readonly<ApprovalRequestRecord>,
|
||||
): boolean {
|
||||
return (
|
||||
left.id === right.id &&
|
||||
left.projectId === right.projectId &&
|
||||
sameApprovalAction(left.action, right.action) &&
|
||||
left.risk === right.risk &&
|
||||
sameApprovalSubject(left.requestedBy, right.requestedBy) &&
|
||||
left.requestedAtMs === right.requestedAtMs &&
|
||||
left.expiresAtMs === right.expiresAtMs
|
||||
);
|
||||
}
|
||||
|
||||
function sameDispatch(
|
||||
left: Readonly<ApprovedActionDispatchRecord>,
|
||||
right: Readonly<ApprovedActionDispatchRecord>,
|
||||
): boolean {
|
||||
return (
|
||||
left.id === right.id &&
|
||||
left.approvalRequestId === right.approvalRequestId &&
|
||||
left.approvalRequestVersion === right.approvalRequestVersion &&
|
||||
left.projectId === right.projectId &&
|
||||
left.state === right.state &&
|
||||
sameApprovalAction(left.action, right.action) &&
|
||||
sameApprovalSubject(left.requestedBy, right.requestedBy) &&
|
||||
sameApprovalSubject(left.consumedBy, right.consumedBy) &&
|
||||
left.createdAtMs === right.createdAtMs
|
||||
);
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
function isApprovalError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof ApprovalMutationConflictError ||
|
||||
error instanceof InvalidApprovalValueError ||
|
||||
error instanceof ApprovalPolicyFenceConflictError ||
|
||||
error instanceof ApprovalRequestExpiredError ||
|
||||
error instanceof ApprovalRequestNotFoundError ||
|
||||
error instanceof ApprovalRequestStateConflictError ||
|
||||
error instanceof ApprovalRequestVersionConflictError ||
|
||||
error instanceof ApprovalUnavailableError
|
||||
);
|
||||
}
|
||||
|
||||
export class LegacySequelizeApprovalRequestRepository
|
||||
implements ApprovalRequestRepository
|
||||
{
|
||||
private readonly requests: ModelStatic<ApprovalRequestInstance>;
|
||||
private readonly dispatches: ModelStatic<ApprovedActionDispatchInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Approval request repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.requests = defineApprovalRequestModel(database);
|
||||
this.dispatches = defineApprovedActionDispatchModel(database);
|
||||
}
|
||||
|
||||
async findById(id: string): Promise<Readonly<ApprovalRequestRecord> | null> {
|
||||
const row = await this.requests.findByPk(id, { raw: true });
|
||||
return row ? rowToRequest(row) : null;
|
||||
}
|
||||
|
||||
private async assertFence(
|
||||
projectId: string,
|
||||
subject: Readonly<ApprovalRequestRecord['requestedBy']>,
|
||||
requestedFence: Readonly<ProjectPolicyFence>,
|
||||
transaction: Transaction,
|
||||
): Promise<void> {
|
||||
const normalizedSubject = normalizePolicySubject(subject);
|
||||
const fence = normalizeApprovalPolicyFence(requestedFence);
|
||||
const rows = await this.database.query<PolicyFenceRow>(
|
||||
`SELECT project.version AS project_version,
|
||||
(SELECT MAX(binding.version)
|
||||
FROM "${PROJECT_ROLE_BINDING_TABLE}" AS binding
|
||||
WHERE binding.project_id = project.id
|
||||
AND binding.subject_type = :subjectType
|
||||
AND binding.subject_id = :subjectId) AS binding_version
|
||||
FROM "${PROJECT_TABLE}" AS project
|
||||
WHERE project.id = :projectId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
projectId,
|
||||
subjectType: normalizedSubject.type,
|
||||
subjectId: normalizedSubject.id,
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (rows.length !== 1) throw new ApprovalPolicyFenceConflictError();
|
||||
const currentProjectVersion = Number(rows[0].project_version);
|
||||
const currentBindingVersion =
|
||||
rows[0].binding_version === null ? null : Number(rows[0].binding_version);
|
||||
if (
|
||||
currentProjectVersion !== fence.projectVersion ||
|
||||
currentBindingVersion !== fence.bindingVersion
|
||||
) {
|
||||
throw new ApprovalPolicyFenceConflictError();
|
||||
}
|
||||
}
|
||||
|
||||
private async findDecisionReplay(
|
||||
decisionId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<Readonly<ApprovalRequestRecord> | null> {
|
||||
const row = await this.requests.findOne({
|
||||
where: { decisionId },
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
return row ? rowToRequest(row) : null;
|
||||
}
|
||||
|
||||
private async findConsumptionReplay(
|
||||
consumptionId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<{
|
||||
request: Readonly<ApprovalRequestRecord>;
|
||||
dispatch: Readonly<ApprovedActionDispatchRecord>;
|
||||
} | null> {
|
||||
const row = await this.requests.findOne({
|
||||
where: { consumptionId },
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!row) return null;
|
||||
const request = rowToRequest(row);
|
||||
if (!request.dispatchId) throw new ApprovalUnavailableError();
|
||||
const dispatchRow = await this.dispatches.findByPk(request.dispatchId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!dispatchRow) throw new ApprovalUnavailableError();
|
||||
const executionRows = await this.database.query<{
|
||||
dispatch_id: string;
|
||||
project_id: string;
|
||||
}>(
|
||||
`SELECT dispatch_id, project_id
|
||||
FROM "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
WHERE dispatch_id = :dispatchId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { dispatchId: request.dispatchId },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (
|
||||
executionRows.length !== 1 ||
|
||||
executionRows[0].project_id !== request.projectId
|
||||
) {
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
return { request, dispatch: rowToDispatch(dispatchRow) };
|
||||
}
|
||||
|
||||
async create(
|
||||
command: CreateApprovalRequestCommand,
|
||||
): Promise<CreateApprovalRequestResult> {
|
||||
const request = normalizeApprovalRequestRecord(command.request);
|
||||
const fence = normalizeApprovalPolicyFence(command.authorizationFence);
|
||||
if (request.state !== 'pending' || request.version !== 1) {
|
||||
throw new ApprovalRequestStateConflictError();
|
||||
}
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const existing = await this.requests.findByPk(request.id, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (existing) {
|
||||
const previous = rowToRequest(existing);
|
||||
if (!sameRequestCreation(previous, request)) {
|
||||
throw new ApprovalMutationConflictError();
|
||||
}
|
||||
return { status: 'existing', request: previous };
|
||||
}
|
||||
await this.assertFence(
|
||||
request.projectId,
|
||||
request.requestedBy,
|
||||
fence,
|
||||
transaction,
|
||||
);
|
||||
await this.requests.create(requestToRow(request), { transaction });
|
||||
return { status: 'created', request };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (isApprovalError(error)) throw error;
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
|
||||
async decide(
|
||||
command: DecideApprovalRequestCommand,
|
||||
): Promise<DecideApprovalRequestResult> {
|
||||
const decidedBy = normalizePolicySubject(command.decidedBy);
|
||||
const fence = normalizeApprovalPolicyFence(command.authorizationFence);
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const replay = await this.findDecisionReplay(
|
||||
command.decisionId,
|
||||
transaction,
|
||||
);
|
||||
if (replay) {
|
||||
if (
|
||||
replay.id !== command.requestId ||
|
||||
command.expectedVersion !== 1 ||
|
||||
replay.decisionId !== command.decisionId ||
|
||||
replay.decision !== command.decision ||
|
||||
replay.decisionReasonCode !== command.reasonCode ||
|
||||
!replay.decidedBy ||
|
||||
!sameApprovalSubject(replay.decidedBy, decidedBy) ||
|
||||
replay.decidedAtMs !== command.decidedAtMs
|
||||
) {
|
||||
throw new ApprovalMutationConflictError();
|
||||
}
|
||||
return { status: 'existing', request: replay };
|
||||
}
|
||||
const row = await this.requests.findByPk(command.requestId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!row) throw new ApprovalRequestNotFoundError();
|
||||
const current = rowToRequest(row);
|
||||
if (command.decidedAtMs >= current.expiresAtMs) {
|
||||
throw new ApprovalRequestExpiredError();
|
||||
}
|
||||
if (current.version !== command.expectedVersion) {
|
||||
throw new ApprovalRequestVersionConflictError();
|
||||
}
|
||||
if (current.state !== 'pending') {
|
||||
throw new ApprovalRequestStateConflictError();
|
||||
}
|
||||
await this.assertFence(
|
||||
current.projectId,
|
||||
decidedBy,
|
||||
fence,
|
||||
transaction,
|
||||
);
|
||||
const decided = normalizeApprovalRequestRecord({
|
||||
...current,
|
||||
version: 2,
|
||||
state: command.decision,
|
||||
decisionId: command.decisionId,
|
||||
decision: command.decision,
|
||||
decisionReasonCode: command.reasonCode,
|
||||
decidedBy,
|
||||
decidedAtMs: command.decidedAtMs,
|
||||
});
|
||||
const [updated] = await this.requests.update(
|
||||
requestToRow(decided),
|
||||
{
|
||||
where: {
|
||||
id: current.id,
|
||||
version: command.expectedVersion,
|
||||
state: 'pending',
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (updated !== 1) {
|
||||
throw new ApprovalRequestVersionConflictError();
|
||||
}
|
||||
return { status: 'decided', request: decided };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (isApprovalError(error)) throw error;
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
|
||||
async consume(
|
||||
command: ConsumeApprovalRequestCommand,
|
||||
): Promise<ConsumeApprovalRequestResult> {
|
||||
const action = normalizeApprovalActionBinding(command.action);
|
||||
const requestedBy = normalizePolicySubject(command.requestedBy);
|
||||
const consumedBy = normalizePolicySubject(command.consumedBy);
|
||||
const fence = normalizeApprovalPolicyFence(command.authorizationFence);
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const replay = await this.findConsumptionReplay(
|
||||
command.consumptionId,
|
||||
transaction,
|
||||
);
|
||||
if (replay) {
|
||||
const expectedDispatch = normalizeApprovedActionDispatchRecord({
|
||||
id: command.dispatchId,
|
||||
approvalRequestId: command.requestId,
|
||||
approvalRequestVersion: 3,
|
||||
projectId: replay.request.projectId,
|
||||
state: 'pending',
|
||||
action,
|
||||
requestedBy,
|
||||
consumedBy,
|
||||
createdAtMs: command.consumedAtMs,
|
||||
});
|
||||
if (
|
||||
command.expectedVersion !== 2 ||
|
||||
replay.request.id !== command.requestId ||
|
||||
replay.request.consumptionId !== command.consumptionId ||
|
||||
!sameDispatch(replay.dispatch, expectedDispatch)
|
||||
) {
|
||||
throw new ApprovalMutationConflictError();
|
||||
}
|
||||
return {
|
||||
status: 'existing',
|
||||
request: replay.request,
|
||||
dispatch: replay.dispatch,
|
||||
};
|
||||
}
|
||||
const row = await this.requests.findByPk(command.requestId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!row) throw new ApprovalRequestNotFoundError();
|
||||
const current = rowToRequest(row);
|
||||
if (command.consumedAtMs >= current.expiresAtMs) {
|
||||
throw new ApprovalRequestExpiredError();
|
||||
}
|
||||
if (current.version !== command.expectedVersion) {
|
||||
throw new ApprovalRequestVersionConflictError();
|
||||
}
|
||||
if (current.state !== 'approved') {
|
||||
throw new ApprovalRequestStateConflictError();
|
||||
}
|
||||
if (
|
||||
!sameApprovalAction(current.action, action) ||
|
||||
!sameApprovalSubject(current.requestedBy, requestedBy)
|
||||
) {
|
||||
throw new ApprovalMutationConflictError();
|
||||
}
|
||||
await this.assertFence(
|
||||
current.projectId,
|
||||
requestedBy,
|
||||
fence,
|
||||
transaction,
|
||||
);
|
||||
const dispatch = normalizeApprovedActionDispatchRecord({
|
||||
id: command.dispatchId,
|
||||
approvalRequestId: current.id,
|
||||
approvalRequestVersion: 3,
|
||||
projectId: current.projectId,
|
||||
state: 'pending',
|
||||
action,
|
||||
requestedBy,
|
||||
consumedBy,
|
||||
createdAtMs: command.consumedAtMs,
|
||||
});
|
||||
const dispatchCollision = await this.dispatches.findByPk(
|
||||
dispatch.id,
|
||||
{ raw: true, transaction },
|
||||
);
|
||||
if (dispatchCollision) throw new ApprovalMutationConflictError();
|
||||
await this.dispatches.create(dispatchToRow(dispatch), {
|
||||
transaction,
|
||||
});
|
||||
await this.database.query(
|
||||
`INSERT INTO "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
(dispatch_id, project_id, status, version, attempt_count,
|
||||
max_attempts, eligible_at_ms, next_attempt_at_ms,
|
||||
lease_owner, lease_token, lease_expires_at_ms, started_at_ms,
|
||||
last_result_code, completed_at_ms, created_at_ms, updated_at_ms)
|
||||
VALUES
|
||||
(:dispatchId, :projectId, 'pending', 0, 0, :maxAttempts,
|
||||
:createdAtMs, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
|
||||
:createdAtMs, :createdAtMs)`,
|
||||
{
|
||||
replacements: {
|
||||
dispatchId: dispatch.id,
|
||||
projectId: dispatch.projectId,
|
||||
maxAttempts: DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS,
|
||||
createdAtMs: dispatch.createdAtMs,
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
const consumed = normalizeApprovalRequestRecord({
|
||||
...current,
|
||||
version: 3,
|
||||
state: 'consumed',
|
||||
consumptionId: command.consumptionId,
|
||||
dispatchId: command.dispatchId,
|
||||
consumedBy,
|
||||
consumedAtMs: command.consumedAtMs,
|
||||
});
|
||||
const [updated] = await this.requests.update(
|
||||
requestToRow(consumed),
|
||||
{
|
||||
where: {
|
||||
id: current.id,
|
||||
version: command.expectedVersion,
|
||||
state: 'approved',
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (updated !== 1) {
|
||||
throw new ApprovalRequestVersionConflictError();
|
||||
}
|
||||
return { status: 'consumed', request: consumed, dispatch };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (isApprovalError(error)) throw error;
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
throw new ApprovalUnavailableError();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,427 @@
|
||||
import {
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
type Transaction as SequelizeTransaction,
|
||||
} from 'sequelize';
|
||||
import { APPROVED_RUN_ACTION_RECEIPT_TABLE } from '../../../migrations/0023-approved-run-action-receipts';
|
||||
import { APPROVED_ACTION_DISPATCH_EXECUTION_TABLE } from '../../../migrations/0021-approved-action-dispatch-executions';
|
||||
import {
|
||||
APPROVED_RUN_ACTION_TYPE,
|
||||
APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
ApprovedRunActionBindingConflictError,
|
||||
ApprovedRunActionRepositoryError,
|
||||
InvalidApprovedRunActionError,
|
||||
digestApprovedRunCreationPlan,
|
||||
digestApprovedRunCreationReceipt,
|
||||
normalizeApprovedRunCreationPlan,
|
||||
normalizeApprovedRunCreationReceipt,
|
||||
type ApprovedRunCreationReceipt,
|
||||
} from '../../domain/approvedRunAction';
|
||||
import {
|
||||
normalizeApprovedActionDispatchExecutionRecord,
|
||||
type ApprovedActionDispatchExecutionSnapshot,
|
||||
} from '../../domain/approvedActionDispatchExecution';
|
||||
import { normalizeApprovedActionDispatchRecord } from '../../domain/approvalRequest';
|
||||
import { DuplicateIdempotencyKeyError } from '../../domain/repositoryErrors';
|
||||
import type { RunRecord } from '../../domain/run';
|
||||
import {
|
||||
PrimaryRunCreator,
|
||||
type PrimaryRunIdFactory,
|
||||
} from '../../application/primaryRunCreator';
|
||||
import type {
|
||||
ApprovedRunActionRepository,
|
||||
ApprovedRunReference,
|
||||
CreateApprovedRunCommand,
|
||||
} from '../../ports/approvedRunActionRepository';
|
||||
import type { RunRepositoryTransaction } from '../../ports/runRepository';
|
||||
import {
|
||||
LegacySequelizeRunRepository,
|
||||
LegacySequelizeRunTransaction,
|
||||
} from './runRepository';
|
||||
|
||||
interface ApprovedRunReceiptRow {
|
||||
schema_version: number;
|
||||
dispatch_id: string;
|
||||
approval_request_id: string;
|
||||
project_id: string;
|
||||
action_type: string;
|
||||
action_digest: string;
|
||||
execution_attempt: number;
|
||||
execution_version: number;
|
||||
started_at_ms: number;
|
||||
idempotency_key: string;
|
||||
outcome: string;
|
||||
result_code: string;
|
||||
resource_type: string;
|
||||
resource_id: string;
|
||||
finished_at_ms: number;
|
||||
evidence_digest: string;
|
||||
created_at_ms: number;
|
||||
}
|
||||
|
||||
interface ReceiptBinding {
|
||||
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>;
|
||||
clock: () => number;
|
||||
}
|
||||
|
||||
interface ExecutionFenceRow {
|
||||
project_id: string;
|
||||
status: string;
|
||||
version: number;
|
||||
attempt_count: number;
|
||||
lease_owner: string | null;
|
||||
lease_token: string | null;
|
||||
started_at_ms: number | null;
|
||||
}
|
||||
|
||||
function rowToReceipt(
|
||||
row: ApprovedRunReceiptRow,
|
||||
): Readonly<ApprovedRunCreationReceipt> {
|
||||
return normalizeApprovedRunCreationReceipt({
|
||||
schemaVersion: row.schema_version as 1,
|
||||
dispatchId: row.dispatch_id,
|
||||
approvalRequestId: row.approval_request_id,
|
||||
projectId: row.project_id,
|
||||
actionType: row.action_type as typeof APPROVED_RUN_ACTION_TYPE,
|
||||
actionDigest: row.action_digest,
|
||||
executionAttempt: row.execution_attempt,
|
||||
executionVersion: row.execution_version,
|
||||
startedAtMs: row.started_at_ms,
|
||||
idempotencyKey: row.idempotency_key,
|
||||
outcome: row.outcome as 'succeeded',
|
||||
resultCode: row.result_code as typeof APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
resourceType: row.resource_type as 'run',
|
||||
resourceId: row.resource_id,
|
||||
finishedAtMs: row.finished_at_ms,
|
||||
evidenceDigest: row.evidence_digest,
|
||||
createdAtMs: row.created_at_ms,
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeSnapshot(
|
||||
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
|
||||
): Readonly<ApprovedActionDispatchExecutionSnapshot> {
|
||||
if (!snapshot || typeof snapshot !== 'object' || Array.isArray(snapshot)) {
|
||||
throw new InvalidApprovedRunActionError('execution snapshot is invalid');
|
||||
}
|
||||
const dispatch = normalizeApprovedActionDispatchRecord(snapshot.dispatch);
|
||||
const execution = normalizeApprovedActionDispatchExecutionRecord(
|
||||
snapshot.execution,
|
||||
);
|
||||
if (
|
||||
execution.dispatchId !== dispatch.id ||
|
||||
execution.projectId !== dispatch.projectId ||
|
||||
execution.status !== 'executing' ||
|
||||
execution.startedAtMs === null
|
||||
) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
return Object.freeze({ dispatch, execution });
|
||||
}
|
||||
|
||||
function receiptMatches(
|
||||
receipt: Readonly<ApprovedRunCreationReceipt>,
|
||||
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
|
||||
): boolean {
|
||||
return (
|
||||
receipt.dispatchId === snapshot.dispatch.id &&
|
||||
receipt.approvalRequestId === snapshot.dispatch.approvalRequestId &&
|
||||
receipt.projectId === snapshot.dispatch.projectId &&
|
||||
receipt.actionType === snapshot.dispatch.action.actionType &&
|
||||
receipt.actionDigest === snapshot.dispatch.action.actionDigest &&
|
||||
receipt.executionAttempt === snapshot.execution.attemptCount &&
|
||||
receipt.startedAtMs === snapshot.execution.startedAtMs &&
|
||||
receipt.idempotencyKey === snapshot.dispatch.id
|
||||
);
|
||||
}
|
||||
|
||||
class AtomicApprovedRunRepository extends LegacySequelizeRunRepository {
|
||||
constructor(
|
||||
private readonly approvedDatabase: Sequelize,
|
||||
private readonly binding: Readonly<ReceiptBinding>,
|
||||
) {
|
||||
super(approvedDatabase);
|
||||
}
|
||||
|
||||
override async transaction<T>(
|
||||
work: (transaction: RunRepositoryTransaction) => Promise<T>,
|
||||
): Promise<T> {
|
||||
return this.approvedDatabase.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const executionVersion = await this.requireCurrentExecutionFence(
|
||||
transaction,
|
||||
);
|
||||
const result = await work(
|
||||
new LegacySequelizeRunTransaction(this.models, transaction),
|
||||
);
|
||||
const run = this.requireCreatedRun(result);
|
||||
await this.insertReceipt(run, executionVersion, transaction);
|
||||
return result;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async requireCurrentExecutionFence(
|
||||
transaction: SequelizeTransaction,
|
||||
): Promise<number> {
|
||||
const { dispatch, execution } = this.binding.snapshot;
|
||||
const rows = await this.approvedDatabase.query<ExecutionFenceRow>(
|
||||
`SELECT project_id, status, version, attempt_count, lease_owner,
|
||||
lease_token, started_at_ms
|
||||
FROM "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
|
||||
WHERE dispatch_id = :dispatchId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { dispatchId: dispatch.id },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
const current = rows[0];
|
||||
if (
|
||||
rows.length !== 1 ||
|
||||
current.project_id !== dispatch.projectId ||
|
||||
current.status !== 'executing' ||
|
||||
current.version < execution.version ||
|
||||
current.attempt_count !== execution.attemptCount ||
|
||||
current.lease_owner !== execution.leaseOwner ||
|
||||
current.lease_token !== execution.leaseToken ||
|
||||
current.started_at_ms !== execution.startedAtMs
|
||||
) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
return current.version;
|
||||
}
|
||||
|
||||
private requireCreatedRun(value: unknown): Readonly<RunRecord> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!('id' in value) ||
|
||||
typeof value.id !== 'string' ||
|
||||
!('projectId' in value) ||
|
||||
value.projectId !== this.binding.snapshot.dispatch.projectId ||
|
||||
!('idempotencyKey' in value) ||
|
||||
value.idempotencyKey !== this.binding.snapshot.dispatch.id ||
|
||||
!('requestId' in value) ||
|
||||
value.requestId !== this.binding.snapshot.dispatch.approvalRequestId ||
|
||||
!('status' in value) ||
|
||||
value.status !== 'queued'
|
||||
) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
return value as Readonly<RunRecord>;
|
||||
}
|
||||
|
||||
private async insertReceipt(
|
||||
run: Readonly<RunRecord>,
|
||||
executionVersion: number,
|
||||
transaction: SequelizeTransaction,
|
||||
): Promise<void> {
|
||||
const { dispatch, execution } = this.binding.snapshot;
|
||||
const finishedAtMs = this.nowAtOrAfter(execution.startedAtMs!);
|
||||
const unsigned: Omit<ApprovedRunCreationReceipt, 'evidenceDigest'> = {
|
||||
schemaVersion: 1,
|
||||
dispatchId: dispatch.id,
|
||||
approvalRequestId: dispatch.approvalRequestId,
|
||||
projectId: dispatch.projectId,
|
||||
actionType: APPROVED_RUN_ACTION_TYPE,
|
||||
actionDigest: dispatch.action.actionDigest,
|
||||
executionAttempt: execution.attemptCount,
|
||||
executionVersion,
|
||||
startedAtMs: execution.startedAtMs!,
|
||||
idempotencyKey: dispatch.id,
|
||||
outcome: 'succeeded',
|
||||
resultCode: APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
resourceType: 'run',
|
||||
resourceId: run.id,
|
||||
finishedAtMs,
|
||||
createdAtMs: finishedAtMs,
|
||||
};
|
||||
const receipt = normalizeApprovedRunCreationReceipt({
|
||||
...unsigned,
|
||||
evidenceDigest: digestApprovedRunCreationReceipt(unsigned),
|
||||
});
|
||||
await this.approvedDatabase.query(
|
||||
`INSERT INTO "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
(dispatch_id, approval_request_id, project_id, schema_version,
|
||||
action_type, action_digest, execution_attempt, execution_version,
|
||||
started_at_ms, idempotency_key, outcome, result_code, resource_type,
|
||||
resource_id, finished_at_ms, evidence_digest, created_at_ms)
|
||||
VALUES
|
||||
(:dispatchId, :approvalRequestId, :projectId, :schemaVersion,
|
||||
:actionType, :actionDigest, :executionAttempt, :executionVersion,
|
||||
:startedAtMs, :idempotencyKey, :outcome, :resultCode, :resourceType,
|
||||
:resourceId, :finishedAtMs, :evidenceDigest, :createdAtMs)`,
|
||||
{ replacements: receipt, transaction },
|
||||
);
|
||||
}
|
||||
|
||||
private nowAtOrAfter(minimum: number): number {
|
||||
const nowMs = this.binding.clock();
|
||||
if (!Number.isSafeInteger(nowMs) || nowMs < minimum) {
|
||||
throw new RangeError('clock must not precede the action start barrier');
|
||||
}
|
||||
return nowMs;
|
||||
}
|
||||
}
|
||||
|
||||
export interface LegacySequelizeApprovedRunActionRepositoryOptions {
|
||||
clock?: () => number;
|
||||
createId?: PrimaryRunIdFactory;
|
||||
}
|
||||
|
||||
export class LegacySequelizeApprovedRunActionRepository
|
||||
implements ApprovedRunActionRepository
|
||||
{
|
||||
private readonly runs: LegacySequelizeRunRepository;
|
||||
private readonly clock: () => number;
|
||||
private readonly createId?: PrimaryRunIdFactory;
|
||||
|
||||
constructor(
|
||||
private readonly database: Sequelize,
|
||||
options: LegacySequelizeApprovedRunActionRepositoryOptions = {},
|
||||
) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Approved Run action repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.runs = new LegacySequelizeRunRepository(database);
|
||||
this.clock = options.clock ?? Date.now;
|
||||
this.createId = options.createId;
|
||||
}
|
||||
|
||||
async create(
|
||||
command: Readonly<CreateApprovedRunCommand>,
|
||||
): Promise<Readonly<ApprovedRunReference>> {
|
||||
try {
|
||||
const snapshot = normalizeSnapshot(command.snapshot);
|
||||
const plan = normalizeApprovedRunCreationPlan(command.plan);
|
||||
this.assertPlanBinding(snapshot, plan);
|
||||
const replay = await this.findReplay(snapshot);
|
||||
if (replay) return replay;
|
||||
|
||||
const atomic = new AtomicApprovedRunRepository(this.database, {
|
||||
snapshot,
|
||||
clock: this.clock,
|
||||
});
|
||||
const creator = new PrimaryRunCreator(atomic, this.createId);
|
||||
try {
|
||||
return await creator.create(
|
||||
{
|
||||
projectId: plan.projectId,
|
||||
taskId: plan.taskId,
|
||||
taskRevision: plan.taskRevision,
|
||||
...(plan.taskName === undefined ? {} : { taskName: plan.taskName }),
|
||||
...(plan.taskSnapshotRef === undefined
|
||||
? {}
|
||||
: { taskSnapshotRef: plan.taskSnapshotRef }),
|
||||
triggerType: 'approved_action',
|
||||
executionOrigin: 'system',
|
||||
triggeredBy: `approved-action:${snapshot.dispatch.id}`,
|
||||
requestId: snapshot.dispatch.approvalRequestId,
|
||||
priority: plan.priority,
|
||||
idempotencyKey: snapshot.dispatch.id,
|
||||
...(plan.inputRef === undefined ? {} : { inputRef: plan.inputRef }),
|
||||
acceptedAtMs: snapshot.execution.startedAtMs!,
|
||||
actor: { type: 'system', id: 'approved-action-dispatcher' },
|
||||
},
|
||||
plan.executorType,
|
||||
);
|
||||
} catch (error) {
|
||||
if (!(error instanceof DuplicateIdempotencyKeyError)) throw error;
|
||||
const raced = await this.findReplay(snapshot);
|
||||
if (raced) return raced;
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof ApprovedRunActionBindingConflictError ||
|
||||
error instanceof InvalidApprovedRunActionError ||
|
||||
error instanceof RangeError ||
|
||||
error instanceof TypeError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new ApprovedRunActionRepositoryError();
|
||||
}
|
||||
}
|
||||
|
||||
private assertPlanBinding(
|
||||
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
|
||||
plan: Readonly<ReturnType<typeof normalizeApprovedRunCreationPlan>>,
|
||||
): void {
|
||||
if (
|
||||
snapshot.dispatch.action.actionType !== APPROVED_RUN_ACTION_TYPE ||
|
||||
snapshot.dispatch.action.actionRef !== plan.actionRef ||
|
||||
snapshot.dispatch.projectId !== plan.projectId ||
|
||||
snapshot.dispatch.action.actionDigest !==
|
||||
digestApprovedRunCreationPlan(plan)
|
||||
) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
}
|
||||
|
||||
private async findReplay(
|
||||
snapshot: Readonly<ApprovedActionDispatchExecutionSnapshot>,
|
||||
): Promise<Readonly<ApprovedRunReference> | null> {
|
||||
const rows = await this.database.query<ApprovedRunReceiptRow>(
|
||||
`SELECT schema_version, dispatch_id, approval_request_id, project_id,
|
||||
action_type, action_digest, execution_attempt, execution_version,
|
||||
started_at_ms, idempotency_key, outcome, result_code,
|
||||
resource_type, resource_id, finished_at_ms, evidence_digest,
|
||||
created_at_ms
|
||||
FROM "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
WHERE dispatch_id = :dispatchId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { dispatchId: snapshot.dispatch.id },
|
||||
},
|
||||
);
|
||||
if (rows.length > 1) throw new ApprovedRunActionBindingConflictError();
|
||||
if (rows.length === 0) {
|
||||
const collisions = await this.database.query<{ id: string }>(
|
||||
`SELECT id FROM "Runs"
|
||||
WHERE project_id = :projectId AND idempotency_key = :idempotencyKey
|
||||
LIMIT 1`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
projectId: snapshot.dispatch.projectId,
|
||||
idempotencyKey: snapshot.dispatch.id,
|
||||
},
|
||||
},
|
||||
);
|
||||
if (collisions.length > 0) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const receipt = rowToReceipt(rows[0]);
|
||||
if (!receiptMatches(receipt, snapshot)) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
const run = await this.runs.findRunById(receipt.resourceId);
|
||||
const attempt = await this.runs.findLatestAttemptByRunId(
|
||||
receipt.resourceId,
|
||||
);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
run.projectId !== receipt.projectId ||
|
||||
run.idempotencyKey !== receipt.idempotencyKey ||
|
||||
run.requestId !== receipt.approvalRequestId ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
run.executionOrigin !== 'system' ||
|
||||
run.triggerType !== 'approved_action'
|
||||
) {
|
||||
throw new ApprovedRunActionBindingConflictError();
|
||||
}
|
||||
return Object.freeze({ run, attempt });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import { APPROVED_RUN_ACTION_RECEIPT_TABLE } from '../../../migrations/0023-approved-run-action-receipts';
|
||||
import {
|
||||
APPROVED_RUN_ACTION_TYPE,
|
||||
APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
InvalidApprovedRunActionError,
|
||||
normalizeApprovedRunCreationReceipt,
|
||||
type ApprovedRunCreationReceipt,
|
||||
} from '../../domain/approvedRunAction';
|
||||
import type {
|
||||
ApprovedActionRecoveryEvidence,
|
||||
ApprovedActionRecoveryEvidenceContext,
|
||||
ApprovedActionRecoveryEvidenceProvider,
|
||||
} from '../../ports/approvedActionRecoveryEvidenceProvider';
|
||||
import { LegacySequelizeRunRepository } from './runRepository';
|
||||
|
||||
interface ReceiptRow {
|
||||
schema_version: number;
|
||||
dispatch_id: string;
|
||||
approval_request_id: string;
|
||||
project_id: string;
|
||||
action_type: string;
|
||||
action_digest: string;
|
||||
execution_attempt: number;
|
||||
execution_version: number;
|
||||
started_at_ms: number;
|
||||
idempotency_key: string;
|
||||
outcome: string;
|
||||
result_code: string;
|
||||
resource_type: string;
|
||||
resource_id: string;
|
||||
finished_at_ms: number;
|
||||
evidence_digest: string;
|
||||
created_at_ms: number;
|
||||
}
|
||||
|
||||
function normalizeRow(row: ReceiptRow): Readonly<ApprovedRunCreationReceipt> {
|
||||
return normalizeApprovedRunCreationReceipt({
|
||||
schemaVersion: row.schema_version as 1,
|
||||
dispatchId: row.dispatch_id,
|
||||
approvalRequestId: row.approval_request_id,
|
||||
projectId: row.project_id,
|
||||
actionType: row.action_type as typeof APPROVED_RUN_ACTION_TYPE,
|
||||
actionDigest: row.action_digest,
|
||||
executionAttempt: row.execution_attempt,
|
||||
executionVersion: row.execution_version,
|
||||
startedAtMs: row.started_at_ms,
|
||||
idempotencyKey: row.idempotency_key,
|
||||
outcome: row.outcome as 'succeeded',
|
||||
resultCode: row.result_code as typeof APPROVED_RUN_RECEIPT_RESULT_CODE,
|
||||
resourceType: row.resource_type as 'run',
|
||||
resourceId: row.resource_id,
|
||||
finishedAtMs: row.finished_at_ms,
|
||||
evidenceDigest: row.evidence_digest,
|
||||
createdAtMs: row.created_at_ms,
|
||||
});
|
||||
}
|
||||
|
||||
const CONFLICT: ApprovedActionRecoveryEvidence = Object.freeze({
|
||||
finding: 'conflict',
|
||||
resultCode: 'approved_run_receipt_conflict',
|
||||
});
|
||||
|
||||
export class LegacySequelizeApprovedRunRecoveryEvidenceProvider
|
||||
implements ApprovedActionRecoveryEvidenceProvider
|
||||
{
|
||||
readonly actionType = APPROVED_RUN_ACTION_TYPE;
|
||||
readonly capability = 'automatic' as const;
|
||||
private readonly runs: LegacySequelizeRunRepository;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Approved Run recovery provider is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.runs = new LegacySequelizeRunRepository(database);
|
||||
}
|
||||
|
||||
async inspect(
|
||||
context: Readonly<ApprovedActionRecoveryEvidenceContext>,
|
||||
): Promise<ApprovedActionRecoveryEvidence> {
|
||||
const snapshot = context.snapshot;
|
||||
const dispatch = snapshot.action.dispatch;
|
||||
const execution = snapshot.action.execution;
|
||||
if (
|
||||
dispatch.action.actionType !== this.actionType ||
|
||||
context.idempotencyKey !== dispatch.id ||
|
||||
execution.dispatchId !== dispatch.id ||
|
||||
execution.projectId !== dispatch.projectId ||
|
||||
execution.startedAtMs === null
|
||||
) {
|
||||
return CONFLICT;
|
||||
}
|
||||
const rows = await this.database.query<ReceiptRow>(
|
||||
`SELECT schema_version, dispatch_id, approval_request_id, project_id,
|
||||
action_type, action_digest, execution_attempt, execution_version,
|
||||
started_at_ms, idempotency_key, outcome, result_code,
|
||||
resource_type, resource_id, finished_at_ms, evidence_digest,
|
||||
created_at_ms
|
||||
FROM "${APPROVED_RUN_ACTION_RECEIPT_TABLE}"
|
||||
WHERE dispatch_id = :dispatchId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { dispatchId: dispatch.id },
|
||||
},
|
||||
);
|
||||
if (rows.length > 1) return CONFLICT;
|
||||
if (rows.length === 0) {
|
||||
const collisions = await this.database.query<{ id: string }>(
|
||||
`SELECT id FROM "Runs"
|
||||
WHERE project_id = :projectId AND idempotency_key = :idempotencyKey
|
||||
LIMIT 1`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
projectId: dispatch.projectId,
|
||||
idempotencyKey: dispatch.id,
|
||||
},
|
||||
},
|
||||
);
|
||||
return collisions.length === 0
|
||||
? {
|
||||
finding: 'missing',
|
||||
resultCode: 'approved_run_receipt_missing',
|
||||
}
|
||||
: CONFLICT;
|
||||
}
|
||||
|
||||
let receipt: Readonly<ApprovedRunCreationReceipt>;
|
||||
try {
|
||||
receipt = normalizeRow(rows[0]);
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidApprovedRunActionError) return CONFLICT;
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
receipt.dispatchId !== dispatch.id ||
|
||||
receipt.approvalRequestId !== dispatch.approvalRequestId ||
|
||||
receipt.projectId !== dispatch.projectId ||
|
||||
receipt.actionType !== dispatch.action.actionType ||
|
||||
receipt.actionDigest !== dispatch.action.actionDigest ||
|
||||
receipt.executionAttempt !== execution.attemptCount ||
|
||||
receipt.executionVersion > execution.version ||
|
||||
receipt.startedAtMs !== execution.startedAtMs ||
|
||||
receipt.idempotencyKey !== context.idempotencyKey
|
||||
) {
|
||||
return CONFLICT;
|
||||
}
|
||||
const run = await this.runs.findRunById(receipt.resourceId);
|
||||
const attempt = await this.runs.findLatestAttemptByRunId(
|
||||
receipt.resourceId,
|
||||
);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
run.projectId !== receipt.projectId ||
|
||||
run.idempotencyKey !== receipt.idempotencyKey ||
|
||||
run.requestId !== receipt.approvalRequestId ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
run.executionOrigin !== 'system' ||
|
||||
run.triggerType !== 'approved_action'
|
||||
) {
|
||||
return CONFLICT;
|
||||
}
|
||||
return {
|
||||
finding: 'verified_succeeded',
|
||||
resultCode: 'approved_run_receipt_verified',
|
||||
evidenceDigest: receipt.evidenceDigest,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,735 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
RUN_EVENT_TABLE,
|
||||
RUN_TABLE,
|
||||
RUN_ATTEMPT_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { RUN_CANCELLATION_DISPATCH_TABLE } from '../../../migrations/0005-run-cancellation-dispatch';
|
||||
import {
|
||||
CANCELLATION_DISPATCH_RESULTS,
|
||||
CANCELLATION_DISPATCH_STATUSES,
|
||||
type CancellationDispatchRecord,
|
||||
type CancellationDispatchResult,
|
||||
type CancellationDispatchStatus,
|
||||
} from '../../domain/cancellationDispatch';
|
||||
import {
|
||||
CancellationDispatchBindingConflictError,
|
||||
CancellationDispatchFenceRejectedError,
|
||||
CancellationDispatchRepositoryError,
|
||||
InvalidCancellationDispatchCommandError,
|
||||
} from '../../domain/cancellationDispatchErrors';
|
||||
import type { RunEventRecord, RunStatus } from '../../domain/run';
|
||||
import type {
|
||||
CancellationDispatchRepository,
|
||||
ClaimCancellationDispatchCommand,
|
||||
ClaimCancellationDispatchResult,
|
||||
RecordCancellationDispatchResult,
|
||||
RecordCancellationDispatchResultCommand,
|
||||
} from '../../ports/cancellationDispatchRepository';
|
||||
|
||||
const ACTIVE_RUN_STATUSES: readonly RunStatus[] = [
|
||||
'created',
|
||||
'queued',
|
||||
'dispatching',
|
||||
'running',
|
||||
'waiting_approval',
|
||||
'retry_wait',
|
||||
'lost',
|
||||
];
|
||||
const ACTIVE_ATTEMPT_STATUSES = ['claimed', 'starting', 'running'] as const;
|
||||
const RETRYABLE_RESULTS: readonly CancellationDispatchResult[] = [
|
||||
'controller_missing',
|
||||
'handle_missing',
|
||||
'dispatch_error',
|
||||
];
|
||||
const BLOCKING_RESULTS: readonly CancellationDispatchResult[] = [
|
||||
'identity_mismatch',
|
||||
'pid_mismatch',
|
||||
'unsupported',
|
||||
'invalid',
|
||||
];
|
||||
|
||||
interface CancellationDispatchRow {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
status: string;
|
||||
version: number;
|
||||
dispatchCount: number;
|
||||
nextAttemptAtMs: number | null;
|
||||
leaseOwner: string | null;
|
||||
leaseToken: string | null;
|
||||
leaseExpiresAtMs: number | null;
|
||||
lastResult: string | null;
|
||||
lastDispatchedAtMs: number | null;
|
||||
createdAtMs: number;
|
||||
updatedAtMs: number;
|
||||
}
|
||||
|
||||
interface CancellationDispatchRunRow {
|
||||
id: string;
|
||||
executionOwner: string;
|
||||
status: string;
|
||||
version: number;
|
||||
eventSequence: number;
|
||||
cancelRequestedAtMs: number | null;
|
||||
}
|
||||
|
||||
interface CancellationDispatchAttemptRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
status: string;
|
||||
}
|
||||
|
||||
interface CancellationDispatchEventRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
sequence: number;
|
||||
type: string;
|
||||
dedupeKey: string;
|
||||
actorType: string;
|
||||
actorId: string;
|
||||
attemptId: string;
|
||||
payload: Readonly<Record<string, unknown>>;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
interface CancellationDispatchInstance
|
||||
extends Model<CancellationDispatchRow, CancellationDispatchRow>,
|
||||
CancellationDispatchRow {}
|
||||
interface CancellationDispatchRunInstance
|
||||
extends Model<CancellationDispatchRunRow, CancellationDispatchRunRow>,
|
||||
CancellationDispatchRunRow {}
|
||||
interface CancellationDispatchAttemptInstance
|
||||
extends Model<CancellationDispatchAttemptRow, CancellationDispatchAttemptRow>,
|
||||
CancellationDispatchAttemptRow {}
|
||||
interface CancellationDispatchEventInstance
|
||||
extends Model<CancellationDispatchEventRow, CancellationDispatchEventRow>,
|
||||
CancellationDispatchEventRow {}
|
||||
|
||||
function defineDispatchModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationDispatchInstance> {
|
||||
return database.define<CancellationDispatchInstance>(
|
||||
'Ql3CancellationDispatch',
|
||||
{
|
||||
runId: { field: 'run_id', type: DataTypes.STRING(36), primaryKey: true },
|
||||
attemptId: {
|
||||
field: 'attempt_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
version: { type: DataTypes.INTEGER, allowNull: false },
|
||||
dispatchCount: {
|
||||
field: 'dispatch_count',
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
},
|
||||
nextAttemptAtMs: {
|
||||
field: 'next_attempt_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
leaseOwner: {
|
||||
field: 'lease_owner',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: true,
|
||||
},
|
||||
leaseToken: {
|
||||
field: 'lease_token',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: true,
|
||||
},
|
||||
leaseExpiresAtMs: {
|
||||
field: 'lease_expires_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
lastResult: {
|
||||
field: 'last_result',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: true,
|
||||
},
|
||||
lastDispatchedAtMs: {
|
||||
field: 'last_dispatched_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
updatedAtMs: {
|
||||
field: 'updated_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: RUN_CANCELLATION_DISPATCH_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function defineRunModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationDispatchRunInstance> {
|
||||
return database.define<CancellationDispatchRunInstance>(
|
||||
'Ql3CancellationDispatchRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
executionOwner: {
|
||||
field: 'execution_owner',
|
||||
type: DataTypes.STRING(16),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
version: { type: DataTypes.INTEGER, allowNull: false },
|
||||
eventSequence: {
|
||||
field: 'event_sequence',
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
},
|
||||
cancelRequestedAtMs: {
|
||||
field: 'cancel_requested_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function defineAttemptModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationDispatchAttemptInstance> {
|
||||
return database.define<CancellationDispatchAttemptInstance>(
|
||||
'Ql3CancellationDispatchAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
runId: { field: 'run_id', type: DataTypes.STRING(36), allowNull: false },
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
},
|
||||
{ tableName: RUN_ATTEMPT_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function defineEventModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationDispatchEventInstance> {
|
||||
return database.define<CancellationDispatchEventInstance>(
|
||||
'Ql3CancellationDispatchEvent',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
runId: { field: 'run_id', type: DataTypes.STRING(36), allowNull: false },
|
||||
sequence: { type: DataTypes.INTEGER, allowNull: false },
|
||||
type: { type: DataTypes.STRING(128), allowNull: false },
|
||||
dedupeKey: {
|
||||
field: 'dedupe_key',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
actorType: {
|
||||
field: 'actor_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
actorId: {
|
||||
field: 'actor_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
attemptId: {
|
||||
field: 'attempt_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
payload: { type: DataTypes.JSON, allowNull: false },
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_EVENT_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function assertId(name: string, value: string, maxLength = 36): void {
|
||||
if (!value || value.length > maxLength) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
`${name} must be between 1 and ${maxLength} characters`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertTimestamp(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
`${name} must be a non-negative safe integer`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertClaim(command: ClaimCancellationDispatchCommand): void {
|
||||
assertId('runId', command.runId);
|
||||
assertId('attemptId', command.attemptId);
|
||||
assertId('owner', command.owner, 128);
|
||||
assertId('leaseToken', command.leaseToken, 128);
|
||||
assertTimestamp('requestedAtMs', command.requestedAtMs);
|
||||
assertTimestamp('nowMs', command.nowMs);
|
||||
if (
|
||||
!Number.isSafeInteger(command.leaseDurationMs) ||
|
||||
command.leaseDurationMs < 1
|
||||
) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'leaseDurationMs must be a positive safe integer',
|
||||
);
|
||||
}
|
||||
if (!Number.isSafeInteger(command.nowMs + command.leaseDurationMs)) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'lease expiry exceeds the supported range',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertRecordResult(
|
||||
command: RecordCancellationDispatchResultCommand,
|
||||
): void {
|
||||
assertId('runId', command.runId);
|
||||
assertId('attemptId', command.attemptId);
|
||||
assertId('owner', command.owner, 128);
|
||||
assertId('leaseToken', command.leaseToken, 128);
|
||||
assertId('eventId', command.eventId);
|
||||
assertTimestamp('atMs', command.atMs);
|
||||
if (
|
||||
!Number.isSafeInteger(command.expectedVersion) ||
|
||||
command.expectedVersion < 1
|
||||
) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'expectedVersion must be a positive safe integer',
|
||||
);
|
||||
}
|
||||
if (!CANCELLATION_DISPATCH_RESULTS.includes(command.result)) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'result is not supported',
|
||||
);
|
||||
}
|
||||
if (RETRYABLE_RESULTS.includes(command.result)) {
|
||||
if (
|
||||
command.nextAttemptAtMs === undefined ||
|
||||
!Number.isSafeInteger(command.nextAttemptAtMs) ||
|
||||
command.nextAttemptAtMs <= command.atMs
|
||||
) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'retryable results require nextAttemptAtMs greater than atMs',
|
||||
);
|
||||
}
|
||||
} else if (command.nextAttemptAtMs !== undefined) {
|
||||
throw new InvalidCancellationDispatchCommandError(
|
||||
'terminal results must not include nextAttemptAtMs',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function rowToDispatch(
|
||||
row: CancellationDispatchRow,
|
||||
): CancellationDispatchRecord {
|
||||
if (
|
||||
!CANCELLATION_DISPATCH_STATUSES.includes(
|
||||
row.status as CancellationDispatchStatus,
|
||||
)
|
||||
) {
|
||||
throw new CancellationDispatchRepositoryError(
|
||||
new Error(`Unsupported cancellation dispatch status: ${row.status}`),
|
||||
);
|
||||
}
|
||||
if (
|
||||
row.lastResult !== null &&
|
||||
!CANCELLATION_DISPATCH_RESULTS.includes(
|
||||
row.lastResult as CancellationDispatchResult,
|
||||
)
|
||||
) {
|
||||
throw new CancellationDispatchRepositoryError(
|
||||
new Error(`Unsupported cancellation dispatch result: ${row.lastResult}`),
|
||||
);
|
||||
}
|
||||
for (const [name, value] of [
|
||||
['version', row.version],
|
||||
['dispatchCount', row.dispatchCount],
|
||||
['createdAtMs', row.createdAtMs],
|
||||
['updatedAtMs', row.updatedAtMs],
|
||||
['nextAttemptAtMs', row.nextAttemptAtMs],
|
||||
['leaseExpiresAtMs', row.leaseExpiresAtMs],
|
||||
['lastDispatchedAtMs', row.lastDispatchedAtMs],
|
||||
] as const) {
|
||||
if (
|
||||
value !== null &&
|
||||
(!Number.isSafeInteger(Number(value)) || Number(value) < 0)
|
||||
) {
|
||||
throw new CancellationDispatchRepositoryError(
|
||||
new Error(`Invalid cancellation dispatch ${name}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
const status = row.status as CancellationDispatchStatus;
|
||||
const hasCompleteLease =
|
||||
row.leaseOwner !== null &&
|
||||
row.leaseToken !== null &&
|
||||
row.leaseExpiresAtMs !== null;
|
||||
const hasAnyLease =
|
||||
row.leaseOwner !== null ||
|
||||
row.leaseToken !== null ||
|
||||
row.leaseExpiresAtMs !== null;
|
||||
if (
|
||||
(status === 'leased' && !hasCompleteLease) ||
|
||||
(status !== 'leased' && hasAnyLease) ||
|
||||
((status === 'pending' || status === 'retry_wait') &&
|
||||
row.nextAttemptAtMs === null) ||
|
||||
((status === 'dispatched' || status === 'blocked' || status === 'leased') &&
|
||||
row.nextAttemptAtMs !== null) ||
|
||||
((status === 'dispatched' || status === 'blocked') &&
|
||||
row.lastResult === null)
|
||||
) {
|
||||
throw new CancellationDispatchRepositoryError(
|
||||
new Error('Cancellation dispatch lease/status fields are inconsistent'),
|
||||
);
|
||||
}
|
||||
return {
|
||||
runId: row.runId,
|
||||
attemptId: row.attemptId,
|
||||
status,
|
||||
version: Number(row.version),
|
||||
dispatchCount: Number(row.dispatchCount),
|
||||
createdAtMs: Number(row.createdAtMs),
|
||||
updatedAtMs: Number(row.updatedAtMs),
|
||||
...(row.nextAttemptAtMs === null
|
||||
? {}
|
||||
: { nextAttemptAtMs: Number(row.nextAttemptAtMs) }),
|
||||
...(row.leaseOwner === null ? {} : { leaseOwner: row.leaseOwner }),
|
||||
...(row.leaseToken === null ? {} : { leaseToken: row.leaseToken }),
|
||||
...(row.leaseExpiresAtMs === null
|
||||
? {}
|
||||
: { leaseExpiresAtMs: Number(row.leaseExpiresAtMs) }),
|
||||
...(row.lastResult === null
|
||||
? {}
|
||||
: { lastResult: row.lastResult as CancellationDispatchResult }),
|
||||
...(row.lastDispatchedAtMs === null
|
||||
? {}
|
||||
: { lastDispatchedAtMs: Number(row.lastDispatchedAtMs) }),
|
||||
};
|
||||
}
|
||||
|
||||
function resultState(result: CancellationDispatchResult): {
|
||||
status: CancellationDispatchStatus;
|
||||
eventType: string;
|
||||
} {
|
||||
if (RETRYABLE_RESULTS.includes(result)) {
|
||||
return { status: 'retry_wait', eventType: 'run.cancel_dispatch_failed' };
|
||||
}
|
||||
if (BLOCKING_RESULTS.includes(result)) {
|
||||
return { status: 'blocked', eventType: 'run.cancel_dispatch_blocked' };
|
||||
}
|
||||
return { status: 'dispatched', eventType: 'run.cancel_dispatched' };
|
||||
}
|
||||
|
||||
function withoutScheduleAndLease(
|
||||
dispatch: CancellationDispatchRecord,
|
||||
): Omit<
|
||||
CancellationDispatchRecord,
|
||||
'nextAttemptAtMs' | 'leaseOwner' | 'leaseToken' | 'leaseExpiresAtMs'
|
||||
> {
|
||||
const {
|
||||
nextAttemptAtMs: _nextAttemptAtMs,
|
||||
leaseOwner: _leaseOwner,
|
||||
leaseToken: _leaseToken,
|
||||
leaseExpiresAtMs: _leaseExpiresAtMs,
|
||||
...rest
|
||||
} = dispatch;
|
||||
return rest;
|
||||
}
|
||||
|
||||
export class LegacySequelizeCancellationDispatchRepository
|
||||
implements CancellationDispatchRepository
|
||||
{
|
||||
private readonly dispatch: ModelStatic<CancellationDispatchInstance>;
|
||||
private readonly run: ModelStatic<CancellationDispatchRunInstance>;
|
||||
private readonly attempt: ModelStatic<CancellationDispatchAttemptInstance>;
|
||||
private readonly event: ModelStatic<CancellationDispatchEventInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
this.dispatch = defineDispatchModel(database);
|
||||
this.run = defineRunModel(database);
|
||||
this.attempt = defineAttemptModel(database);
|
||||
this.event = defineEventModel(database);
|
||||
}
|
||||
|
||||
async findByRunId(runId: string): Promise<CancellationDispatchRecord | null> {
|
||||
assertId('runId', runId);
|
||||
const row = (await this.dispatch.findByPk(runId, {
|
||||
raw: true,
|
||||
})) as unknown as CancellationDispatchRow | null;
|
||||
return row === null ? null : rowToDispatch(row);
|
||||
}
|
||||
|
||||
async claim(
|
||||
command: ClaimCancellationDispatchCommand,
|
||||
): Promise<ClaimCancellationDispatchResult> {
|
||||
assertClaim(command);
|
||||
return this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const [run, attempt] = await Promise.all([
|
||||
this.run.findByPk(command.runId, { raw: true, transaction }),
|
||||
this.attempt.findByPk(command.attemptId, { raw: true, transaction }),
|
||||
]);
|
||||
const runRow = run as unknown as CancellationDispatchRunRow | null;
|
||||
const attemptRow =
|
||||
attempt as unknown as CancellationDispatchAttemptRow | null;
|
||||
if (
|
||||
runRow === null ||
|
||||
attemptRow === null ||
|
||||
runRow.executionOwner !== 'runtime' ||
|
||||
!ACTIVE_RUN_STATUSES.includes(runRow.status as RunStatus) ||
|
||||
runRow.cancelRequestedAtMs === null ||
|
||||
Number(runRow.cancelRequestedAtMs) !== command.requestedAtMs ||
|
||||
attemptRow.runId !== command.runId ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.includes(
|
||||
attemptRow.status as (typeof ACTIVE_ATTEMPT_STATUSES)[number],
|
||||
)
|
||||
) {
|
||||
return { status: 'not_eligible' as const };
|
||||
}
|
||||
|
||||
let row = (await this.dispatch.findByPk(command.runId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
})) as unknown as CancellationDispatchRow | null;
|
||||
if (row === null) {
|
||||
try {
|
||||
const created = await this.dispatch.create(
|
||||
{
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
status: 'pending',
|
||||
version: 0,
|
||||
dispatchCount: 0,
|
||||
nextAttemptAtMs: command.requestedAtMs,
|
||||
leaseOwner: null,
|
||||
leaseToken: null,
|
||||
leaseExpiresAtMs: null,
|
||||
lastResult: null,
|
||||
lastDispatchedAtMs: null,
|
||||
createdAtMs: command.nowMs,
|
||||
updatedAtMs: command.nowMs,
|
||||
},
|
||||
{ transaction },
|
||||
);
|
||||
row = created.get({ plain: true }) as CancellationDispatchRow;
|
||||
} catch (error) {
|
||||
if (error instanceof UniqueConstraintError) {
|
||||
throw new CancellationDispatchRepositoryError(error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (row.attemptId !== command.attemptId) {
|
||||
throw new CancellationDispatchBindingConflictError(
|
||||
command.runId,
|
||||
command.attemptId,
|
||||
);
|
||||
}
|
||||
const dispatch = rowToDispatch(row);
|
||||
if (dispatch.status === 'dispatched' || dispatch.status === 'blocked') {
|
||||
return { status: dispatch.status, dispatch };
|
||||
}
|
||||
if (
|
||||
dispatch.status === 'leased' &&
|
||||
dispatch.leaseExpiresAtMs !== undefined &&
|
||||
dispatch.leaseExpiresAtMs > command.nowMs
|
||||
) {
|
||||
return { status: 'leased', dispatch };
|
||||
}
|
||||
if (
|
||||
dispatch.status !== 'leased' &&
|
||||
dispatch.nextAttemptAtMs !== undefined &&
|
||||
dispatch.nextAttemptAtMs > command.nowMs
|
||||
) {
|
||||
return { status: 'not_due', dispatch };
|
||||
}
|
||||
|
||||
const nextVersion = dispatch.version + 1;
|
||||
const nextCount = dispatch.dispatchCount + 1;
|
||||
const leaseExpiresAtMs = command.nowMs + command.leaseDurationMs;
|
||||
const [affected] = await this.dispatch.update(
|
||||
{
|
||||
status: 'leased',
|
||||
version: nextVersion,
|
||||
dispatchCount: nextCount,
|
||||
nextAttemptAtMs: null,
|
||||
leaseOwner: command.owner,
|
||||
leaseToken: command.leaseToken,
|
||||
leaseExpiresAtMs,
|
||||
updatedAtMs: command.nowMs,
|
||||
},
|
||||
{
|
||||
where: { runId: command.runId, version: dispatch.version },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (affected !== 1) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
return {
|
||||
status: 'claimed',
|
||||
dispatch: {
|
||||
...withoutScheduleAndLease(dispatch),
|
||||
status: 'leased',
|
||||
version: nextVersion,
|
||||
dispatchCount: nextCount,
|
||||
leaseOwner: command.owner,
|
||||
leaseToken: command.leaseToken,
|
||||
leaseExpiresAtMs,
|
||||
updatedAtMs: command.nowMs,
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async recordResult(
|
||||
command: RecordCancellationDispatchResultCommand,
|
||||
): Promise<RecordCancellationDispatchResult> {
|
||||
assertRecordResult(command);
|
||||
return this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const row = (await this.dispatch.findByPk(command.runId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
})) as unknown as CancellationDispatchRow | null;
|
||||
if (
|
||||
row === null ||
|
||||
row.attemptId !== command.attemptId ||
|
||||
row.status !== 'leased' ||
|
||||
row.version !== command.expectedVersion ||
|
||||
row.leaseOwner !== command.owner ||
|
||||
row.leaseToken !== command.leaseToken
|
||||
) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
const run = (await this.run.findByPk(command.runId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
})) as unknown as CancellationDispatchRunRow | null;
|
||||
if (run === null) {
|
||||
throw new CancellationDispatchRepositoryError(
|
||||
new Error('Run disappeared while recording cancellation result'),
|
||||
);
|
||||
}
|
||||
const state = resultState(command.result);
|
||||
const controllerInvoked = ![
|
||||
'controller_missing',
|
||||
'handle_missing',
|
||||
].includes(command.result);
|
||||
const nextVersion = row.version + 1;
|
||||
const nextSequence = Number(run.eventSequence) + 1;
|
||||
const [runAffected] = await this.run.update(
|
||||
{ version: Number(run.version) + 1, eventSequence: nextSequence },
|
||||
{ where: { id: command.runId, version: run.version }, transaction },
|
||||
);
|
||||
if (runAffected !== 1) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
const [dispatchAffected] = await this.dispatch.update(
|
||||
{
|
||||
status: state.status,
|
||||
version: nextVersion,
|
||||
nextAttemptAtMs: command.nextAttemptAtMs ?? null,
|
||||
leaseOwner: null,
|
||||
leaseToken: null,
|
||||
leaseExpiresAtMs: null,
|
||||
lastResult: command.result,
|
||||
lastDispatchedAtMs: controllerInvoked
|
||||
? command.atMs
|
||||
: row.lastDispatchedAtMs,
|
||||
updatedAtMs: command.atMs,
|
||||
},
|
||||
{
|
||||
where: {
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
status: 'leased',
|
||||
version: command.expectedVersion,
|
||||
leaseOwner: command.owner,
|
||||
leaseToken: command.leaseToken,
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (dispatchAffected !== 1) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
const event: RunEventRecord = {
|
||||
id: command.eventId,
|
||||
runId: command.runId,
|
||||
sequence: nextSequence,
|
||||
type: state.eventType,
|
||||
dedupeKey: `cancel-dispatch:${command.attemptId}:${row.dispatchCount}`,
|
||||
actorType: 'worker',
|
||||
actorId: command.owner,
|
||||
attemptId: command.attemptId,
|
||||
payload: {
|
||||
attempt_id: command.attemptId,
|
||||
dispatch_count: row.dispatchCount,
|
||||
result: command.result,
|
||||
},
|
||||
createdAtMs: command.atMs,
|
||||
};
|
||||
await this.event.create(
|
||||
{
|
||||
id: event.id,
|
||||
runId: event.runId,
|
||||
sequence: event.sequence,
|
||||
type: event.type,
|
||||
dedupeKey: event.dedupeKey!,
|
||||
actorType: event.actorType,
|
||||
actorId: event.actorId!,
|
||||
attemptId: event.attemptId!,
|
||||
payload: event.payload,
|
||||
createdAtMs: event.createdAtMs,
|
||||
},
|
||||
{ transaction },
|
||||
);
|
||||
return {
|
||||
dispatch: {
|
||||
...withoutScheduleAndLease(rowToDispatch(row)),
|
||||
status: state.status,
|
||||
version: nextVersion,
|
||||
...(command.nextAttemptAtMs === undefined
|
||||
? {}
|
||||
: { nextAttemptAtMs: command.nextAttemptAtMs }),
|
||||
lastResult: command.result,
|
||||
...(controllerInvoked
|
||||
? { lastDispatchedAtMs: command.atMs }
|
||||
: row.lastDispatchedAtMs === null
|
||||
? {}
|
||||
: { lastDispatchedAtMs: Number(row.lastDispatchedAtMs) }),
|
||||
updatedAtMs: command.atMs,
|
||||
},
|
||||
event,
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Op,
|
||||
Sequelize,
|
||||
UniqueConstraintError,
|
||||
type WhereOptions,
|
||||
} from 'sequelize';
|
||||
import { RUN_ATTEMPT_TABLE } from '../../../migrations/0002-run-schema';
|
||||
import { COMPLETION_RECEIPT_JOURNAL_TABLE } from '../../../migrations/0007-completion-receipt-journal';
|
||||
import {
|
||||
COMPLETION_RECEIPT_JOURNAL_STATES,
|
||||
type CompletionReceiptJournalCandidate,
|
||||
type CompletionReceiptJournalCursor,
|
||||
type CompletionReceiptJournalRecord,
|
||||
type CompletionReceiptJournalState,
|
||||
} from '../../domain/completionReceiptJournal';
|
||||
import { assertCompletionReceiptId } from '../../domain/completionReceipt';
|
||||
import type { RunAttemptStatus } from '../../domain/run';
|
||||
import {
|
||||
MAX_COMPLETION_RECEIPT_JOURNAL_BATCH_SIZE,
|
||||
type CompletionReceiptJournal,
|
||||
type QuarantineCompletionReceiptCommand,
|
||||
type RegisterCompletionReceiptCommand,
|
||||
} from '../../ports/completionReceiptJournal';
|
||||
|
||||
interface JournalRow {
|
||||
attemptId: string;
|
||||
runId: string;
|
||||
state: string;
|
||||
quarantineRef: string | null;
|
||||
purgeAfterMs: number | null;
|
||||
registeredAtMs: number;
|
||||
updatedAtMs: number;
|
||||
}
|
||||
|
||||
interface AttemptRow {
|
||||
id: string;
|
||||
status: string;
|
||||
executorType: string;
|
||||
finishedAtMs: number | null;
|
||||
}
|
||||
|
||||
interface JournalInstance extends Model<JournalRow, JournalRow>, JournalRow {}
|
||||
interface AttemptInstance extends Model<AttemptRow, AttemptRow>, AttemptRow {}
|
||||
|
||||
function defineJournalModel(database: Sequelize): ModelStatic<JournalInstance> {
|
||||
return database.define<JournalInstance>(
|
||||
'Ql3CompletionReceiptJournal',
|
||||
{
|
||||
attemptId: {
|
||||
field: 'attempt_id',
|
||||
type: DataTypes.STRING(36),
|
||||
primaryKey: true,
|
||||
},
|
||||
runId: {
|
||||
field: 'run_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
state: { type: DataTypes.STRING(16), allowNull: false },
|
||||
quarantineRef: {
|
||||
field: 'quarantine_ref',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: true,
|
||||
},
|
||||
purgeAfterMs: {
|
||||
field: 'purge_after_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
registeredAtMs: {
|
||||
field: 'registered_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
updatedAtMs: {
|
||||
field: 'updated_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: COMPLETION_RECEIPT_JOURNAL_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function defineAttemptModel(database: Sequelize): ModelStatic<AttemptInstance> {
|
||||
return database.define<AttemptInstance>(
|
||||
'Ql3CompletionReceiptJournalAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
executorType: {
|
||||
field: 'executor_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
finishedAtMs: {
|
||||
field: 'finished_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_ATTEMPT_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function assertNonNegativeTimestamp(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new RangeError(`${name} must be a non-negative safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertCursor(cursor: CompletionReceiptJournalCursor): void {
|
||||
assertNonNegativeTimestamp('cursor.updatedAtMs', cursor.updatedAtMs);
|
||||
assertCompletionReceiptId(cursor.attemptId, 'attemptId');
|
||||
}
|
||||
|
||||
function assertQuarantineRef(value: string): void {
|
||||
if (
|
||||
value.length < 1 ||
|
||||
value.length > 255 ||
|
||||
!value.startsWith('.quarantine/') ||
|
||||
value.includes('..') ||
|
||||
value.includes('\\') ||
|
||||
value.includes('\0')
|
||||
) {
|
||||
throw new TypeError('quarantineRef is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function toRecord(row: JournalRow): CompletionReceiptJournalRecord {
|
||||
if (!COMPLETION_RECEIPT_JOURNAL_STATES.includes(row.state as never)) {
|
||||
throw new Error('Completion receipt journal state is corrupt');
|
||||
}
|
||||
return {
|
||||
attemptId: row.attemptId,
|
||||
runId: row.runId,
|
||||
state: row.state as CompletionReceiptJournalState,
|
||||
registeredAtMs: Number(row.registeredAtMs),
|
||||
updatedAtMs: Number(row.updatedAtMs),
|
||||
...(row.quarantineRef === null ? {} : { quarantineRef: row.quarantineRef }),
|
||||
...(row.purgeAfterMs === null
|
||||
? {}
|
||||
: { purgeAfterMs: Number(row.purgeAfterMs) }),
|
||||
};
|
||||
}
|
||||
|
||||
export class LegacySequelizeCompletionReceiptJournal
|
||||
implements CompletionReceiptJournal
|
||||
{
|
||||
private readonly journal: ModelStatic<JournalInstance>;
|
||||
private readonly attempt: ModelStatic<AttemptInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.journal = defineJournalModel(database);
|
||||
this.attempt = defineAttemptModel(database);
|
||||
}
|
||||
|
||||
async register(command: RegisterCompletionReceiptCommand): Promise<void> {
|
||||
assertCompletionReceiptId(command.attemptId, 'attemptId');
|
||||
assertCompletionReceiptId(command.runId, 'runId');
|
||||
assertNonNegativeTimestamp('registeredAtMs', command.registeredAtMs);
|
||||
const values: JournalRow = {
|
||||
attemptId: command.attemptId,
|
||||
runId: command.runId,
|
||||
state: 'pending',
|
||||
quarantineRef: null,
|
||||
purgeAfterMs: null,
|
||||
registeredAtMs: command.registeredAtMs,
|
||||
updatedAtMs: command.registeredAtMs,
|
||||
};
|
||||
try {
|
||||
await this.journal.create(values);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!(error instanceof UniqueConstraintError)) throw error;
|
||||
}
|
||||
const current = (await this.journal.findByPk(command.attemptId, {
|
||||
raw: true,
|
||||
})) as unknown as JournalRow | null;
|
||||
if (
|
||||
!current ||
|
||||
current.runId !== command.runId ||
|
||||
Number(current.registeredAtMs) !== command.registeredAtMs
|
||||
) {
|
||||
throw new Error('Completion receipt journal registration conflicts');
|
||||
}
|
||||
}
|
||||
|
||||
async markQuarantined(
|
||||
command: QuarantineCompletionReceiptCommand,
|
||||
): Promise<void> {
|
||||
assertCompletionReceiptId(command.attemptId, 'attemptId');
|
||||
assertQuarantineRef(command.quarantineRef);
|
||||
assertNonNegativeTimestamp('updatedAtMs', command.updatedAtMs);
|
||||
assertNonNegativeTimestamp('purgeAfterMs', command.purgeAfterMs);
|
||||
if (command.purgeAfterMs < command.updatedAtMs) {
|
||||
throw new RangeError('purgeAfterMs must not precede updatedAtMs');
|
||||
}
|
||||
const [updated] = await this.journal.update(
|
||||
{
|
||||
state: 'quarantined',
|
||||
quarantineRef: command.quarantineRef,
|
||||
purgeAfterMs: command.purgeAfterMs,
|
||||
updatedAtMs: command.updatedAtMs,
|
||||
},
|
||||
{ where: { attemptId: command.attemptId, state: 'pending' } },
|
||||
);
|
||||
if (updated === 1) return;
|
||||
const current = (await this.journal.findByPk(command.attemptId, {
|
||||
raw: true,
|
||||
})) as unknown as JournalRow | null;
|
||||
if (
|
||||
current?.state === 'quarantined' &&
|
||||
current.quarantineRef === command.quarantineRef &&
|
||||
Number(current.purgeAfterMs) === command.purgeAfterMs
|
||||
) {
|
||||
return;
|
||||
}
|
||||
throw new Error('Completion receipt journal quarantine transition failed');
|
||||
}
|
||||
|
||||
async resolve(attemptId: string): Promise<boolean> {
|
||||
assertCompletionReceiptId(attemptId, 'attemptId');
|
||||
return (await this.journal.destroy({ where: { attemptId } })) === 1;
|
||||
}
|
||||
|
||||
async listCandidates({
|
||||
observedAtMs,
|
||||
cursor,
|
||||
limit = 32,
|
||||
}: {
|
||||
observedAtMs: number;
|
||||
cursor?: CompletionReceiptJournalCursor;
|
||||
limit?: number;
|
||||
}) {
|
||||
assertNonNegativeTimestamp('observedAtMs', observedAtMs);
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_COMPLETION_RECEIPT_JOURNAL_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_COMPLETION_RECEIPT_JOURNAL_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (cursor) assertCursor(cursor);
|
||||
|
||||
const eligible: WhereOptions<JournalRow> = {
|
||||
[Op.or]: [
|
||||
{ state: 'pending' },
|
||||
{
|
||||
state: 'quarantined',
|
||||
purgeAfterMs: { [Op.lte]: observedAtMs },
|
||||
},
|
||||
],
|
||||
};
|
||||
const afterCursor: WhereOptions<JournalRow> | undefined = cursor
|
||||
? {
|
||||
[Op.or]: [
|
||||
{ updatedAtMs: { [Op.gt]: cursor.updatedAtMs } },
|
||||
{
|
||||
updatedAtMs: cursor.updatedAtMs,
|
||||
attemptId: { [Op.gt]: cursor.attemptId },
|
||||
},
|
||||
],
|
||||
}
|
||||
: undefined;
|
||||
const where: WhereOptions<JournalRow> = afterCursor
|
||||
? { [Op.and]: [eligible, afterCursor] }
|
||||
: eligible;
|
||||
const rows = (await this.journal.findAll({
|
||||
where,
|
||||
order: [
|
||||
['updatedAtMs', 'ASC'],
|
||||
['attemptId', 'ASC'],
|
||||
],
|
||||
limit: limit + 1,
|
||||
raw: true,
|
||||
})) as unknown as JournalRow[];
|
||||
const truncated = rows.length > limit;
|
||||
const bounded = rows.slice(0, limit);
|
||||
if (bounded.length === 0) return { candidates: [], truncated: false };
|
||||
|
||||
const attempts = (await this.attempt.findAll({
|
||||
where: { id: { [Op.in]: bounded.map((row) => row.attemptId) } },
|
||||
raw: true,
|
||||
})) as unknown as AttemptRow[];
|
||||
const attemptById = new Map(attempts.map((row) => [row.id, row]));
|
||||
const candidates: CompletionReceiptJournalCandidate[] = bounded.map(
|
||||
(row) => {
|
||||
const attempt = attemptById.get(row.attemptId);
|
||||
if (!attempt) {
|
||||
throw new Error('Completion receipt journal Attempt is missing');
|
||||
}
|
||||
return {
|
||||
...toRecord(row),
|
||||
attemptStatus: attempt.status as RunAttemptStatus,
|
||||
executorType: attempt.executorType,
|
||||
...(attempt.finishedAtMs === null
|
||||
? {}
|
||||
: { finishedAtMs: Number(attempt.finishedAtMs) }),
|
||||
};
|
||||
},
|
||||
);
|
||||
const last = bounded[bounded.length - 1];
|
||||
return {
|
||||
candidates,
|
||||
truncated,
|
||||
nextCursor: {
|
||||
updatedAtMs: Number(last.updatedAtMs),
|
||||
attemptId: last.attemptId,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import {
|
||||
IDENTITY_AUTHENTICATION_BINDING_TABLE,
|
||||
IDENTITY_SUBJECT_TABLE,
|
||||
} from '../../../migrations/0019-identity-directory';
|
||||
import {
|
||||
IdentityDirectoryUnavailableError,
|
||||
assertIdentityProvider,
|
||||
assertIdentityProviderSubject,
|
||||
normalizeIdentityAuthenticationBindingRecord,
|
||||
normalizeIdentitySubjectRecord,
|
||||
} from '../../domain/identityDirectory';
|
||||
import type { PolicySubject } from '../../domain/projectPolicy';
|
||||
import type { IdentityDirectoryRepository } from '../../ports/identityDirectoryRepository';
|
||||
|
||||
interface IdentityAuthenticationRow {
|
||||
provider: string;
|
||||
provider_subject: string;
|
||||
binding_version: number;
|
||||
binding_state: string;
|
||||
binding_subject_id: string;
|
||||
binding_created_at_ms: number | string;
|
||||
subject_id: string | null;
|
||||
subject_type: string | null;
|
||||
subject_status: string | null;
|
||||
subject_version: number | null;
|
||||
subject_created_at_ms: number | string | null;
|
||||
subject_updated_at_ms: number | string | null;
|
||||
}
|
||||
|
||||
export class LegacySequelizeIdentityDirectoryRepository
|
||||
implements IdentityDirectoryRepository
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Identity directory repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async resolveAuthenticationSubject(
|
||||
provider: string,
|
||||
providerSubject: string,
|
||||
): Promise<Readonly<PolicySubject> | null> {
|
||||
assertIdentityProvider(provider);
|
||||
assertIdentityProviderSubject(providerSubject);
|
||||
try {
|
||||
const rows = await this.database.query<IdentityAuthenticationRow>(
|
||||
`SELECT binding.provider AS provider,
|
||||
binding.provider_subject AS provider_subject,
|
||||
binding.version AS binding_version,
|
||||
binding.state AS binding_state,
|
||||
binding.subject_id AS binding_subject_id,
|
||||
binding.created_at_ms AS binding_created_at_ms,
|
||||
subject.id AS subject_id,
|
||||
subject.type AS subject_type,
|
||||
subject.status AS subject_status,
|
||||
subject.version AS subject_version,
|
||||
subject.created_at_ms AS subject_created_at_ms,
|
||||
subject.updated_at_ms AS subject_updated_at_ms
|
||||
FROM "${IDENTITY_AUTHENTICATION_BINDING_TABLE}" AS binding
|
||||
LEFT JOIN "${IDENTITY_SUBJECT_TABLE}" AS subject
|
||||
ON subject.id = binding.subject_id
|
||||
WHERE binding.provider = :provider
|
||||
AND binding.provider_subject = :providerSubject
|
||||
AND binding.version = (
|
||||
SELECT MAX(current.version)
|
||||
FROM "${IDENTITY_AUTHENTICATION_BINDING_TABLE}" AS current
|
||||
WHERE current.provider = binding.provider
|
||||
AND current.provider_subject = binding.provider_subject
|
||||
)
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { provider, providerSubject },
|
||||
},
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length !== 1) throw new IdentityDirectoryUnavailableError();
|
||||
const row = rows[0];
|
||||
const binding = normalizeIdentityAuthenticationBindingRecord({
|
||||
provider: row.provider,
|
||||
providerSubject: row.provider_subject,
|
||||
version: Number(row.binding_version),
|
||||
state: row.binding_state as 'active' | 'revoked',
|
||||
subjectId: row.binding_subject_id,
|
||||
createdAtMs: Number(row.binding_created_at_ms),
|
||||
});
|
||||
const subject = normalizeIdentitySubjectRecord({
|
||||
subject: {
|
||||
type: row.subject_type as PolicySubject['type'],
|
||||
id: row.subject_id!,
|
||||
},
|
||||
status: row.subject_status as 'active' | 'disabled',
|
||||
version: Number(row.subject_version),
|
||||
createdAtMs: Number(row.subject_created_at_ms),
|
||||
updatedAtMs: Number(row.subject_updated_at_ms),
|
||||
});
|
||||
if (binding.subjectId !== subject.subject.id) {
|
||||
throw new IdentityDirectoryUnavailableError();
|
||||
}
|
||||
if (
|
||||
binding.state !== 'active' ||
|
||||
subject.status !== 'active' ||
|
||||
subject.subject.type !== 'user'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return subject.subject;
|
||||
} catch (error) {
|
||||
if (error instanceof IdentityDirectoryUnavailableError) throw error;
|
||||
throw new IdentityDirectoryUnavailableError();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { LOCAL_ARTIFACT_RETENTION_TABLE } from '../../../migrations/0015-local-artifact-retention';
|
||||
import {
|
||||
normalizeLocalArtifactReadMetadata,
|
||||
type LocalArtifactReadMetadata,
|
||||
} from '../../domain/artifactRead';
|
||||
import type { LocalArtifactReadMetadataRepository } from '../../ports/localArtifactReadMetadataRepository';
|
||||
|
||||
interface ArtifactMetadataRow {
|
||||
project_id: string;
|
||||
run_id: string;
|
||||
attempt_id: string;
|
||||
attempt_finished_at_ms: number | string | null;
|
||||
log_artifact_id: string;
|
||||
retention_log_artifact_id: string | null;
|
||||
retention_disposition: string | null;
|
||||
retention_finished_at_ms: number | string | null;
|
||||
retention_eligible_at_ms: number | string | null;
|
||||
retention_bytes_reclaimed: number | string | null;
|
||||
retention_recorded_at_ms: number | string | null;
|
||||
}
|
||||
|
||||
export class CorruptLocalArtifactReadMetadataError extends Error {
|
||||
constructor() {
|
||||
super('Local Artifact read metadata is corrupt or ambiguous');
|
||||
this.name = 'CorruptLocalArtifactReadMetadataError';
|
||||
}
|
||||
}
|
||||
|
||||
function rowToMetadata(
|
||||
row: ArtifactMetadataRow,
|
||||
): Readonly<LocalArtifactReadMetadata> {
|
||||
const retentionValues = [
|
||||
row.retention_log_artifact_id,
|
||||
row.retention_disposition,
|
||||
row.retention_finished_at_ms,
|
||||
row.retention_eligible_at_ms,
|
||||
row.retention_bytes_reclaimed,
|
||||
row.retention_recorded_at_ms,
|
||||
];
|
||||
const hasRetention = retentionValues.every((value) => value !== null);
|
||||
if (!hasRetention && retentionValues.some((value) => value !== null)) {
|
||||
throw new CorruptLocalArtifactReadMetadataError();
|
||||
}
|
||||
if (hasRetention && row.retention_log_artifact_id !== row.log_artifact_id) {
|
||||
throw new CorruptLocalArtifactReadMetadataError();
|
||||
}
|
||||
if (
|
||||
hasRetention &&
|
||||
(row.attempt_finished_at_ms === null ||
|
||||
Number(row.retention_finished_at_ms) !==
|
||||
Number(row.attempt_finished_at_ms))
|
||||
) {
|
||||
throw new CorruptLocalArtifactReadMetadataError();
|
||||
}
|
||||
try {
|
||||
return normalizeLocalArtifactReadMetadata({
|
||||
projectId: row.project_id,
|
||||
runId: row.run_id,
|
||||
attemptId: row.attempt_id,
|
||||
logArtifactId: row.log_artifact_id,
|
||||
...(hasRetention
|
||||
? {
|
||||
retention: {
|
||||
disposition: row.retention_disposition as NonNullable<
|
||||
LocalArtifactReadMetadata['retention']
|
||||
>['disposition'],
|
||||
finishedAtMs: Number(row.retention_finished_at_ms),
|
||||
eligibleAtMs: Number(row.retention_eligible_at_ms),
|
||||
bytesReclaimed: Number(row.retention_bytes_reclaimed),
|
||||
recordedAtMs: Number(row.retention_recorded_at_ms),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof CorruptLocalArtifactReadMetadataError) throw error;
|
||||
throw new CorruptLocalArtifactReadMetadataError();
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeLocalArtifactReadMetadataRepository
|
||||
implements LocalArtifactReadMetadataRepository
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Local Artifact read metadata repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async find({
|
||||
projectId,
|
||||
runId,
|
||||
logArtifactId,
|
||||
}: Parameters<
|
||||
LocalArtifactReadMetadataRepository['find']
|
||||
>[0]): Promise<Readonly<LocalArtifactReadMetadata> | null> {
|
||||
const rows = await this.database.query<ArtifactMetadataRow>(
|
||||
`SELECT run.project_id,
|
||||
run.id AS run_id,
|
||||
attempt.id AS attempt_id,
|
||||
attempt.finished_at_ms AS attempt_finished_at_ms,
|
||||
attempt.log_artifact_id,
|
||||
retained.log_artifact_id AS retention_log_artifact_id,
|
||||
retained.disposition AS retention_disposition,
|
||||
retained.finished_at_ms AS retention_finished_at_ms,
|
||||
retained.eligible_at_ms AS retention_eligible_at_ms,
|
||||
retained.bytes_reclaimed AS retention_bytes_reclaimed,
|
||||
retained.recorded_at_ms AS retention_recorded_at_ms
|
||||
FROM "${RUN_TABLE}" AS run
|
||||
JOIN "${RUN_ATTEMPT_TABLE}" AS attempt ON attempt.run_id = run.id
|
||||
LEFT JOIN "${LOCAL_ARTIFACT_RETENTION_TABLE}" AS retained
|
||||
ON retained.attempt_id = attempt.id
|
||||
WHERE run.project_id = :projectId
|
||||
AND run.id = :runId
|
||||
AND run.execution_owner = 'runtime'
|
||||
AND attempt.executor_type = 'local_process'
|
||||
AND attempt.log_artifact_id = :logArtifactId
|
||||
AND attempt.log_artifact_id LIKE 'local-%'
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId, runId, logArtifactId },
|
||||
},
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length !== 1) throw new CorruptLocalArtifactReadMetadataError();
|
||||
return rowToMetadata(rows[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
type ModelStatic,
|
||||
Sequelize,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE } from '../../../migrations/0016-local-artifact-maintenance-cursor';
|
||||
import {
|
||||
normalizeLocalArtifactRetentionCursor,
|
||||
assertLocalArtifactRetentionTimestamp,
|
||||
} from '../../domain/localArtifactRetention';
|
||||
import {
|
||||
normalizeLocalArtifactRetentionCheckpoint,
|
||||
type LocalArtifactRetentionCheckpoint,
|
||||
} from '../../domain/localArtifactRetentionCheckpoint';
|
||||
import type { LocalArtifactRetentionCheckpointStore } from '../../ports/localArtifactRetentionCheckpointStore';
|
||||
|
||||
const RETENTION_SCOPE = 'retention';
|
||||
|
||||
interface CursorRow {
|
||||
scope: string;
|
||||
cursorFinishedAtMs: number | string | null;
|
||||
cursorAttemptId: string | null;
|
||||
version: number | string;
|
||||
updatedAtMs: number | string;
|
||||
}
|
||||
|
||||
interface CursorInstance extends Model<CursorRow, CursorRow>, CursorRow {}
|
||||
|
||||
function defineCursorModel(database: Sequelize): ModelStatic<CursorInstance> {
|
||||
return database.define<CursorInstance>(
|
||||
'Ql3LocalArtifactMaintenanceCursor',
|
||||
{
|
||||
scope: { type: DataTypes.STRING(32), allowNull: false, primaryKey: true },
|
||||
cursorFinishedAtMs: {
|
||||
field: 'cursor_finished_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
cursorAttemptId: {
|
||||
field: 'cursor_attempt_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: true,
|
||||
},
|
||||
version: { type: DataTypes.BIGINT, allowNull: false },
|
||||
updatedAtMs: {
|
||||
field: 'updated_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function rowToCheckpoint(
|
||||
row: CursorRow,
|
||||
): Readonly<LocalArtifactRetentionCheckpoint> {
|
||||
const finishedAtMs =
|
||||
row.cursorFinishedAtMs === null ? null : Number(row.cursorFinishedAtMs);
|
||||
const attemptId = row.cursorAttemptId;
|
||||
if ((finishedAtMs === null) !== (attemptId === null)) {
|
||||
throw new TypeError('Local Artifact retention cursor row is corrupt');
|
||||
}
|
||||
return normalizeLocalArtifactRetentionCheckpoint({
|
||||
version: Number(row.version),
|
||||
...(finishedAtMs === null || attemptId === null
|
||||
? {}
|
||||
: { cursor: { finishedAtMs, attemptId } }),
|
||||
});
|
||||
}
|
||||
|
||||
export class LegacySequelizeLocalArtifactRetentionCheckpointStore
|
||||
implements LocalArtifactRetentionCheckpointStore
|
||||
{
|
||||
private readonly cursors: ModelStatic<CursorInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Local Artifact retention checkpoint store is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.cursors = defineCursorModel(database);
|
||||
}
|
||||
|
||||
async load(): Promise<Readonly<LocalArtifactRetentionCheckpoint>> {
|
||||
const row = await this.cursors.findByPk(RETENTION_SCOPE, { raw: true });
|
||||
return row
|
||||
? rowToCheckpoint(row)
|
||||
: normalizeLocalArtifactRetentionCheckpoint({ version: 0 });
|
||||
}
|
||||
|
||||
async compareAndSet({
|
||||
expectedVersion,
|
||||
cursor,
|
||||
updatedAtMs,
|
||||
}: Parameters<
|
||||
LocalArtifactRetentionCheckpointStore['compareAndSet']
|
||||
>[0]): Promise<boolean> {
|
||||
const checkpoint = normalizeLocalArtifactRetentionCheckpoint({
|
||||
version: expectedVersion,
|
||||
...(cursor ? { cursor } : {}),
|
||||
});
|
||||
assertLocalArtifactRetentionTimestamp('updatedAtMs', updatedAtMs);
|
||||
const next = {
|
||||
scope: RETENTION_SCOPE,
|
||||
cursorFinishedAtMs: checkpoint.cursor?.finishedAtMs ?? null,
|
||||
cursorAttemptId: checkpoint.cursor?.attemptId ?? null,
|
||||
version: checkpoint.version + 1,
|
||||
updatedAtMs,
|
||||
};
|
||||
if (checkpoint.version === 0) {
|
||||
try {
|
||||
await this.cursors.create(next);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error instanceof UniqueConstraintError) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const [updated] = await this.cursors.update(next, {
|
||||
where: { scope: RETENTION_SCOPE, version: checkpoint.version },
|
||||
});
|
||||
return updated === 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { COMPLETION_RECEIPT_JOURNAL_TABLE } from '../../../migrations/0007-completion-receipt-journal';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { LOCAL_ARTIFACT_RETENTION_TABLE } from '../../../migrations/0015-local-artifact-retention';
|
||||
import {
|
||||
normalizeLocalArtifactRetentionCandidate,
|
||||
normalizeLocalArtifactRetentionCursor,
|
||||
normalizeLocalArtifactRetentionRecord,
|
||||
assertLocalArtifactRetentionTimestamp,
|
||||
type LocalArtifactRetentionCandidate,
|
||||
type LocalArtifactRetentionRecord,
|
||||
} from '../../domain/localArtifactRetention';
|
||||
import type {
|
||||
LocalArtifactRetentionPage,
|
||||
LocalArtifactRetentionRepository,
|
||||
} from '../../ports/localArtifactRetentionRepository';
|
||||
import { MAX_LOCAL_ARTIFACT_RETENTION_PAGE_SIZE } from '../../ports/localArtifactRetentionRepository';
|
||||
|
||||
interface LocalArtifactRetentionRow {
|
||||
attemptId: string;
|
||||
logArtifactId: string;
|
||||
finishedAtMs: number | string;
|
||||
eligibleAtMs: number | string;
|
||||
disposition: string;
|
||||
bytesReclaimed: number | string;
|
||||
recordedAtMs: number | string;
|
||||
}
|
||||
|
||||
interface LocalArtifactRetentionInstance
|
||||
extends Model<LocalArtifactRetentionRow, LocalArtifactRetentionRow>,
|
||||
LocalArtifactRetentionRow {}
|
||||
|
||||
interface CandidateRow {
|
||||
attempt_id: string;
|
||||
log_artifact_id: string;
|
||||
finished_at_ms: number | string;
|
||||
}
|
||||
|
||||
function defineRetentionModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<LocalArtifactRetentionInstance> {
|
||||
return database.define<LocalArtifactRetentionInstance>(
|
||||
'Ql3LocalArtifactRetention',
|
||||
{
|
||||
attemptId: {
|
||||
field: 'attempt_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
logArtifactId: {
|
||||
field: 'log_artifact_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
finishedAtMs: {
|
||||
field: 'finished_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
eligibleAtMs: {
|
||||
field: 'eligible_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
disposition: { type: DataTypes.STRING(16), allowNull: false },
|
||||
bytesReclaimed: {
|
||||
field: 'bytes_reclaimed',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
recordedAtMs: {
|
||||
field: 'recorded_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: LOCAL_ARTIFACT_RETENTION_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function rowToRecord(
|
||||
row: LocalArtifactRetentionRow,
|
||||
): LocalArtifactRetentionRecord {
|
||||
return normalizeLocalArtifactRetentionRecord({
|
||||
attemptId: row.attemptId,
|
||||
logArtifactId: row.logArtifactId,
|
||||
finishedAtMs: Number(row.finishedAtMs),
|
||||
eligibleAtMs: Number(row.eligibleAtMs),
|
||||
disposition: row.disposition as LocalArtifactRetentionRecord['disposition'],
|
||||
bytesReclaimed: Number(row.bytesReclaimed),
|
||||
recordedAtMs: Number(row.recordedAtMs),
|
||||
});
|
||||
}
|
||||
|
||||
function sameRetirementIdentity(
|
||||
left: LocalArtifactRetentionRecord,
|
||||
right: LocalArtifactRetentionRecord,
|
||||
): boolean {
|
||||
return (
|
||||
left.attemptId === right.attemptId &&
|
||||
left.logArtifactId === right.logArtifactId &&
|
||||
left.finishedAtMs === right.finishedAtMs
|
||||
);
|
||||
}
|
||||
|
||||
export class LocalArtifactRetentionRecordConflictError extends Error {
|
||||
constructor() {
|
||||
super('Local Artifact retention record conflicts with existing evidence');
|
||||
this.name = 'LocalArtifactRetentionRecordConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeLocalArtifactRetentionRepository
|
||||
implements LocalArtifactRetentionRepository
|
||||
{
|
||||
private readonly retention: ModelStatic<LocalArtifactRetentionInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Local Artifact retention repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.retention = defineRetentionModel(database);
|
||||
}
|
||||
|
||||
async list({
|
||||
cutoffMs,
|
||||
cursor,
|
||||
limit,
|
||||
}: Parameters<
|
||||
LocalArtifactRetentionRepository['list']
|
||||
>[0]): Promise<LocalArtifactRetentionPage> {
|
||||
assertLocalArtifactRetentionTimestamp('cutoffMs', cutoffMs);
|
||||
const normalizedCursor = cursor
|
||||
? normalizeLocalArtifactRetentionCursor(cursor)
|
||||
: undefined;
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_LOCAL_ARTIFACT_RETENTION_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError('Local Artifact retention page size is invalid');
|
||||
}
|
||||
const replacements: Record<string, string | number> = {
|
||||
cutoffMs,
|
||||
fetchLimit: limit + 1,
|
||||
...(normalizedCursor
|
||||
? {
|
||||
cursorFinishedAtMs: normalizedCursor.finishedAtMs,
|
||||
cursorAttemptId: normalizedCursor.attemptId,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const cursorPredicate = normalizedCursor
|
||||
? `AND (
|
||||
attempt.finished_at_ms > :cursorFinishedAtMs OR
|
||||
(attempt.finished_at_ms = :cursorFinishedAtMs AND attempt.id > :cursorAttemptId)
|
||||
)`
|
||||
: '';
|
||||
const rows = await this.database.query<CandidateRow>(
|
||||
`SELECT attempt.id AS attempt_id,
|
||||
attempt.log_artifact_id,
|
||||
attempt.finished_at_ms
|
||||
FROM "${RUN_ATTEMPT_TABLE}" AS attempt
|
||||
JOIN "${RUN_TABLE}" AS run ON run.id = attempt.run_id
|
||||
WHERE run.execution_owner = 'runtime'
|
||||
AND run.status IN ('succeeded','failed','cancelled','timed_out')
|
||||
AND attempt.status IN ('succeeded','failed','cancelled','timed_out')
|
||||
AND attempt.executor_type = 'local_process'
|
||||
AND attempt.log_artifact_id LIKE 'local-%'
|
||||
AND attempt.finished_at_ms IS NOT NULL
|
||||
AND attempt.finished_at_ms <= :cutoffMs
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM "${COMPLETION_RECEIPT_JOURNAL_TABLE}" AS receipt
|
||||
WHERE receipt.attempt_id = attempt.id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM "${LOCAL_ARTIFACT_RETENTION_TABLE}" AS retained
|
||||
WHERE retained.attempt_id = attempt.id
|
||||
)
|
||||
${cursorPredicate}
|
||||
ORDER BY attempt.finished_at_ms ASC, attempt.id ASC
|
||||
LIMIT :fetchLimit`,
|
||||
{ type: QueryTypes.SELECT, replacements },
|
||||
);
|
||||
const truncated = rows.length > limit;
|
||||
const selected = truncated ? rows.slice(0, limit) : rows;
|
||||
const candidates: LocalArtifactRetentionCandidate[] = selected.map((row) =>
|
||||
normalizeLocalArtifactRetentionCandidate({
|
||||
attemptId: row.attempt_id,
|
||||
logArtifactId: row.log_artifact_id,
|
||||
finishedAtMs: Number(row.finished_at_ms),
|
||||
}),
|
||||
);
|
||||
const last = candidates[candidates.length - 1];
|
||||
return Object.freeze({
|
||||
candidates: Object.freeze(candidates),
|
||||
truncated,
|
||||
...(truncated && last
|
||||
? {
|
||||
nextCursor: Object.freeze({
|
||||
finishedAtMs: last.finishedAtMs,
|
||||
attemptId: last.attemptId,
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
async record(
|
||||
value: LocalArtifactRetentionRecord,
|
||||
): Promise<'inserted' | 'existing'> {
|
||||
const record = normalizeLocalArtifactRetentionRecord(value);
|
||||
const row: LocalArtifactRetentionRow = {
|
||||
attemptId: record.attemptId,
|
||||
logArtifactId: record.logArtifactId,
|
||||
finishedAtMs: record.finishedAtMs,
|
||||
eligibleAtMs: record.eligibleAtMs,
|
||||
disposition: record.disposition,
|
||||
bytesReclaimed: record.bytesReclaimed,
|
||||
recordedAtMs: record.recordedAtMs,
|
||||
};
|
||||
try {
|
||||
await this.retention.create(row);
|
||||
return 'inserted';
|
||||
} catch (error) {
|
||||
if (!(error instanceof UniqueConstraintError)) throw error;
|
||||
}
|
||||
const existing = await this.retention.findByPk(record.attemptId, {
|
||||
raw: true,
|
||||
});
|
||||
if (existing && sameRetirementIdentity(rowToRecord(existing), record))
|
||||
return 'existing';
|
||||
throw new LocalArtifactRetentionRecordConflictError();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Sequelize,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE } from '../../../migrations/0013-local-execution-context-recipes';
|
||||
import {
|
||||
assertLocalExecutionContextRef,
|
||||
createLocalExecutionContextRecipeRecord,
|
||||
localExecutionContextRecipeDigest,
|
||||
normalizeLocalExecutionContextRecipe,
|
||||
type LocalExecutionContextRecipe,
|
||||
} from '../../domain/localExecutionContextRecipe';
|
||||
import type {
|
||||
InsertLocalExecutionContextRecipeResult,
|
||||
LocalExecutionContextRecipeRepository,
|
||||
} from '../../ports/localExecutionContextRecipeRepository';
|
||||
|
||||
interface LocalExecutionContextRecipeRow {
|
||||
contextRef: string;
|
||||
environmentRecipe: string;
|
||||
contentDigest: string;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface LocalExecutionContextRecipeInstance
|
||||
extends Model<LocalExecutionContextRecipeRow, LocalExecutionContextRecipeRow>,
|
||||
LocalExecutionContextRecipeRow {}
|
||||
|
||||
export class LocalExecutionContextRecipeConflictError extends Error {
|
||||
constructor(readonly contextRef: string) {
|
||||
super(`Local execution context recipe ${contextRef} is immutable`);
|
||||
this.name = 'LocalExecutionContextRecipeConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalExecutionContextRecipeCorruptError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'LocalExecutionContextRecipeCorruptError';
|
||||
}
|
||||
}
|
||||
|
||||
function defineRecipeModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<LocalExecutionContextRecipeInstance> {
|
||||
return database.define<LocalExecutionContextRecipeInstance>(
|
||||
'Ql3LocalExecutionContextRecipe',
|
||||
{
|
||||
contextRef: {
|
||||
field: 'context_ref',
|
||||
type: DataTypes.STRING(512),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
environmentRecipe: {
|
||||
field: 'environment_recipe',
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: false,
|
||||
},
|
||||
contentDigest: {
|
||||
field: 'content_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
function rowToRecipe(
|
||||
row: LocalExecutionContextRecipeRow,
|
||||
): LocalExecutionContextRecipe {
|
||||
let environment: unknown;
|
||||
try {
|
||||
environment = JSON.parse(row.environmentRecipe);
|
||||
} catch {
|
||||
throw new LocalExecutionContextRecipeCorruptError(
|
||||
'Stored local execution context recipe is not valid JSON',
|
||||
);
|
||||
}
|
||||
try {
|
||||
const normalized = normalizeLocalExecutionContextRecipe({
|
||||
contextRef: row.contextRef,
|
||||
environment: environment as LocalExecutionContextRecipe['environment'],
|
||||
});
|
||||
if (JSON.stringify(normalized.environment) !== row.environmentRecipe) {
|
||||
throw new LocalExecutionContextRecipeCorruptError(
|
||||
'Stored local execution context recipe is not canonical',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!/^[0-9a-f]{64}$/.test(row.contentDigest) ||
|
||||
localExecutionContextRecipeDigest(normalized) !== row.contentDigest
|
||||
) {
|
||||
throw new LocalExecutionContextRecipeCorruptError(
|
||||
'Stored local execution context recipe digest does not match',
|
||||
);
|
||||
}
|
||||
return createLocalExecutionContextRecipeRecord(
|
||||
normalized,
|
||||
Number(row.createdAtMs),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalExecutionContextRecipeCorruptError) throw error;
|
||||
throw new LocalExecutionContextRecipeCorruptError(
|
||||
`Stored local execution context recipe is invalid: ${
|
||||
error instanceof Error ? error.message : 'unknown validation error'
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeLocalExecutionContextRecipeRepository
|
||||
implements LocalExecutionContextRecipeRepository
|
||||
{
|
||||
private readonly recipe: ModelStatic<LocalExecutionContextRecipeInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy local context recipe repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.recipe = defineRecipeModel(database);
|
||||
}
|
||||
|
||||
async resolve(
|
||||
contextRef: string,
|
||||
): Promise<LocalExecutionContextRecipe | null> {
|
||||
assertLocalExecutionContextRef(contextRef);
|
||||
const row = (await this.recipe.findByPk(contextRef, {
|
||||
raw: true,
|
||||
})) as unknown as LocalExecutionContextRecipeRow | null;
|
||||
return row ? rowToRecipe(row) : null;
|
||||
}
|
||||
|
||||
async insert(
|
||||
recipe: LocalExecutionContextRecipe,
|
||||
createdAtMs: number,
|
||||
): Promise<InsertLocalExecutionContextRecipeResult> {
|
||||
const record = createLocalExecutionContextRecipeRecord(recipe, createdAtMs);
|
||||
const values: LocalExecutionContextRecipeRow = {
|
||||
contextRef: record.contextRef,
|
||||
environmentRecipe: JSON.stringify(record.environment),
|
||||
contentDigest: record.contentDigest,
|
||||
createdAtMs: record.createdAtMs,
|
||||
};
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
try {
|
||||
await this.recipe.create(values);
|
||||
return 'inserted';
|
||||
} catch (error) {
|
||||
if (error instanceof UniqueConstraintError) {
|
||||
const existing = await this.resolve(record.contextRef);
|
||||
if (
|
||||
existing &&
|
||||
localExecutionContextRecipeDigest(existing) === record.contentDigest
|
||||
) {
|
||||
return 'idempotent';
|
||||
}
|
||||
throw new LocalExecutionContextRecipeConflictError(record.contextRef);
|
||||
}
|
||||
if (errorCode(error) === 'SQLITE_BUSY' && attempt < 4) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new Error('Local context recipe insert retry budget exhausted');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { LOCAL_SECRET_ENVELOPE_TABLE } from '../../../migrations/0014-local-secret-envelopes';
|
||||
import {
|
||||
LOCAL_SECRET_ALGORITHM,
|
||||
LocalSecretUnavailableError,
|
||||
LocalSecretVersionConflictError,
|
||||
assertLocalSecretMutationId,
|
||||
assertLocalSecretName,
|
||||
assertLocalSecretProjectId,
|
||||
createLocalSecretRef,
|
||||
normalizeLocalSecretEnvelope,
|
||||
type LocalSecretEnvelope,
|
||||
type LocalSecretReference,
|
||||
} from '../../domain/localSecret';
|
||||
import type {
|
||||
AppendLocalSecretEnvelopeCommand,
|
||||
AppendLocalSecretEnvelopeResult,
|
||||
LocalSecretEnvelopeRepository,
|
||||
} from '../../ports/localSecretEnvelopeRepository';
|
||||
|
||||
const MAX_BATCH_SIZE = 64;
|
||||
const RETRY_ATTEMPTS = 5;
|
||||
|
||||
interface LocalSecretEnvelopeRow {
|
||||
projectId: string;
|
||||
name: string;
|
||||
version: number;
|
||||
mutationId: string;
|
||||
keyId: string;
|
||||
algorithm: string;
|
||||
nonce: Buffer;
|
||||
ciphertext: Buffer;
|
||||
authTag: Buffer;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface LocalSecretEnvelopeInstance
|
||||
extends Model<LocalSecretEnvelopeRow, LocalSecretEnvelopeRow>,
|
||||
LocalSecretEnvelopeRow {}
|
||||
|
||||
interface ResolvedSecretRow {
|
||||
position: number;
|
||||
project_id: string | null;
|
||||
secret_name: string | null;
|
||||
version: number | null;
|
||||
mutation_id: string | null;
|
||||
key_id: string | null;
|
||||
algorithm: string | null;
|
||||
nonce: Buffer | null;
|
||||
ciphertext: Buffer | null;
|
||||
auth_tag: Buffer | null;
|
||||
created_at_ms: number | string | null;
|
||||
}
|
||||
|
||||
function defineLocalSecretEnvelopeModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<LocalSecretEnvelopeInstance> {
|
||||
return database.define<LocalSecretEnvelopeInstance>(
|
||||
'Ql3LocalSecretEnvelope',
|
||||
{
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
name: {
|
||||
field: 'secret_name',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
version: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
mutationId: {
|
||||
field: 'mutation_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
keyId: {
|
||||
field: 'key_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
},
|
||||
algorithm: {
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
nonce: { type: DataTypes.BLOB, allowNull: false },
|
||||
ciphertext: { type: DataTypes.BLOB, allowNull: false },
|
||||
authTag: {
|
||||
field: 'auth_tag',
|
||||
type: DataTypes.BLOB,
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: LOCAL_SECRET_ENVELOPE_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function rowToEnvelope(row: LocalSecretEnvelopeRow): LocalSecretEnvelope {
|
||||
try {
|
||||
return normalizeLocalSecretEnvelope({
|
||||
projectId: row.projectId,
|
||||
name: row.name,
|
||||
version: Number(row.version),
|
||||
mutationId: row.mutationId,
|
||||
keyId: row.keyId,
|
||||
algorithm: row.algorithm as typeof LOCAL_SECRET_ALGORITHM,
|
||||
nonce: Buffer.from(row.nonce).toString('base64url'),
|
||||
ciphertext: Buffer.from(row.ciphertext).toString('base64url'),
|
||||
authTag: Buffer.from(row.authTag).toString('base64url'),
|
||||
createdAtMs: Number(row.createdAtMs),
|
||||
});
|
||||
} catch {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function resolvedRowToEnvelope(
|
||||
row: ResolvedSecretRow,
|
||||
): LocalSecretEnvelope | null {
|
||||
if (row.version === null) return null;
|
||||
if (
|
||||
row.project_id === null ||
|
||||
row.secret_name === null ||
|
||||
row.mutation_id === null ||
|
||||
row.key_id === null ||
|
||||
row.algorithm === null ||
|
||||
row.nonce === null ||
|
||||
row.ciphertext === null ||
|
||||
row.auth_tag === null ||
|
||||
row.created_at_ms === null
|
||||
) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
return rowToEnvelope({
|
||||
projectId: row.project_id,
|
||||
name: row.secret_name,
|
||||
version: row.version,
|
||||
mutationId: row.mutation_id,
|
||||
keyId: row.key_id,
|
||||
algorithm: row.algorithm,
|
||||
nonce: row.nonce,
|
||||
ciphertext: row.ciphertext,
|
||||
authTag: row.auth_tag,
|
||||
createdAtMs: row.created_at_ms,
|
||||
});
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
function assertExpectedVersion(value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0 || value >= 2_147_483_647) {
|
||||
throw new TypeError('Local Secret expected current version is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeLocalSecretEnvelopeRepository
|
||||
implements LocalSecretEnvelopeRepository
|
||||
{
|
||||
private readonly envelope: ModelStatic<LocalSecretEnvelopeInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Local Secret envelope repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.envelope = defineLocalSecretEnvelopeModel(database);
|
||||
}
|
||||
|
||||
async append(
|
||||
command: AppendLocalSecretEnvelopeCommand,
|
||||
): Promise<AppendLocalSecretEnvelopeResult> {
|
||||
assertExpectedVersion(command.expectedCurrentVersion);
|
||||
const envelope = normalizeLocalSecretEnvelope(command.envelope);
|
||||
if (envelope.version !== command.expectedCurrentVersion + 1) {
|
||||
throw new LocalSecretVersionConflictError();
|
||||
}
|
||||
const values = this.values(envelope);
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const replay = await this.envelope.findOne({
|
||||
where: {
|
||||
projectId: envelope.projectId,
|
||||
name: envelope.name,
|
||||
mutationId: envelope.mutationId,
|
||||
},
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (replay) {
|
||||
return { status: 'existing', envelope: rowToEnvelope(replay) };
|
||||
}
|
||||
const current = await this.envelope.findOne({
|
||||
where: { projectId: envelope.projectId, name: envelope.name },
|
||||
order: [['version', 'DESC']],
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
const currentVersion = current ? Number(current.version) : 0;
|
||||
if (currentVersion !== command.expectedCurrentVersion) {
|
||||
throw new LocalSecretVersionConflictError();
|
||||
}
|
||||
await this.envelope.create(values, { transaction });
|
||||
return { status: 'inserted', envelope };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalSecretVersionConflictError) throw error;
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
|
||||
async findByMutation(
|
||||
projectId: string,
|
||||
name: string,
|
||||
mutationId: string,
|
||||
): Promise<LocalSecretEnvelope | null> {
|
||||
assertLocalSecretProjectId(projectId);
|
||||
assertLocalSecretName(name);
|
||||
assertLocalSecretMutationId(mutationId);
|
||||
const row = await this.envelope.findOne({
|
||||
where: { projectId, name, mutationId },
|
||||
raw: true,
|
||||
});
|
||||
return row ? rowToEnvelope(row) : null;
|
||||
}
|
||||
|
||||
async resolveMany(
|
||||
references: readonly LocalSecretReference[],
|
||||
): Promise<readonly (LocalSecretEnvelope | null)[]> {
|
||||
if (!Array.isArray(references) || references.length > MAX_BATCH_SIZE) {
|
||||
throw new RangeError('Local Secret batch is too large');
|
||||
}
|
||||
if (references.length === 0) return Object.freeze([]);
|
||||
const replacements: Record<string, string | number | null> = {};
|
||||
const requestedValues = references.map((reference, position) => {
|
||||
createLocalSecretRef(reference);
|
||||
replacements[`position${position}`] = position;
|
||||
replacements[`project${position}`] = reference.projectId;
|
||||
replacements[`name${position}`] = reference.name;
|
||||
replacements[`version${position}`] = reference.version ?? null;
|
||||
return `(:position${position}, :project${position}, :name${position}, :version${position})`;
|
||||
});
|
||||
const rows = await this.database.query<ResolvedSecretRow>(
|
||||
`WITH requested(position, project_id, secret_name, requested_version) AS (
|
||||
VALUES ${requestedValues.join(', ')}
|
||||
)
|
||||
SELECT requested.position,
|
||||
envelope.project_id,
|
||||
envelope.secret_name,
|
||||
envelope.version,
|
||||
envelope.mutation_id,
|
||||
envelope.key_id,
|
||||
envelope.algorithm,
|
||||
envelope.nonce,
|
||||
envelope.ciphertext,
|
||||
envelope.auth_tag,
|
||||
envelope.created_at_ms
|
||||
FROM requested
|
||||
LEFT JOIN "${LOCAL_SECRET_ENVELOPE_TABLE}" AS envelope
|
||||
ON envelope.project_id = requested.project_id
|
||||
AND envelope.secret_name = requested.secret_name
|
||||
AND envelope.version = COALESCE(
|
||||
requested.requested_version,
|
||||
(SELECT MAX(current.version)
|
||||
FROM "${LOCAL_SECRET_ENVELOPE_TABLE}" AS current
|
||||
WHERE current.project_id = requested.project_id
|
||||
AND current.secret_name = requested.secret_name)
|
||||
)
|
||||
ORDER BY requested.position ASC`,
|
||||
{ type: QueryTypes.SELECT, replacements },
|
||||
);
|
||||
if (rows.length !== references.length) {
|
||||
throw new LocalSecretUnavailableError();
|
||||
}
|
||||
return Object.freeze(rows.map(resolvedRowToEnvelope));
|
||||
}
|
||||
|
||||
private values(envelope: LocalSecretEnvelope): LocalSecretEnvelopeRow {
|
||||
return {
|
||||
projectId: envelope.projectId,
|
||||
name: envelope.name,
|
||||
version: envelope.version,
|
||||
mutationId: envelope.mutationId,
|
||||
keyId: envelope.keyId,
|
||||
algorithm: envelope.algorithm,
|
||||
nonce: Buffer.from(envelope.nonce, 'base64url'),
|
||||
ciphertext: Buffer.from(envelope.ciphertext, 'base64url'),
|
||||
authTag: Buffer.from(envelope.authTag, 'base64url'),
|
||||
createdAtMs: envelope.createdAtMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Op,
|
||||
Sequelize,
|
||||
type WhereOptions,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import type { ExecutionStopKind, ExecutorType } from '../../domain/execution';
|
||||
import type {
|
||||
PrimaryCancellationAttemptReference,
|
||||
PrimaryCancellationCandidate,
|
||||
PrimaryCancellationCursor,
|
||||
PrimaryCancellationPage,
|
||||
PrimaryCancellationSource,
|
||||
} from '../../ports/primaryCancellationSource';
|
||||
import { MAX_PRIMARY_CANCELLATION_BATCH_SIZE } from '../../ports/primaryCancellationSource';
|
||||
|
||||
interface CancellationRunRow {
|
||||
id: string;
|
||||
executionOwner: string;
|
||||
status: string;
|
||||
cancelRequestedAtMs: number | null;
|
||||
cancelReason: string | null;
|
||||
}
|
||||
|
||||
interface CancellationRequestedRunRow extends CancellationRunRow {
|
||||
cancelRequestedAtMs: number;
|
||||
cancelReason: string;
|
||||
}
|
||||
|
||||
interface CancellationAttemptRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
attempt: number;
|
||||
status: string;
|
||||
executorType: string;
|
||||
executorHandle: string | null;
|
||||
pid: number | null;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
interface CancellationRunInstance
|
||||
extends Model<CancellationRunRow, CancellationRunRow>,
|
||||
CancellationRunRow {}
|
||||
interface CancellationAttemptInstance
|
||||
extends Model<CancellationAttemptRow, CancellationAttemptRow>,
|
||||
CancellationAttemptRow {}
|
||||
|
||||
function defineCancellationRunModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationRunInstance> {
|
||||
return database.define<CancellationRunInstance>(
|
||||
'Ql3PrimaryCancellationRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
executionOwner: {
|
||||
field: 'execution_owner',
|
||||
type: DataTypes.STRING(16),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
cancelRequestedAtMs: {
|
||||
field: 'cancel_requested_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
cancelReason: {
|
||||
field: 'cancel_reason',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function defineCancellationAttemptModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<CancellationAttemptInstance> {
|
||||
return database.define<CancellationAttemptInstance>(
|
||||
'Ql3PrimaryCancellationAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
runId: {
|
||||
field: 'run_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
attempt: { type: DataTypes.INTEGER, allowNull: false },
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
executorType: {
|
||||
field: 'executor_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
executorHandle: {
|
||||
field: 'executor_handle',
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: true,
|
||||
},
|
||||
pid: { type: DataTypes.INTEGER, allowNull: true },
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: RUN_ATTEMPT_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function assertCursor(cursor: PrimaryCancellationCursor): void {
|
||||
if (!Number.isSafeInteger(cursor.requestedAtMs) || cursor.requestedAtMs < 0) {
|
||||
throw new RangeError('cursor.requestedAtMs must be a non-negative integer');
|
||||
}
|
||||
if (!cursor.runId || cursor.runId.length > 36) {
|
||||
throw new RangeError('cursor.runId must be between 1 and 36 characters');
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizePrimaryCancellationSource
|
||||
implements PrimaryCancellationSource
|
||||
{
|
||||
private readonly run: ModelStatic<CancellationRunInstance>;
|
||||
private readonly attempt: ModelStatic<CancellationAttemptInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.run = defineCancellationRunModel(database);
|
||||
this.attempt = defineCancellationAttemptModel(database);
|
||||
}
|
||||
|
||||
async listCandidates({
|
||||
cursor,
|
||||
limit = 32,
|
||||
}: {
|
||||
cursor?: PrimaryCancellationCursor;
|
||||
limit?: number;
|
||||
} = {}): Promise<PrimaryCancellationPage> {
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_PRIMARY_CANCELLATION_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_PRIMARY_CANCELLATION_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (cursor) assertCursor(cursor);
|
||||
|
||||
const where: WhereOptions<CancellationRunRow> = {
|
||||
executionOwner: 'runtime',
|
||||
status: {
|
||||
[Op.in]: [
|
||||
'created',
|
||||
'queued',
|
||||
'dispatching',
|
||||
'running',
|
||||
'waiting_approval',
|
||||
'retry_wait',
|
||||
'lost',
|
||||
],
|
||||
},
|
||||
cancelRequestedAtMs: {
|
||||
[Op.ne]: null,
|
||||
},
|
||||
cancelReason: { [Op.ne]: null },
|
||||
...(cursor === undefined
|
||||
? {}
|
||||
: {
|
||||
[Op.or]: [
|
||||
{ cancelRequestedAtMs: { [Op.gt]: cursor.requestedAtMs } },
|
||||
{
|
||||
cancelRequestedAtMs: cursor.requestedAtMs,
|
||||
id: { [Op.gt]: cursor.runId },
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
const runRows = (await this.run.findAll({
|
||||
attributes: ['id', 'cancelRequestedAtMs', 'cancelReason'],
|
||||
where,
|
||||
order: [
|
||||
['cancelRequestedAtMs', 'ASC'],
|
||||
['id', 'ASC'],
|
||||
],
|
||||
limit: limit + 1,
|
||||
raw: true,
|
||||
})) as unknown as CancellationRunRow[];
|
||||
const truncated = runRows.length > limit;
|
||||
const boundedRuns = runRows
|
||||
.filter(
|
||||
(run): run is CancellationRequestedRunRow =>
|
||||
run.cancelRequestedAtMs !== null && run.cancelReason !== null,
|
||||
)
|
||||
.slice(0, limit);
|
||||
if (boundedRuns.length === 0) {
|
||||
return {
|
||||
candidates: [],
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
};
|
||||
}
|
||||
|
||||
const maxAttemptRows = limit * 2;
|
||||
const attemptRows = (await this.attempt.findAll({
|
||||
attributes: [
|
||||
'id',
|
||||
'runId',
|
||||
'attempt',
|
||||
'executorType',
|
||||
'executorHandle',
|
||||
'pid',
|
||||
],
|
||||
where: {
|
||||
runId: { [Op.in]: boundedRuns.map((run) => run.id) },
|
||||
status: { [Op.in]: ['claimed', 'starting', 'running'] },
|
||||
},
|
||||
order: [
|
||||
['runId', 'ASC'],
|
||||
['attempt', 'DESC'],
|
||||
['createdAtMs', 'DESC'],
|
||||
['id', 'DESC'],
|
||||
],
|
||||
limit: maxAttemptRows + 1,
|
||||
raw: true,
|
||||
})) as unknown as CancellationAttemptRow[];
|
||||
if (attemptRows.length > maxAttemptRows) {
|
||||
return {
|
||||
candidates: [],
|
||||
truncated,
|
||||
unsafeAttemptOverflow: true,
|
||||
};
|
||||
}
|
||||
|
||||
const attemptsByRun = new Map<
|
||||
string,
|
||||
PrimaryCancellationAttemptReference[]
|
||||
>();
|
||||
for (const attempt of attemptRows) {
|
||||
const references = attemptsByRun.get(attempt.runId) ?? [];
|
||||
references.push({
|
||||
attemptId: attempt.id,
|
||||
executorType: attempt.executorType as ExecutorType,
|
||||
...(attempt.executorHandle === null
|
||||
? {}
|
||||
: { executorHandle: attempt.executorHandle }),
|
||||
...(attempt.pid === null ? {} : { pid: attempt.pid }),
|
||||
});
|
||||
attemptsByRun.set(attempt.runId, references);
|
||||
}
|
||||
|
||||
const candidates: PrimaryCancellationCandidate[] = boundedRuns.map(
|
||||
(run) => ({
|
||||
runId: run.id,
|
||||
requestedAtMs: run.cancelRequestedAtMs,
|
||||
reason: run.cancelReason as ExecutionStopKind,
|
||||
attempts: attemptsByRun.get(run.id) ?? [],
|
||||
}),
|
||||
);
|
||||
const last = boundedRuns[boundedRuns.length - 1];
|
||||
return {
|
||||
candidates,
|
||||
truncated,
|
||||
unsafeAttemptOverflow: false,
|
||||
nextCursor: {
|
||||
requestedAtMs: last.cancelRequestedAtMs,
|
||||
runId: last.id,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
type ModelStatic,
|
||||
Op,
|
||||
type Sequelize,
|
||||
type Transaction,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { RUNNING_INSTANCE_TABLE } from '../../../migrations/0003-running-instance-run-reference';
|
||||
import type { RunAttemptStatus, RunStatus } from '../../domain/run';
|
||||
import { parseLegacyLogOutputRef } from '../../compatibility/legacyLogOutputRef';
|
||||
import type {
|
||||
SequelizeRunProjectionContext,
|
||||
SequelizeRunProjectionParticipant,
|
||||
} from './projectedRunRepository';
|
||||
|
||||
const CRONTAB_TABLE = 'Crontabs';
|
||||
const CRONTAB_STATUS_RUNNING = 0;
|
||||
const CRONTAB_STATUS_IDLE = 1;
|
||||
const CRONTAB_STATUS_QUEUED = 3;
|
||||
const INSTANCE_STATUS_RUNNING = 0;
|
||||
const INSTANCE_STATUS_FINISHED = 1;
|
||||
const INSTANCE_STATUS_STOPPED = 2;
|
||||
const INSTANCE_STATUS_ERROR = 3;
|
||||
|
||||
const RUNNING_RUN_STATUSES: readonly RunStatus[] = [
|
||||
'running',
|
||||
'waiting_approval',
|
||||
];
|
||||
const QUEUED_RUN_STATUSES: readonly RunStatus[] = [
|
||||
'created',
|
||||
'queued',
|
||||
'dispatching',
|
||||
'retry_wait',
|
||||
];
|
||||
const TERMINAL_RUN_STATUSES: readonly RunStatus[] = [
|
||||
'lost',
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
];
|
||||
|
||||
interface ProjectionRunRow {
|
||||
id: string;
|
||||
legacyCronId: number | null;
|
||||
executionOwner: string;
|
||||
status: RunStatus;
|
||||
outputRef: string | null;
|
||||
createdAtMs: number;
|
||||
startedAtMs: number | null;
|
||||
finishedAtMs: number | null;
|
||||
}
|
||||
|
||||
interface ProjectionAttemptRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
attempt: number;
|
||||
status: RunAttemptStatus;
|
||||
pid: number | null;
|
||||
createdAtMs: number;
|
||||
startedAtMs: number | null;
|
||||
finishedAtMs: number | null;
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
interface ProjectionCrontabRow {
|
||||
id: number;
|
||||
status: number | null;
|
||||
pid: number | null;
|
||||
logPath: string | null;
|
||||
lastRunningTime: number | null;
|
||||
lastExecutionTime: number | null;
|
||||
}
|
||||
|
||||
interface ProjectionInstanceRow {
|
||||
id?: number;
|
||||
cronId: number;
|
||||
runId: string | null;
|
||||
attemptId: string | null;
|
||||
pid: number | null;
|
||||
logPath: string | null;
|
||||
startedAt: number;
|
||||
finishedAt: number | null;
|
||||
status: number;
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
interface ProjectionRunInstance
|
||||
extends Model<ProjectionRunRow, ProjectionRunRow>,
|
||||
ProjectionRunRow {}
|
||||
interface ProjectionAttemptInstance
|
||||
extends Model<ProjectionAttemptRow, ProjectionAttemptRow>,
|
||||
ProjectionAttemptRow {}
|
||||
interface ProjectionCrontabInstance
|
||||
extends Model<ProjectionCrontabRow, ProjectionCrontabRow>,
|
||||
ProjectionCrontabRow {}
|
||||
interface ProjectionInstanceInstance
|
||||
extends Model<ProjectionInstanceRow, ProjectionInstanceRow>,
|
||||
ProjectionInstanceRow {}
|
||||
|
||||
interface ProjectionModels {
|
||||
run: ModelStatic<ProjectionRunInstance>;
|
||||
attempt: ModelStatic<ProjectionAttemptInstance>;
|
||||
crontab: ModelStatic<ProjectionCrontabInstance>;
|
||||
instance: ModelStatic<ProjectionInstanceInstance>;
|
||||
}
|
||||
|
||||
interface SelectedRun {
|
||||
run: ProjectionRunRow;
|
||||
attempt: ProjectionAttemptRow | null;
|
||||
}
|
||||
|
||||
function defineProjectionModels(database: Sequelize): ProjectionModels {
|
||||
const common = { timestamps: false, freezeTableName: true } as const;
|
||||
const run = database.define<ProjectionRunInstance>(
|
||||
'Ql3PrimaryCronProjectionRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
legacyCronId: { field: 'legacy_cron_id', type: DataTypes.INTEGER },
|
||||
executionOwner: {
|
||||
field: 'execution_owner',
|
||||
type: DataTypes.STRING(16),
|
||||
},
|
||||
status: { type: DataTypes.STRING(32) },
|
||||
outputRef: { field: 'output_ref', type: DataTypes.STRING(512) },
|
||||
createdAtMs: { field: 'created_at_ms', type: DataTypes.BIGINT },
|
||||
startedAtMs: { field: 'started_at_ms', type: DataTypes.BIGINT },
|
||||
finishedAtMs: { field: 'finished_at_ms', type: DataTypes.BIGINT },
|
||||
},
|
||||
{ ...common, tableName: RUN_TABLE },
|
||||
);
|
||||
const attempt = database.define<ProjectionAttemptInstance>(
|
||||
'Ql3PrimaryCronProjectionAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), primaryKey: true },
|
||||
runId: { field: 'run_id', type: DataTypes.STRING(36) },
|
||||
attempt: { type: DataTypes.INTEGER },
|
||||
status: { type: DataTypes.STRING(32) },
|
||||
pid: { type: DataTypes.INTEGER },
|
||||
createdAtMs: { field: 'created_at_ms', type: DataTypes.BIGINT },
|
||||
startedAtMs: { field: 'started_at_ms', type: DataTypes.BIGINT },
|
||||
finishedAtMs: { field: 'finished_at_ms', type: DataTypes.BIGINT },
|
||||
exitCode: { field: 'exit_code', type: DataTypes.INTEGER },
|
||||
},
|
||||
{ ...common, tableName: RUN_ATTEMPT_TABLE },
|
||||
);
|
||||
const crontab = database.define<ProjectionCrontabInstance>(
|
||||
'Ql3PrimaryCronProjectionCrontab',
|
||||
{
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true },
|
||||
status: { type: DataTypes.INTEGER },
|
||||
pid: { type: DataTypes.INTEGER },
|
||||
logPath: { field: 'log_path', type: DataTypes.STRING },
|
||||
lastRunningTime: {
|
||||
field: 'last_running_time',
|
||||
type: DataTypes.INTEGER,
|
||||
},
|
||||
lastExecutionTime: {
|
||||
field: 'last_execution_time',
|
||||
type: DataTypes.INTEGER,
|
||||
},
|
||||
},
|
||||
{ ...common, tableName: CRONTAB_TABLE },
|
||||
);
|
||||
const instance = database.define<ProjectionInstanceInstance>(
|
||||
'Ql3PrimaryCronProjectionInstance',
|
||||
{
|
||||
id: { type: DataTypes.INTEGER, primaryKey: true, autoIncrement: true },
|
||||
cronId: { field: 'cron_id', type: DataTypes.INTEGER },
|
||||
runId: { field: 'run_id', type: DataTypes.STRING(36) },
|
||||
attemptId: { field: 'attempt_id', type: DataTypes.STRING(36) },
|
||||
pid: { type: DataTypes.INTEGER },
|
||||
logPath: { field: 'log_path', type: DataTypes.STRING },
|
||||
startedAt: { field: 'started_at', type: DataTypes.INTEGER },
|
||||
finishedAt: { field: 'finished_at', type: DataTypes.INTEGER },
|
||||
status: { type: DataTypes.INTEGER },
|
||||
exitCode: { field: 'exit_code', type: DataTypes.INTEGER },
|
||||
},
|
||||
{ ...common, tableName: RUNNING_INSTANCE_TABLE },
|
||||
);
|
||||
return { run, attempt, crontab, instance };
|
||||
}
|
||||
|
||||
function toUnixSeconds(milliseconds: number): number {
|
||||
return Math.floor(milliseconds / 1000);
|
||||
}
|
||||
|
||||
function instanceStatus(status: RunAttemptStatus): number | null {
|
||||
switch (status) {
|
||||
case 'claimed':
|
||||
return null;
|
||||
case 'starting':
|
||||
case 'running':
|
||||
return INSTANCE_STATUS_RUNNING;
|
||||
case 'succeeded':
|
||||
return INSTANCE_STATUS_FINISHED;
|
||||
case 'cancelled':
|
||||
return INSTANCE_STATUS_STOPPED;
|
||||
case 'failed':
|
||||
case 'timed_out':
|
||||
case 'lost':
|
||||
return INSTANCE_STATUS_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
function isAttemptActive(status: RunAttemptStatus): boolean {
|
||||
return status === 'starting' || status === 'running';
|
||||
}
|
||||
|
||||
/**
|
||||
* Projects runtime-owned Run state into the legacy UI tables before the same
|
||||
* SQLite transaction commits. It never projects legacy-owned Shadow Runs.
|
||||
*/
|
||||
export class PrimaryCronProjection
|
||||
implements SequelizeRunProjectionParticipant
|
||||
{
|
||||
private readonly models: ProjectionModels;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.models = defineProjectionModels(database);
|
||||
}
|
||||
|
||||
async apply(context: SequelizeRunProjectionContext): Promise<void> {
|
||||
const cronIds = new Set<number>();
|
||||
for (const attemptId of context.changedAttemptIds) {
|
||||
const cronId = await this.projectAttempt(attemptId, context.transaction);
|
||||
if (cronId !== null) cronIds.add(cronId);
|
||||
}
|
||||
for (const runId of context.changedRunIds) {
|
||||
const run = await context.runs.findRunById(runId);
|
||||
if (run?.executionOwner === 'runtime' && run.legacyCronId !== undefined) {
|
||||
cronIds.add(run.legacyCronId);
|
||||
}
|
||||
}
|
||||
for (const cronId of cronIds) {
|
||||
await this.projectCrontab(cronId, context.transaction);
|
||||
}
|
||||
}
|
||||
|
||||
private async projectAttempt(
|
||||
attemptId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<number | null> {
|
||||
const attempt = await this.models.attempt.findByPk(attemptId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!attempt) return null;
|
||||
const run = await this.models.run.findByPk(attempt.runId, {
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!run || run.executionOwner !== 'runtime' || run.legacyCronId === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const status = instanceStatus(attempt.status);
|
||||
if (status === null) return run.legacyCronId;
|
||||
const logPath = parseLegacyLogOutputRef(run.outputRef ?? undefined);
|
||||
const values: ProjectionInstanceRow = {
|
||||
cronId: run.legacyCronId,
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
pid: attempt.pid,
|
||||
logPath,
|
||||
startedAt: toUnixSeconds(
|
||||
attempt.startedAtMs ?? run.startedAtMs ?? attempt.createdAtMs,
|
||||
),
|
||||
finishedAt:
|
||||
attempt.finishedAtMs === null
|
||||
? null
|
||||
: toUnixSeconds(attempt.finishedAtMs),
|
||||
status,
|
||||
exitCode: attempt.exitCode,
|
||||
};
|
||||
const existing = await this.models.instance.findOne({
|
||||
where: { attemptId: attempt.id },
|
||||
transaction,
|
||||
});
|
||||
if (existing) {
|
||||
await existing.update(values, { transaction });
|
||||
} else {
|
||||
await this.models.instance.create(values, { transaction });
|
||||
}
|
||||
return run.legacyCronId;
|
||||
}
|
||||
|
||||
private async projectCrontab(
|
||||
cronId: number,
|
||||
transaction: Transaction,
|
||||
): Promise<void> {
|
||||
const running = await this.findSelectedRun(
|
||||
cronId,
|
||||
RUNNING_RUN_STATUSES,
|
||||
transaction,
|
||||
);
|
||||
if (running) {
|
||||
await this.updateCrontab(
|
||||
cronId,
|
||||
CRONTAB_STATUS_RUNNING,
|
||||
running,
|
||||
transaction,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const queued = await this.findSelectedRun(
|
||||
cronId,
|
||||
QUEUED_RUN_STATUSES,
|
||||
transaction,
|
||||
);
|
||||
if (queued) {
|
||||
await this.updateCrontab(
|
||||
cronId,
|
||||
CRONTAB_STATUS_QUEUED,
|
||||
queued,
|
||||
transaction,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const terminal = await this.findSelectedRun(
|
||||
cronId,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
transaction,
|
||||
);
|
||||
await this.updateCrontab(
|
||||
cronId,
|
||||
CRONTAB_STATUS_IDLE,
|
||||
terminal,
|
||||
transaction,
|
||||
);
|
||||
}
|
||||
|
||||
private async findSelectedRun(
|
||||
cronId: number,
|
||||
statuses: readonly RunStatus[],
|
||||
transaction: Transaction,
|
||||
): Promise<SelectedRun | null> {
|
||||
const run = await this.models.run.findOne({
|
||||
where: {
|
||||
legacyCronId: cronId,
|
||||
executionOwner: 'runtime',
|
||||
status: { [Op.in]: [...statuses] },
|
||||
},
|
||||
order: [
|
||||
['createdAtMs', 'DESC'],
|
||||
['id', 'DESC'],
|
||||
],
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (!run) return null;
|
||||
const attempt = await this.models.attempt.findOne({
|
||||
where: { runId: run.id },
|
||||
order: [
|
||||
['attempt', 'DESC'],
|
||||
['id', 'DESC'],
|
||||
],
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async updateCrontab(
|
||||
cronId: number,
|
||||
status: number,
|
||||
selected: SelectedRun | null,
|
||||
transaction: Transaction,
|
||||
): Promise<void> {
|
||||
const run = selected?.run;
|
||||
const attempt = selected?.attempt;
|
||||
const startedAtMs = attempt?.startedAtMs ?? run?.startedAtMs ?? null;
|
||||
const finishedAtMs = attempt?.finishedAtMs ?? run?.finishedAtMs ?? null;
|
||||
const values: Partial<ProjectionCrontabRow> = {
|
||||
status,
|
||||
pid: attempt && isAttemptActive(attempt.status) ? attempt.pid : null,
|
||||
logPath: parseLegacyLogOutputRef(run?.outputRef ?? undefined),
|
||||
};
|
||||
if (startedAtMs !== null) {
|
||||
values.lastExecutionTime = toUnixSeconds(startedAtMs);
|
||||
}
|
||||
if (startedAtMs !== null && finishedAtMs !== null) {
|
||||
values.lastRunningTime = Math.max(
|
||||
0,
|
||||
Math.floor((finishedAtMs - startedAtMs) / 1000),
|
||||
);
|
||||
}
|
||||
await this.models.crontab.update(values, {
|
||||
where: { id: cronId },
|
||||
transaction,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { DataTypes, Model, ModelStatic, Sequelize } from 'sequelize';
|
||||
import { RUN_TABLE } from '../../../migrations/0002-run-schema';
|
||||
import type { PrimaryRunIdempotencyLookup } from '../../ports/primaryRunIdempotencyLookup';
|
||||
|
||||
interface IdempotentRunRow {
|
||||
id: string;
|
||||
projectId: string;
|
||||
idempotencyKey: string | null;
|
||||
}
|
||||
|
||||
interface IdempotentRunInstance
|
||||
extends Model<IdempotentRunRow, IdempotentRunRow>,
|
||||
IdempotentRunRow {}
|
||||
|
||||
function defineIdempotentRunModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<IdempotentRunInstance> {
|
||||
return database.define<IdempotentRunInstance>(
|
||||
'Ql3PrimaryIdempotentRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
},
|
||||
idempotencyKey: {
|
||||
field: 'idempotency_key',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: RUN_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export class LegacySequelizePrimaryRunIdempotencyLookup
|
||||
implements PrimaryRunIdempotencyLookup
|
||||
{
|
||||
private readonly run: ModelStatic<IdempotentRunInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.run = defineIdempotentRunModel(database);
|
||||
}
|
||||
|
||||
async findRunId(
|
||||
projectId: string,
|
||||
idempotencyKey: string,
|
||||
): Promise<string | null> {
|
||||
if (!projectId || projectId.length > 128) {
|
||||
throw new RangeError('projectId must be between 1 and 128 characters');
|
||||
}
|
||||
if (!idempotencyKey || idempotencyKey.length > 255) {
|
||||
throw new RangeError(
|
||||
'idempotencyKey must be between 1 and 255 characters',
|
||||
);
|
||||
}
|
||||
const row = await this.run.findOne({
|
||||
attributes: ['id'],
|
||||
where: { projectId, idempotencyKey },
|
||||
raw: true,
|
||||
});
|
||||
return row?.id ?? null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Op,
|
||||
Sequelize,
|
||||
type WhereOptions,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import type { ExecutorType } from '../../domain/execution';
|
||||
import type {
|
||||
PrimaryRunRecoveryAttemptReference,
|
||||
PrimaryRunRecoveryCandidate,
|
||||
PrimaryRunRecoveryCursor,
|
||||
PrimaryRunRecoveryPage,
|
||||
PrimaryRunRecoverySource,
|
||||
} from '../../ports/primaryRunRecoverySource';
|
||||
import { MAX_PRIMARY_RECOVERY_BATCH_SIZE } from '../../ports/primaryRunRecoverySource';
|
||||
|
||||
interface RecoveryRunRow {
|
||||
id: string;
|
||||
executionOwner: string;
|
||||
status: string;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
interface RecoveryAttemptRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
attempt: number;
|
||||
status: string;
|
||||
executorType: string;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
interface RecoveryRunInstance
|
||||
extends Model<RecoveryRunRow, RecoveryRunRow>,
|
||||
RecoveryRunRow {}
|
||||
interface RecoveryAttemptInstance
|
||||
extends Model<RecoveryAttemptRow, RecoveryAttemptRow>,
|
||||
RecoveryAttemptRow {}
|
||||
|
||||
function defineRecoveryRunModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<RecoveryRunInstance> {
|
||||
return database.define<RecoveryRunInstance>(
|
||||
'Ql3PrimaryRecoveryRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
executionOwner: {
|
||||
field: 'execution_owner',
|
||||
type: DataTypes.STRING(16),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function defineRecoveryAttemptModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<RecoveryAttemptInstance> {
|
||||
return database.define<RecoveryAttemptInstance>(
|
||||
'Ql3PrimaryRecoveryAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
runId: {
|
||||
field: 'run_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
attempt: { type: DataTypes.INTEGER, allowNull: false },
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
executorType: {
|
||||
field: 'executor_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: RUN_ATTEMPT_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function assertCursor(cursor: PrimaryRunRecoveryCursor): void {
|
||||
if (!Number.isSafeInteger(cursor.createdAtMs) || cursor.createdAtMs < 0) {
|
||||
throw new RangeError('cursor.createdAtMs must be a non-negative integer');
|
||||
}
|
||||
if (!cursor.runId || cursor.runId.length > 36) {
|
||||
throw new RangeError('cursor.runId must be between 1 and 36 characters');
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizePrimaryRunRecoverySource
|
||||
implements PrimaryRunRecoverySource
|
||||
{
|
||||
private readonly run: ModelStatic<RecoveryRunInstance>;
|
||||
private readonly attempt: ModelStatic<RecoveryAttemptInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.run = defineRecoveryRunModel(database);
|
||||
this.attempt = defineRecoveryAttemptModel(database);
|
||||
}
|
||||
|
||||
async listCandidates({
|
||||
cursor,
|
||||
limit = 32,
|
||||
}: {
|
||||
cursor?: PrimaryRunRecoveryCursor;
|
||||
limit?: number;
|
||||
} = {}): Promise<PrimaryRunRecoveryPage> {
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_PRIMARY_RECOVERY_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_PRIMARY_RECOVERY_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (cursor) assertCursor(cursor);
|
||||
|
||||
const where: WhereOptions<RecoveryRunRow> = {
|
||||
executionOwner: 'runtime',
|
||||
status: { [Op.in]: ['dispatching', 'running'] },
|
||||
...(cursor === undefined
|
||||
? {}
|
||||
: {
|
||||
[Op.or]: [
|
||||
{ createdAtMs: { [Op.gt]: cursor.createdAtMs } },
|
||||
{
|
||||
createdAtMs: cursor.createdAtMs,
|
||||
id: { [Op.gt]: cursor.runId },
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
const runRows = (await this.run.findAll({
|
||||
attributes: ['id', 'createdAtMs'],
|
||||
where,
|
||||
order: [
|
||||
['createdAtMs', 'ASC'],
|
||||
['id', 'ASC'],
|
||||
],
|
||||
limit: limit + 1,
|
||||
raw: true,
|
||||
})) as unknown as RecoveryRunRow[];
|
||||
const truncated = runRows.length > limit;
|
||||
const boundedRuns = runRows.slice(0, limit);
|
||||
if (boundedRuns.length === 0) {
|
||||
return {
|
||||
candidates: [],
|
||||
truncated: false,
|
||||
unsafeAttemptOverflow: false,
|
||||
};
|
||||
}
|
||||
|
||||
const maxAttemptRows = limit * 2;
|
||||
const attemptRows = (await this.attempt.findAll({
|
||||
attributes: ['id', 'runId', 'attempt', 'executorType'],
|
||||
where: {
|
||||
runId: { [Op.in]: boundedRuns.map((run) => run.id) },
|
||||
status: { [Op.in]: ['claimed', 'starting', 'running'] },
|
||||
},
|
||||
order: [
|
||||
['runId', 'ASC'],
|
||||
['attempt', 'DESC'],
|
||||
['createdAtMs', 'DESC'],
|
||||
['id', 'DESC'],
|
||||
],
|
||||
limit: maxAttemptRows + 1,
|
||||
raw: true,
|
||||
})) as unknown as RecoveryAttemptRow[];
|
||||
if (attemptRows.length > maxAttemptRows) {
|
||||
return {
|
||||
candidates: [],
|
||||
truncated,
|
||||
unsafeAttemptOverflow: true,
|
||||
};
|
||||
}
|
||||
|
||||
const attemptsByRun = new Map<
|
||||
string,
|
||||
PrimaryRunRecoveryAttemptReference[]
|
||||
>();
|
||||
for (const attempt of attemptRows) {
|
||||
const references = attemptsByRun.get(attempt.runId) ?? [];
|
||||
references.push({
|
||||
attemptId: attempt.id,
|
||||
executorType: attempt.executorType as ExecutorType,
|
||||
});
|
||||
attemptsByRun.set(attempt.runId, references);
|
||||
}
|
||||
const candidates: PrimaryRunRecoveryCandidate[] = boundedRuns.map(
|
||||
(run) => ({
|
||||
runId: run.id,
|
||||
attempts: attemptsByRun.get(run.id) ?? [],
|
||||
}),
|
||||
);
|
||||
const last = boundedRuns[boundedRuns.length - 1];
|
||||
return {
|
||||
candidates,
|
||||
truncated,
|
||||
unsafeAttemptOverflow: false,
|
||||
nextCursor: { createdAtMs: last.createdAtMs, runId: last.id },
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Op,
|
||||
Sequelize,
|
||||
type WhereOptions,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import {
|
||||
MAX_PRIMARY_TIMEOUT_BATCH_SIZE,
|
||||
type PrimaryTimeoutCursor,
|
||||
type PrimaryTimeoutPage,
|
||||
type PrimaryTimeoutSource,
|
||||
} from '../../ports/primaryTimeoutSource';
|
||||
|
||||
interface TimeoutAttemptRow {
|
||||
id: string;
|
||||
runId: string;
|
||||
status: string;
|
||||
deadlineAtMs: number | null;
|
||||
}
|
||||
|
||||
interface TimeoutRunRow {
|
||||
id: string;
|
||||
executionOwner: string;
|
||||
status: string;
|
||||
cancelRequestedAtMs: number | null;
|
||||
}
|
||||
|
||||
interface TimeoutAttemptInstance
|
||||
extends Model<TimeoutAttemptRow, TimeoutAttemptRow>,
|
||||
TimeoutAttemptRow {}
|
||||
interface TimeoutRunInstance
|
||||
extends Model<TimeoutRunRow, TimeoutRunRow>,
|
||||
TimeoutRunRow {}
|
||||
|
||||
function defineTimeoutAttemptModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<TimeoutAttemptInstance> {
|
||||
return database.define<TimeoutAttemptInstance>(
|
||||
'Ql3PrimaryTimeoutAttempt',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
runId: {
|
||||
field: 'run_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
deadlineAtMs: {
|
||||
field: 'deadline_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_ATTEMPT_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function defineTimeoutRunModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<TimeoutRunInstance> {
|
||||
return database.define<TimeoutRunInstance>(
|
||||
'Ql3PrimaryTimeoutRun',
|
||||
{
|
||||
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
|
||||
executionOwner: {
|
||||
field: 'execution_owner',
|
||||
type: DataTypes.STRING(16),
|
||||
allowNull: false,
|
||||
},
|
||||
status: { type: DataTypes.STRING(32), allowNull: false },
|
||||
cancelRequestedAtMs: {
|
||||
field: 'cancel_requested_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
{ tableName: RUN_TABLE, timestamps: false, freezeTableName: true },
|
||||
);
|
||||
}
|
||||
|
||||
function assertCursor(cursor: PrimaryTimeoutCursor): void {
|
||||
if (!Number.isSafeInteger(cursor.deadlineAtMs) || cursor.deadlineAtMs < 0) {
|
||||
throw new RangeError('cursor.deadlineAtMs must be a non-negative integer');
|
||||
}
|
||||
if (!cursor.attemptId || cursor.attemptId.length > 36) {
|
||||
throw new RangeError(
|
||||
'cursor.attemptId must be between 1 and 36 characters',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizePrimaryTimeoutSource
|
||||
implements PrimaryTimeoutSource
|
||||
{
|
||||
private readonly attempt: ModelStatic<TimeoutAttemptInstance>;
|
||||
private readonly run: ModelStatic<TimeoutRunInstance>;
|
||||
|
||||
constructor(database: Sequelize) {
|
||||
this.attempt = defineTimeoutAttemptModel(database);
|
||||
this.run = defineTimeoutRunModel(database);
|
||||
this.attempt.belongsTo(this.run, {
|
||||
as: 'timeoutRun',
|
||||
foreignKey: 'runId',
|
||||
targetKey: 'id',
|
||||
constraints: false,
|
||||
});
|
||||
}
|
||||
|
||||
async listOverdue(options: {
|
||||
nowMs: number;
|
||||
cursor?: PrimaryTimeoutCursor;
|
||||
limit?: number;
|
||||
}): Promise<PrimaryTimeoutPage> {
|
||||
if (!Number.isSafeInteger(options.nowMs) || options.nowMs < 0) {
|
||||
throw new RangeError('nowMs must be a non-negative safe integer');
|
||||
}
|
||||
const limit = options.limit ?? 32;
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_PRIMARY_TIMEOUT_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_PRIMARY_TIMEOUT_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (options.cursor) assertCursor(options.cursor);
|
||||
|
||||
const cursorWhere: WhereOptions<TimeoutAttemptRow> = options.cursor
|
||||
? {
|
||||
[Op.or]: [
|
||||
{ deadlineAtMs: { [Op.gt]: options.cursor.deadlineAtMs } },
|
||||
{
|
||||
deadlineAtMs: options.cursor.deadlineAtMs,
|
||||
id: { [Op.gt]: options.cursor.attemptId },
|
||||
},
|
||||
],
|
||||
}
|
||||
: {};
|
||||
const rows = (await this.attempt.findAll({
|
||||
attributes: ['id', 'runId', 'status', 'deadlineAtMs'],
|
||||
where: {
|
||||
status: { [Op.in]: ['starting', 'running'] },
|
||||
deadlineAtMs: { [Op.ne]: null, [Op.lte]: options.nowMs },
|
||||
...cursorWhere,
|
||||
},
|
||||
include: [
|
||||
{
|
||||
model: this.run,
|
||||
as: 'timeoutRun',
|
||||
attributes: [],
|
||||
required: true,
|
||||
where: {
|
||||
executionOwner: 'runtime',
|
||||
status: { [Op.in]: ['dispatching', 'running'] },
|
||||
cancelRequestedAtMs: null,
|
||||
},
|
||||
},
|
||||
],
|
||||
order: [
|
||||
['deadlineAtMs', 'ASC'],
|
||||
['id', 'ASC'],
|
||||
],
|
||||
limit: limit + 1,
|
||||
raw: true,
|
||||
})) as unknown as TimeoutAttemptRow[];
|
||||
|
||||
const truncated = rows.length > limit;
|
||||
const selected = rows.slice(0, limit);
|
||||
const candidates = selected.map((row) => ({
|
||||
runId: row.runId,
|
||||
attemptId: row.id,
|
||||
deadlineAtMs: Number(row.deadlineAtMs),
|
||||
}));
|
||||
const last = candidates[candidates.length - 1];
|
||||
return {
|
||||
candidates,
|
||||
truncated,
|
||||
...(truncated && last
|
||||
? {
|
||||
nextCursor: {
|
||||
deadlineAtMs: last.deadlineAtMs,
|
||||
attemptId: last.attemptId,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,522 @@
|
||||
import { timingSafeEqual } from 'crypto';
|
||||
import {
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
PROJECT_ROLE_BINDING_TABLE,
|
||||
PROJECT_TABLE,
|
||||
} from '../../../migrations/0017-project-policy';
|
||||
import { PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE } from '../../../migrations/0018-project-owner-bootstrap';
|
||||
import {
|
||||
assertProjectPolicyProjectId,
|
||||
normalizePolicySubject,
|
||||
normalizeProjectRoleBindingRecord,
|
||||
type ProjectRoleBindingRecord,
|
||||
} from '../../domain/projectPolicy';
|
||||
import {
|
||||
OWNER_BOOTSTRAP_MAX_VERSION,
|
||||
OWNER_BOOTSTRAP_SYSTEM_SUBJECT,
|
||||
ProjectOwnerBootstrapChallengeActiveError,
|
||||
ProjectOwnerBootstrapClaimRejectedError,
|
||||
ProjectOwnerBootstrapProjectInactiveError,
|
||||
ProjectOwnerBootstrapProjectNotFoundError,
|
||||
ProjectOwnerBootstrapProjectNotPristineError,
|
||||
ProjectOwnerBootstrapUnavailableError,
|
||||
assertProjectOwnerBootstrapChallengeId,
|
||||
assertProjectOwnerBootstrapTokenDigest,
|
||||
normalizeProjectOwnerBootstrapChallengeRecord,
|
||||
type ProjectOwnerBootstrapChallengeRecord,
|
||||
} from '../../domain/projectOwnerBootstrap';
|
||||
import type {
|
||||
ClaimProjectOwnerBootstrapChallengeCommand,
|
||||
ClaimProjectOwnerBootstrapChallengeResult,
|
||||
IssueProjectOwnerBootstrapChallengeCommand,
|
||||
ProjectOwnerBootstrapRepository,
|
||||
} from '../../ports/projectOwnerBootstrapRepository';
|
||||
|
||||
const RETRY_ATTEMPTS = 5;
|
||||
|
||||
interface ProjectStatusRow {
|
||||
status: string;
|
||||
}
|
||||
|
||||
interface BootstrapChallengeRow {
|
||||
project_id: string;
|
||||
version: number;
|
||||
challenge_id: string;
|
||||
token_digest: string;
|
||||
issued_at_ms: number | string;
|
||||
expires_at_ms: number | string;
|
||||
consumed_at_ms: number | string | null;
|
||||
claimed_subject_type: string | null;
|
||||
claimed_subject_id: string | null;
|
||||
}
|
||||
|
||||
interface BootstrapBindingRow {
|
||||
project_id: string;
|
||||
subject_type: string;
|
||||
subject_id: string;
|
||||
version: number;
|
||||
state: string;
|
||||
role: string | null;
|
||||
mutation_id: string;
|
||||
changed_by_type: string;
|
||||
changed_by_id: string;
|
||||
created_at_ms: number | string;
|
||||
}
|
||||
|
||||
function assertExactKeys(value: object, expected: readonly string[]): void {
|
||||
const keys = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
keys.length !== canonical.length ||
|
||||
keys.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
throw new TypeError('Project owner bootstrap command shape is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function assertTimestamp(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError(`Project owner bootstrap ${name} is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeIssueCommand(
|
||||
command: IssueProjectOwnerBootstrapChallengeCommand,
|
||||
): Readonly<IssueProjectOwnerBootstrapChallengeCommand> {
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Project owner bootstrap issue command is invalid');
|
||||
}
|
||||
assertExactKeys(command, [
|
||||
'projectId',
|
||||
'challengeId',
|
||||
'tokenDigest',
|
||||
'issuedAtMs',
|
||||
'expiresAtMs',
|
||||
]);
|
||||
assertProjectPolicyProjectId(command.projectId);
|
||||
assertProjectOwnerBootstrapChallengeId(command.challengeId);
|
||||
assertProjectOwnerBootstrapTokenDigest(command.tokenDigest);
|
||||
assertTimestamp('issuedAtMs', command.issuedAtMs);
|
||||
assertTimestamp('expiresAtMs', command.expiresAtMs);
|
||||
if (command.expiresAtMs <= command.issuedAtMs) {
|
||||
throw new TypeError('Project owner bootstrap lifetime is invalid');
|
||||
}
|
||||
return Object.freeze({ ...command });
|
||||
}
|
||||
|
||||
function normalizeClaimCommand(
|
||||
command: ClaimProjectOwnerBootstrapChallengeCommand,
|
||||
): Readonly<ClaimProjectOwnerBootstrapChallengeCommand> {
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Project owner bootstrap claim command is invalid');
|
||||
}
|
||||
assertExactKeys(command, [
|
||||
'projectId',
|
||||
'challengeId',
|
||||
'tokenDigest',
|
||||
'subject',
|
||||
'claimedAtMs',
|
||||
]);
|
||||
assertProjectPolicyProjectId(command.projectId);
|
||||
assertProjectOwnerBootstrapChallengeId(command.challengeId);
|
||||
assertProjectOwnerBootstrapTokenDigest(command.tokenDigest);
|
||||
const subject = normalizePolicySubject(command.subject);
|
||||
if (subject.type !== 'user') {
|
||||
throw new ProjectOwnerBootstrapClaimRejectedError();
|
||||
}
|
||||
assertTimestamp('claimedAtMs', command.claimedAtMs);
|
||||
return Object.freeze({ ...command, subject });
|
||||
}
|
||||
|
||||
function rowToChallenge(
|
||||
row: BootstrapChallengeRow,
|
||||
): Readonly<ProjectOwnerBootstrapChallengeRecord> {
|
||||
return normalizeProjectOwnerBootstrapChallengeRecord({
|
||||
projectId: row.project_id,
|
||||
version: Number(row.version),
|
||||
challengeId: row.challenge_id,
|
||||
tokenDigest: row.token_digest,
|
||||
issuedAtMs: Number(row.issued_at_ms),
|
||||
expiresAtMs: Number(row.expires_at_ms),
|
||||
...(row.consumed_at_ms === null
|
||||
? {}
|
||||
: {
|
||||
consumedAtMs: Number(row.consumed_at_ms),
|
||||
claimedSubject: {
|
||||
type: row.claimed_subject_type as NonNullable<
|
||||
ProjectOwnerBootstrapChallengeRecord['claimedSubject']
|
||||
>['type'],
|
||||
id: row.claimed_subject_id!,
|
||||
},
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function rowToBinding(
|
||||
row: BootstrapBindingRow,
|
||||
): Readonly<ProjectRoleBindingRecord> {
|
||||
return normalizeProjectRoleBindingRecord({
|
||||
projectId: row.project_id,
|
||||
subject: {
|
||||
type: row.subject_type as ProjectRoleBindingRecord['subject']['type'],
|
||||
id: row.subject_id,
|
||||
},
|
||||
version: Number(row.version),
|
||||
state: row.state as ProjectRoleBindingRecord['state'],
|
||||
...(row.role === null
|
||||
? {}
|
||||
: { role: row.role as NonNullable<ProjectRoleBindingRecord['role']> }),
|
||||
mutationId: row.mutation_id,
|
||||
changedBy: {
|
||||
type: row.changed_by_type as ProjectRoleBindingRecord['changedBy']['type'],
|
||||
id: row.changed_by_id,
|
||||
},
|
||||
createdAtMs: Number(row.created_at_ms),
|
||||
});
|
||||
}
|
||||
|
||||
function digestMatches(expected: string, actual: string): boolean {
|
||||
assertProjectOwnerBootstrapTokenDigest(expected);
|
||||
assertProjectOwnerBootstrapTokenDigest(actual);
|
||||
return timingSafeEqual(
|
||||
Buffer.from(expected, 'hex'),
|
||||
Buffer.from(actual, 'hex'),
|
||||
);
|
||||
}
|
||||
|
||||
function mutationId(challengeId: string): string {
|
||||
return `owner-bootstrap:${challengeId}`;
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
function isExpectedError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof ProjectOwnerBootstrapChallengeActiveError ||
|
||||
error instanceof ProjectOwnerBootstrapClaimRejectedError ||
|
||||
error instanceof ProjectOwnerBootstrapProjectInactiveError ||
|
||||
error instanceof ProjectOwnerBootstrapProjectNotFoundError ||
|
||||
error instanceof ProjectOwnerBootstrapProjectNotPristineError ||
|
||||
error instanceof ProjectOwnerBootstrapUnavailableError
|
||||
);
|
||||
}
|
||||
|
||||
export class LegacySequelizeProjectOwnerBootstrapRepository
|
||||
implements ProjectOwnerBootstrapRepository
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Project owner bootstrap repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async issue(
|
||||
rawCommand: IssueProjectOwnerBootstrapChallengeCommand,
|
||||
): Promise<Readonly<ProjectOwnerBootstrapChallengeRecord>> {
|
||||
const command = normalizeIssueCommand(rawCommand);
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
await this.assertActiveProject(command.projectId, transaction);
|
||||
if ((await this.bindingCount(command.projectId, transaction)) > 0) {
|
||||
throw new ProjectOwnerBootstrapProjectNotPristineError();
|
||||
}
|
||||
const latest = await this.latestChallenge(
|
||||
command.projectId,
|
||||
transaction,
|
||||
);
|
||||
if (
|
||||
latest?.consumedAtMs !== undefined ||
|
||||
(latest && latest.expiresAtMs > command.issuedAtMs)
|
||||
) {
|
||||
if (latest?.consumedAtMs !== undefined) {
|
||||
throw new ProjectOwnerBootstrapProjectNotPristineError();
|
||||
}
|
||||
throw new ProjectOwnerBootstrapChallengeActiveError();
|
||||
}
|
||||
const version = (latest?.version ?? 0) + 1;
|
||||
if (version > OWNER_BOOTSTRAP_MAX_VERSION) {
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
await this.database.query(
|
||||
`INSERT INTO "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}"
|
||||
(project_id, version, challenge_id, token_digest,
|
||||
issued_at_ms, expires_at_ms, consumed_at_ms,
|
||||
claimed_subject_type, claimed_subject_id)
|
||||
VALUES
|
||||
(:projectId, :version, :challengeId, :tokenDigest,
|
||||
:issuedAtMs, :expiresAtMs, NULL, NULL, NULL)`,
|
||||
{
|
||||
type: QueryTypes.INSERT,
|
||||
replacements: { ...command, version },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
return normalizeProjectOwnerBootstrapChallengeRecord({
|
||||
...command,
|
||||
version,
|
||||
});
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (isExpectedError(error)) throw error;
|
||||
if (
|
||||
errorCode(error) === 'SQLITE_BUSY' &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
if (error instanceof UniqueConstraintError) {
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
}
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
|
||||
async claim(
|
||||
rawCommand: ClaimProjectOwnerBootstrapChallengeCommand,
|
||||
): Promise<ClaimProjectOwnerBootstrapChallengeResult> {
|
||||
const command = normalizeClaimCommand(rawCommand);
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
await this.assertActiveProject(command.projectId, transaction);
|
||||
const latest = await this.latestChallenge(
|
||||
command.projectId,
|
||||
transaction,
|
||||
);
|
||||
if (
|
||||
!latest ||
|
||||
latest.challengeId !== command.challengeId ||
|
||||
!digestMatches(latest.tokenDigest, command.tokenDigest)
|
||||
) {
|
||||
throw new ProjectOwnerBootstrapClaimRejectedError();
|
||||
}
|
||||
if (latest.consumedAtMs !== undefined) {
|
||||
if (
|
||||
latest.claimedSubject?.type !== command.subject.type ||
|
||||
latest.claimedSubject.id !== command.subject.id
|
||||
) {
|
||||
throw new ProjectOwnerBootstrapClaimRejectedError();
|
||||
}
|
||||
const binding = await this.bootstrapBinding(
|
||||
command.projectId,
|
||||
latest.challengeId,
|
||||
transaction,
|
||||
);
|
||||
if (
|
||||
!binding ||
|
||||
binding.subject.type !== command.subject.type ||
|
||||
binding.subject.id !== command.subject.id ||
|
||||
binding.role !== 'owner' ||
|
||||
binding.state !== 'active'
|
||||
) {
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
return { status: 'existing', binding };
|
||||
}
|
||||
if (
|
||||
command.claimedAtMs < latest.issuedAtMs ||
|
||||
command.claimedAtMs >= latest.expiresAtMs
|
||||
) {
|
||||
throw new ProjectOwnerBootstrapClaimRejectedError();
|
||||
}
|
||||
if ((await this.bindingCount(command.projectId, transaction)) > 0) {
|
||||
throw new ProjectOwnerBootstrapProjectNotPristineError();
|
||||
}
|
||||
const binding = normalizeProjectRoleBindingRecord({
|
||||
projectId: command.projectId,
|
||||
subject: command.subject,
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role: 'owner',
|
||||
mutationId: mutationId(latest.challengeId),
|
||||
changedBy: OWNER_BOOTSTRAP_SYSTEM_SUBJECT,
|
||||
createdAtMs: command.claimedAtMs,
|
||||
});
|
||||
const [, consumedCount] = await this.database.query(
|
||||
`UPDATE "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}"
|
||||
SET consumed_at_ms = :claimedAtMs,
|
||||
claimed_subject_type = :subjectType,
|
||||
claimed_subject_id = :subjectId
|
||||
WHERE project_id = :projectId
|
||||
AND version = :version
|
||||
AND challenge_id = :challengeId
|
||||
AND consumed_at_ms IS NULL`,
|
||||
{
|
||||
type: QueryTypes.UPDATE,
|
||||
replacements: {
|
||||
projectId: command.projectId,
|
||||
version: latest.version,
|
||||
challengeId: latest.challengeId,
|
||||
claimedAtMs: command.claimedAtMs,
|
||||
subjectType: command.subject.type,
|
||||
subjectId: command.subject.id,
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (consumedCount !== 1) {
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
await this.database.query(
|
||||
`INSERT INTO "${PROJECT_ROLE_BINDING_TABLE}"
|
||||
(project_id, subject_type, subject_id, version, state, role,
|
||||
mutation_id, changed_by_type, changed_by_id, created_at_ms)
|
||||
VALUES
|
||||
(:projectId, :subjectType, :subjectId, 1, 'active', 'owner',
|
||||
:mutationId, :changedByType, :changedById, :createdAtMs)`,
|
||||
{
|
||||
type: QueryTypes.INSERT,
|
||||
replacements: {
|
||||
projectId: binding.projectId,
|
||||
subjectType: binding.subject.type,
|
||||
subjectId: binding.subject.id,
|
||||
mutationId: binding.mutationId,
|
||||
changedByType: binding.changedBy.type,
|
||||
changedById: binding.changedBy.id,
|
||||
createdAtMs: binding.createdAtMs,
|
||||
},
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
return { status: 'claimed', binding };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (isExpectedError(error)) throw error;
|
||||
if (
|
||||
errorCode(error) === 'SQLITE_BUSY' &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
}
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
|
||||
private async assertActiveProject(
|
||||
projectId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<void> {
|
||||
const projects = await this.database.query<ProjectStatusRow>(
|
||||
`SELECT status FROM "${PROJECT_TABLE}" WHERE id = :projectId LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (projects.length === 0) {
|
||||
throw new ProjectOwnerBootstrapProjectNotFoundError();
|
||||
}
|
||||
if (projects.length !== 1 || projects[0].status !== 'active') {
|
||||
throw new ProjectOwnerBootstrapProjectInactiveError();
|
||||
}
|
||||
}
|
||||
|
||||
private async bindingCount(
|
||||
projectId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<number> {
|
||||
const rows = await this.database.query<{ count: number | string }>(
|
||||
`SELECT COUNT(*) AS count
|
||||
FROM "${PROJECT_ROLE_BINDING_TABLE}"
|
||||
WHERE project_id = :projectId`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
const count = Number(rows[0]?.count);
|
||||
if (!Number.isSafeInteger(count) || count < 0) {
|
||||
throw new ProjectOwnerBootstrapUnavailableError();
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private async latestChallenge(
|
||||
projectId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<Readonly<ProjectOwnerBootstrapChallengeRecord> | null> {
|
||||
const rows = await this.database.query<BootstrapChallengeRow>(
|
||||
`SELECT project_id, version, challenge_id, token_digest,
|
||||
issued_at_ms, expires_at_ms, consumed_at_ms,
|
||||
claimed_subject_type, claimed_subject_id
|
||||
FROM "${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE}"
|
||||
WHERE project_id = :projectId
|
||||
ORDER BY version DESC
|
||||
LIMIT 1`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length !== 1) throw new ProjectOwnerBootstrapUnavailableError();
|
||||
return rowToChallenge(rows[0]);
|
||||
}
|
||||
|
||||
private async bootstrapBinding(
|
||||
projectId: string,
|
||||
challengeId: string,
|
||||
transaction: Transaction,
|
||||
): Promise<Readonly<ProjectRoleBindingRecord> | null> {
|
||||
const rows = await this.database.query<BootstrapBindingRow>(
|
||||
`SELECT project_id, subject_type, subject_id, version, state, role,
|
||||
mutation_id, changed_by_type, changed_by_id, created_at_ms
|
||||
FROM "${PROJECT_ROLE_BINDING_TABLE}"
|
||||
WHERE project_id = :projectId
|
||||
AND mutation_id = :mutationId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId, mutationId: mutationId(challengeId) },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length !== 1) throw new ProjectOwnerBootstrapUnavailableError();
|
||||
return rowToBinding(rows[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
QueryTypes,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import {
|
||||
PROJECT_ROLE_BINDING_TABLE,
|
||||
PROJECT_TABLE,
|
||||
} from '../../../migrations/0017-project-policy';
|
||||
import {
|
||||
MAX_PROJECT_ROLE_BINDING_VERSION,
|
||||
ProjectPolicyProjectNotFoundError,
|
||||
ProjectPolicyUnavailableError,
|
||||
ProjectRoleBindingMutationConflictError,
|
||||
ProjectRoleBindingVersionConflictError,
|
||||
assertProjectPolicyProjectId,
|
||||
normalizePolicySubject,
|
||||
normalizeProjectPolicySnapshot,
|
||||
normalizeProjectRoleBindingRecord,
|
||||
type ProjectPolicySnapshot,
|
||||
type ProjectRoleBindingRecord,
|
||||
} from '../../domain/projectPolicy';
|
||||
import type {
|
||||
AppendProjectRoleBindingCommand,
|
||||
AppendProjectRoleBindingResult,
|
||||
ProjectPolicyRepository,
|
||||
} from '../../ports/projectPolicyRepository';
|
||||
|
||||
const RETRY_ATTEMPTS = 5;
|
||||
|
||||
interface ProjectRoleBindingRow {
|
||||
projectId: string;
|
||||
subjectType: string;
|
||||
subjectId: string;
|
||||
version: number;
|
||||
state: string;
|
||||
role: string | null;
|
||||
mutationId: string;
|
||||
changedByType: string;
|
||||
changedById: string;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface ProjectRoleBindingInstance
|
||||
extends Model<ProjectRoleBindingRow, ProjectRoleBindingRow>,
|
||||
ProjectRoleBindingRow {}
|
||||
|
||||
interface ProjectPolicySnapshotRow {
|
||||
project_id: string;
|
||||
project_name: string;
|
||||
project_slug: string;
|
||||
project_status: string;
|
||||
project_version: number;
|
||||
project_created_at_ms: number | string;
|
||||
project_updated_at_ms: number | string;
|
||||
binding_project_id: string | null;
|
||||
binding_subject_type: string | null;
|
||||
binding_subject_id: string | null;
|
||||
binding_version: number | null;
|
||||
binding_state: string | null;
|
||||
binding_role: string | null;
|
||||
binding_mutation_id: string | null;
|
||||
binding_changed_by_type: string | null;
|
||||
binding_changed_by_id: string | null;
|
||||
binding_created_at_ms: number | string | null;
|
||||
}
|
||||
|
||||
function defineBindingModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<ProjectRoleBindingInstance> {
|
||||
return database.define<ProjectRoleBindingInstance>(
|
||||
'Ql3ProjectRoleBinding',
|
||||
{
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
subjectType: {
|
||||
field: 'subject_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
subjectId: {
|
||||
field: 'subject_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
version: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
state: { type: DataTypes.STRING(16), allowNull: false },
|
||||
role: { type: DataTypes.STRING(16), allowNull: true },
|
||||
mutationId: {
|
||||
field: 'mutation_id',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
changedByType: {
|
||||
field: 'changed_by_type',
|
||||
type: DataTypes.STRING(32),
|
||||
allowNull: false,
|
||||
},
|
||||
changedById: {
|
||||
field: 'changed_by_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: PROJECT_ROLE_BINDING_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function rowToBinding(
|
||||
row: ProjectRoleBindingRow,
|
||||
): Readonly<ProjectRoleBindingRecord> {
|
||||
try {
|
||||
return normalizeProjectRoleBindingRecord({
|
||||
projectId: row.projectId,
|
||||
subject: {
|
||||
type: row.subjectType as ProjectRoleBindingRecord['subject']['type'],
|
||||
id: row.subjectId,
|
||||
},
|
||||
version: Number(row.version),
|
||||
state: row.state as ProjectRoleBindingRecord['state'],
|
||||
...(row.role === null
|
||||
? {}
|
||||
: { role: row.role as NonNullable<ProjectRoleBindingRecord['role']> }),
|
||||
mutationId: row.mutationId,
|
||||
changedBy: {
|
||||
type: row.changedByType as ProjectRoleBindingRecord['changedBy']['type'],
|
||||
id: row.changedById,
|
||||
},
|
||||
createdAtMs: Number(row.createdAtMs),
|
||||
});
|
||||
} catch {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function snapshotRowToValue(
|
||||
row: ProjectPolicySnapshotRow,
|
||||
): Readonly<ProjectPolicySnapshot> {
|
||||
const bindingFields = [
|
||||
row.binding_project_id,
|
||||
row.binding_subject_type,
|
||||
row.binding_subject_id,
|
||||
row.binding_version,
|
||||
row.binding_state,
|
||||
row.binding_mutation_id,
|
||||
row.binding_changed_by_type,
|
||||
row.binding_changed_by_id,
|
||||
row.binding_created_at_ms,
|
||||
];
|
||||
const noBinding = bindingFields.every((value) => value === null);
|
||||
if (!noBinding && bindingFields.some((value) => value === null)) {
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
try {
|
||||
return normalizeProjectPolicySnapshot({
|
||||
project: {
|
||||
id: row.project_id,
|
||||
name: row.project_name,
|
||||
slug: row.project_slug,
|
||||
status:
|
||||
row.project_status as ProjectPolicySnapshot['project']['status'],
|
||||
version: Number(row.project_version),
|
||||
createdAtMs: Number(row.project_created_at_ms),
|
||||
updatedAtMs: Number(row.project_updated_at_ms),
|
||||
},
|
||||
...(noBinding
|
||||
? {}
|
||||
: {
|
||||
binding: {
|
||||
projectId: row.binding_project_id!,
|
||||
subject: {
|
||||
type: row.binding_subject_type as ProjectRoleBindingRecord['subject']['type'],
|
||||
id: row.binding_subject_id!,
|
||||
},
|
||||
version: Number(row.binding_version),
|
||||
state: row.binding_state as ProjectRoleBindingRecord['state'],
|
||||
...(row.binding_role === null
|
||||
? {}
|
||||
: {
|
||||
role: row.binding_role as NonNullable<
|
||||
ProjectRoleBindingRecord['role']
|
||||
>,
|
||||
}),
|
||||
mutationId: row.binding_mutation_id!,
|
||||
changedBy: {
|
||||
type: row.binding_changed_by_type as ProjectRoleBindingRecord['changedBy']['type'],
|
||||
id: row.binding_changed_by_id!,
|
||||
},
|
||||
createdAtMs: Number(row.binding_created_at_ms),
|
||||
},
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof ProjectPolicyUnavailableError) throw error;
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
function sameBinding(
|
||||
left: Readonly<ProjectRoleBindingRecord>,
|
||||
right: Readonly<ProjectRoleBindingRecord>,
|
||||
): boolean {
|
||||
return (
|
||||
left.projectId === right.projectId &&
|
||||
left.subject.type === right.subject.type &&
|
||||
left.subject.id === right.subject.id &&
|
||||
left.version === right.version &&
|
||||
left.state === right.state &&
|
||||
left.role === right.role &&
|
||||
left.mutationId === right.mutationId &&
|
||||
left.changedBy.type === right.changedBy.type &&
|
||||
left.changedBy.id === right.changedBy.id &&
|
||||
left.createdAtMs === right.createdAtMs
|
||||
);
|
||||
}
|
||||
|
||||
function assertExpectedVersion(value: number): void {
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < 0 ||
|
||||
value >= MAX_PROJECT_ROLE_BINDING_VERSION
|
||||
) {
|
||||
throw new TypeError('Project role binding expected version is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
export class LegacySequelizeProjectPolicyRepository
|
||||
implements ProjectPolicyRepository
|
||||
{
|
||||
private readonly bindings: ModelStatic<ProjectRoleBindingInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Project policy repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.bindings = defineBindingModel(database);
|
||||
}
|
||||
|
||||
async resolve(
|
||||
projectId: string,
|
||||
requestedSubject: Parameters<ProjectPolicyRepository['resolve']>[1],
|
||||
): Promise<Readonly<ProjectPolicySnapshot> | null> {
|
||||
assertProjectPolicyProjectId(projectId);
|
||||
const subject = normalizePolicySubject(requestedSubject);
|
||||
const rows = await this.database.query<ProjectPolicySnapshotRow>(
|
||||
`SELECT project.id AS project_id,
|
||||
project.name AS project_name,
|
||||
project.slug AS project_slug,
|
||||
project.status AS project_status,
|
||||
project.version AS project_version,
|
||||
project.created_at_ms AS project_created_at_ms,
|
||||
project.updated_at_ms AS project_updated_at_ms,
|
||||
binding.project_id AS binding_project_id,
|
||||
binding.subject_type AS binding_subject_type,
|
||||
binding.subject_id AS binding_subject_id,
|
||||
binding.version AS binding_version,
|
||||
binding.state AS binding_state,
|
||||
binding.role AS binding_role,
|
||||
binding.mutation_id AS binding_mutation_id,
|
||||
binding.changed_by_type AS binding_changed_by_type,
|
||||
binding.changed_by_id AS binding_changed_by_id,
|
||||
binding.created_at_ms AS binding_created_at_ms
|
||||
FROM "${PROJECT_TABLE}" AS project
|
||||
LEFT JOIN "${PROJECT_ROLE_BINDING_TABLE}" AS binding
|
||||
ON binding.project_id = project.id
|
||||
AND binding.subject_type = :subjectType
|
||||
AND binding.subject_id = :subjectId
|
||||
AND binding.version = (
|
||||
SELECT MAX(current.version)
|
||||
FROM "${PROJECT_ROLE_BINDING_TABLE}" AS current
|
||||
WHERE current.project_id = project.id
|
||||
AND current.subject_type = :subjectType
|
||||
AND current.subject_id = :subjectId
|
||||
)
|
||||
WHERE project.id = :projectId
|
||||
LIMIT 2`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
projectId,
|
||||
subjectType: subject.type,
|
||||
subjectId: subject.id,
|
||||
},
|
||||
},
|
||||
);
|
||||
if (rows.length === 0) return null;
|
||||
if (rows.length !== 1) throw new ProjectPolicyUnavailableError();
|
||||
return snapshotRowToValue(rows[0]);
|
||||
}
|
||||
|
||||
async append(
|
||||
command: AppendProjectRoleBindingCommand,
|
||||
): Promise<AppendProjectRoleBindingResult> {
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Project role binding command must be an object');
|
||||
}
|
||||
assertExpectedVersion(command.expectedCurrentVersion);
|
||||
const binding = normalizeProjectRoleBindingRecord(command.binding);
|
||||
if (binding.version !== command.expectedCurrentVersion + 1) {
|
||||
throw new ProjectRoleBindingVersionConflictError();
|
||||
}
|
||||
const values: ProjectRoleBindingRow = {
|
||||
projectId: binding.projectId,
|
||||
subjectType: binding.subject.type,
|
||||
subjectId: binding.subject.id,
|
||||
version: binding.version,
|
||||
state: binding.state,
|
||||
role: binding.role ?? null,
|
||||
mutationId: binding.mutationId,
|
||||
changedByType: binding.changedBy.type,
|
||||
changedById: binding.changedBy.id,
|
||||
createdAtMs: binding.createdAtMs,
|
||||
};
|
||||
for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const replay = await this.bindings.findOne({
|
||||
where: {
|
||||
projectId: binding.projectId,
|
||||
mutationId: binding.mutationId,
|
||||
},
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
if (replay) {
|
||||
const previous = rowToBinding(replay);
|
||||
if (!sameBinding(previous, binding)) {
|
||||
throw new ProjectRoleBindingMutationConflictError();
|
||||
}
|
||||
return { status: 'existing', binding: previous };
|
||||
}
|
||||
const projects = await this.database.query<{ id: string }>(
|
||||
`SELECT id FROM "${PROJECT_TABLE}" WHERE id = :projectId LIMIT 1`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { projectId: binding.projectId },
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (projects.length !== 1) {
|
||||
throw new ProjectPolicyProjectNotFoundError();
|
||||
}
|
||||
const current = await this.bindings.findOne({
|
||||
where: {
|
||||
projectId: binding.projectId,
|
||||
subjectType: binding.subject.type,
|
||||
subjectId: binding.subject.id,
|
||||
},
|
||||
order: [['version', 'DESC']],
|
||||
raw: true,
|
||||
transaction,
|
||||
});
|
||||
const currentVersion = current ? Number(current.version) : 0;
|
||||
if (currentVersion !== command.expectedCurrentVersion) {
|
||||
throw new ProjectRoleBindingVersionConflictError();
|
||||
}
|
||||
await this.bindings.create(values, { transaction });
|
||||
return { status: 'inserted', binding };
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof ProjectRoleBindingVersionConflictError ||
|
||||
error instanceof ProjectRoleBindingMutationConflictError ||
|
||||
error instanceof ProjectPolicyProjectNotFoundError ||
|
||||
error instanceof ProjectPolicyUnavailableError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < RETRY_ATTEMPTS - 1
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new ProjectPolicyUnavailableError();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { Sequelize, Transaction } from 'sequelize';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunAttemptStatus,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '../../domain/run';
|
||||
import type { RunRetryPolicyRecord } from '../../domain/runRetryPolicy';
|
||||
import type { RunRepositoryTransaction } from '../../ports/runRepository';
|
||||
import {
|
||||
LegacySequelizeRunRepository,
|
||||
LegacySequelizeRunTransaction,
|
||||
} from './runRepository';
|
||||
|
||||
export interface SequelizeRunProjectionContext {
|
||||
transaction: Transaction;
|
||||
runs: RunRepositoryTransaction;
|
||||
changedRunIds: readonly string[];
|
||||
changedAttemptIds: readonly string[];
|
||||
}
|
||||
|
||||
export interface SequelizeRunProjectionParticipant {
|
||||
apply(context: SequelizeRunProjectionContext): Promise<void>;
|
||||
}
|
||||
|
||||
class TrackingRunRepositoryTransaction implements RunRepositoryTransaction {
|
||||
readonly changedRunIds = new Set<string>();
|
||||
readonly changedAttemptIds = new Set<string>();
|
||||
|
||||
constructor(private readonly delegate: RunRepositoryTransaction) {}
|
||||
|
||||
findRunById(runId: string): Promise<RunRecord | null> {
|
||||
return this.delegate.findRunById(runId);
|
||||
}
|
||||
|
||||
findAttemptById(attemptId: string): Promise<RunAttemptRecord | null> {
|
||||
return this.delegate.findAttemptById(attemptId);
|
||||
}
|
||||
|
||||
findLatestAttemptByRunId(runId: string): Promise<RunAttemptRecord | null> {
|
||||
return this.delegate.findLatestAttemptByRunId(runId);
|
||||
}
|
||||
|
||||
findRetryPolicyByRunId(runId: string): Promise<RunRetryPolicyRecord | null> {
|
||||
return this.delegate.findRetryPolicyByRunId(runId);
|
||||
}
|
||||
|
||||
listEvents(
|
||||
runId: string,
|
||||
options?: { afterSequence?: number; limit?: number },
|
||||
): Promise<RunEventRecord[]> {
|
||||
return this.delegate.listEvents(runId, options);
|
||||
}
|
||||
|
||||
listCancellationRequested(options?: {
|
||||
beforeMs?: number;
|
||||
limit?: number;
|
||||
}): Promise<RunRecord[]> {
|
||||
return this.delegate.listCancellationRequested(options);
|
||||
}
|
||||
|
||||
async insertRun(run: RunRecord): Promise<void> {
|
||||
await this.delegate.insertRun(run);
|
||||
this.changedRunIds.add(run.id);
|
||||
}
|
||||
|
||||
async insertAttempt(attempt: RunAttemptRecord): Promise<void> {
|
||||
await this.delegate.insertAttempt(attempt);
|
||||
this.changedRunIds.add(attempt.runId);
|
||||
this.changedAttemptIds.add(attempt.id);
|
||||
}
|
||||
|
||||
insertRetryPolicy(policy: RunRetryPolicyRecord): Promise<void> {
|
||||
return this.delegate.insertRetryPolicy(policy);
|
||||
}
|
||||
|
||||
async compareAndSetRun(
|
||||
run: RunRecord,
|
||||
expectedVersion: number,
|
||||
): Promise<boolean> {
|
||||
const updated = await this.delegate.compareAndSetRun(run, expectedVersion);
|
||||
if (updated) this.changedRunIds.add(run.id);
|
||||
return updated;
|
||||
}
|
||||
|
||||
async compareAndSetAttempt(
|
||||
attempt: RunAttemptRecord,
|
||||
expected: { status: RunAttemptStatus; callbackSequence: number },
|
||||
): Promise<boolean> {
|
||||
const updated = await this.delegate.compareAndSetAttempt(attempt, expected);
|
||||
if (updated) {
|
||||
this.changedRunIds.add(attempt.runId);
|
||||
this.changedAttemptIds.add(attempt.id);
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
compareAndSetRetryPolicy(
|
||||
policy: RunRetryPolicyRecord,
|
||||
expectedVersion: number,
|
||||
): Promise<boolean> {
|
||||
return this.delegate.compareAndSetRetryPolicy(policy, expectedVersion);
|
||||
}
|
||||
|
||||
appendEvent(event: RunEventRecord): Promise<void> {
|
||||
return this.delegate.appendEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Primary-only repository. Existing Shadow repositories keep their original
|
||||
* transaction implementation and never execute these projection participants.
|
||||
*/
|
||||
export class LegacySequelizeProjectedRunRepository extends LegacySequelizeRunRepository {
|
||||
private readonly participants: readonly SequelizeRunProjectionParticipant[];
|
||||
|
||||
constructor(
|
||||
private readonly projectedDatabase: Sequelize,
|
||||
participants: readonly SequelizeRunProjectionParticipant[],
|
||||
) {
|
||||
super(projectedDatabase);
|
||||
this.participants = [...participants];
|
||||
}
|
||||
|
||||
override async transaction<T>(
|
||||
work: (transaction: RunRepositoryTransaction) => Promise<T>,
|
||||
): Promise<T> {
|
||||
return this.projectedDatabase.transaction(
|
||||
{ type: Transaction.TYPES.IMMEDIATE },
|
||||
async (transaction) => {
|
||||
const runs = new LegacySequelizeRunTransaction(
|
||||
this.models,
|
||||
transaction,
|
||||
);
|
||||
const tracked = new TrackingRunRepositoryTransaction(runs);
|
||||
const result = await work(tracked);
|
||||
if (
|
||||
tracked.changedRunIds.size > 0 ||
|
||||
tracked.changedAttemptIds.size > 0
|
||||
) {
|
||||
const context: SequelizeRunProjectionContext = {
|
||||
transaction,
|
||||
runs,
|
||||
changedRunIds: [...tracked.changedRunIds],
|
||||
changedAttemptIds: [...tracked.changedAttemptIds],
|
||||
};
|
||||
for (const participant of this.participants) {
|
||||
await participant.apply(context);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { RUN_DISPATCH_LEASE_TABLE } from '../../../migrations/0009-run-dispatch-lease';
|
||||
import { RUN_DISPATCH_CANDIDATE_RUN_INDEX } from '../../../migrations/0010-run-dispatch-candidates';
|
||||
import {
|
||||
MAX_RUN_DISPATCH_CANDIDATE_PAGE_SIZE,
|
||||
assertRunDispatchCandidate,
|
||||
assertRunDispatchCandidateCursor,
|
||||
assertRunDispatchCandidatePageSize,
|
||||
type RunDispatchCandidate,
|
||||
} from '../../domain/runDispatchCandidate';
|
||||
import { assertRunDispatchLeaseVersion } from '../../domain/runDispatchLease';
|
||||
import type {
|
||||
ListRunDispatchCandidatesOptions,
|
||||
RunDispatchCandidateSource,
|
||||
} from '../../ports/runDispatchCandidateSource';
|
||||
|
||||
interface CandidateRow {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
projectId: string;
|
||||
taskId: string;
|
||||
taskRevision: string;
|
||||
priority: number | string;
|
||||
queuedAtMs: number | string;
|
||||
attemptCreatedAtMs: number | string;
|
||||
executorType: string;
|
||||
}
|
||||
|
||||
const CANDIDATE_ORDER = `
|
||||
r.priority DESC,
|
||||
r.queued_at_ms ASC,
|
||||
a.created_at_ms ASC,
|
||||
a.id ASC
|
||||
`;
|
||||
|
||||
export class LegacySequelizeRunDispatchCandidateSource
|
||||
implements RunDispatchCandidateSource
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy Run dispatch candidate source is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async listCandidates({
|
||||
observedAtMs,
|
||||
after,
|
||||
limit = MAX_RUN_DISPATCH_CANDIDATE_PAGE_SIZE,
|
||||
}: ListRunDispatchCandidatesOptions): Promise<RunDispatchCandidate[]> {
|
||||
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
|
||||
assertRunDispatchCandidatePageSize(limit);
|
||||
if (after) assertRunDispatchCandidateCursor(after);
|
||||
|
||||
const cursorPredicate = after
|
||||
? `AND (
|
||||
r.priority < :afterPriority
|
||||
OR (r.priority = :afterPriority AND r.queued_at_ms > :afterQueuedAtMs)
|
||||
OR (
|
||||
r.priority = :afterPriority
|
||||
AND r.queued_at_ms = :afterQueuedAtMs
|
||||
AND a.created_at_ms > :afterAttemptCreatedAtMs
|
||||
)
|
||||
OR (
|
||||
r.priority = :afterPriority
|
||||
AND r.queued_at_ms = :afterQueuedAtMs
|
||||
AND a.created_at_ms = :afterAttemptCreatedAtMs
|
||||
AND a.id > :afterAttemptId
|
||||
)
|
||||
)`
|
||||
: '';
|
||||
const rows = await this.database.query<CandidateRow>(
|
||||
`SELECT
|
||||
r.id AS runId,
|
||||
a.id AS attemptId,
|
||||
r.project_id AS projectId,
|
||||
r.task_id AS taskId,
|
||||
r.task_revision AS taskRevision,
|
||||
r.priority AS priority,
|
||||
r.queued_at_ms AS queuedAtMs,
|
||||
a.created_at_ms AS attemptCreatedAtMs,
|
||||
a.executor_type AS executorType
|
||||
FROM ${RUN_TABLE} r INDEXED BY ${RUN_DISPATCH_CANDIDATE_RUN_INDEX}
|
||||
INNER JOIN ${RUN_ATTEMPT_TABLE} a ON a.run_id = r.id
|
||||
LEFT JOIN ${RUN_DISPATCH_LEASE_TABLE} l ON l.attempt_id = a.id
|
||||
WHERE r.execution_owner = 'runtime'
|
||||
AND r.status IN ('queued', 'dispatching')
|
||||
AND r.queued_at_ms IS NOT NULL
|
||||
AND r.cancel_requested_at_ms IS NULL
|
||||
AND a.status = 'claimed'
|
||||
AND (
|
||||
l.attempt_id IS NULL
|
||||
OR l.status = 'released'
|
||||
OR (l.status = 'leased' AND l.expires_at_ms <= :observedAtMs)
|
||||
)
|
||||
${cursorPredicate}
|
||||
ORDER BY ${CANDIDATE_ORDER}
|
||||
LIMIT :limit`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
observedAtMs,
|
||||
limit,
|
||||
...(after
|
||||
? {
|
||||
afterPriority: after.priority,
|
||||
afterQueuedAtMs: after.queuedAtMs,
|
||||
afterAttemptCreatedAtMs: after.attemptCreatedAtMs,
|
||||
afterAttemptId: after.attemptId,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
);
|
||||
return rows.map((row) => {
|
||||
const candidate: RunDispatchCandidate = {
|
||||
runId: row.runId,
|
||||
attemptId: row.attemptId,
|
||||
projectId: row.projectId,
|
||||
taskId: row.taskId,
|
||||
taskRevision: row.taskRevision,
|
||||
priority: Number(row.priority),
|
||||
queuedAtMs: Number(row.queuedAtMs),
|
||||
attemptCreatedAtMs: Number(row.attemptCreatedAtMs),
|
||||
executorType: row.executorType,
|
||||
};
|
||||
assertRunDispatchCandidate(candidate);
|
||||
return candidate;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import {
|
||||
RUN_DISPATCH_LEASE_EXPIRY_INDEX,
|
||||
RUN_DISPATCH_LEASE_TABLE,
|
||||
} from '../../../migrations/0009-run-dispatch-lease';
|
||||
import {
|
||||
assertRunDispatchId,
|
||||
assertRunDispatchLeaseVersion,
|
||||
} from '../../domain/runDispatchLease';
|
||||
import {
|
||||
MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE,
|
||||
type ExpiredRunDispatchLeaseCandidate,
|
||||
type ListExpiredRunDispatchLeasesOptions,
|
||||
type RunDispatchLeaseExpirySource,
|
||||
} from '../../ports/runDispatchLeaseExpirySource';
|
||||
|
||||
interface ExpiredLeaseRow {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
expiresAtMs: number | string;
|
||||
}
|
||||
|
||||
function assertLimit(limit: number): void {
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
`limit must be between 1 and ${MAX_RUN_DISPATCH_LEASE_EXPIRY_PAGE_SIZE}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeRunDispatchLeaseExpirySource
|
||||
implements RunDispatchLeaseExpirySource
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy Run dispatch lease expiry source is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async listExpired({
|
||||
observedAtMs,
|
||||
after,
|
||||
limit = 16,
|
||||
}: ListExpiredRunDispatchLeasesOptions): Promise<
|
||||
readonly ExpiredRunDispatchLeaseCandidate[]
|
||||
> {
|
||||
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
|
||||
assertLimit(limit);
|
||||
if (after) {
|
||||
assertRunDispatchLeaseVersion('after.expiresAtMs', after.expiresAtMs);
|
||||
assertRunDispatchId('after.attemptId', after.attemptId);
|
||||
}
|
||||
const cursorPredicate = after
|
||||
? `AND (
|
||||
l.expires_at_ms > :afterExpiresAtMs
|
||||
OR (
|
||||
l.expires_at_ms = :afterExpiresAtMs
|
||||
AND l.attempt_id > :afterAttemptId
|
||||
)
|
||||
)`
|
||||
: '';
|
||||
const rows = await this.database.query<ExpiredLeaseRow>(
|
||||
`SELECT
|
||||
l.run_id AS runId,
|
||||
l.attempt_id AS attemptId,
|
||||
l.expires_at_ms AS expiresAtMs
|
||||
FROM ${RUN_DISPATCH_LEASE_TABLE} l INDEXED BY ${RUN_DISPATCH_LEASE_EXPIRY_INDEX}
|
||||
INNER JOIN ${RUN_TABLE} r ON r.id = l.run_id
|
||||
INNER JOIN ${RUN_ATTEMPT_TABLE} a ON a.id = l.attempt_id
|
||||
WHERE l.status = 'leased'
|
||||
AND l.expires_at_ms <= :observedAtMs
|
||||
AND r.execution_owner = 'runtime'
|
||||
AND r.status IN ('dispatching', 'running')
|
||||
AND a.run_id = r.id
|
||||
AND a.status IN ('claimed', 'starting', 'running')
|
||||
${cursorPredicate}
|
||||
ORDER BY l.expires_at_ms ASC, l.attempt_id ASC
|
||||
LIMIT :limit`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
observedAtMs,
|
||||
limit,
|
||||
...(after
|
||||
? {
|
||||
afterExpiresAtMs: after.expiresAtMs,
|
||||
afterAttemptId: after.attemptId,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
);
|
||||
return rows.map((row) => {
|
||||
const candidate = {
|
||||
runId: row.runId,
|
||||
attemptId: row.attemptId,
|
||||
expiresAtMs: Number(row.expiresAtMs),
|
||||
};
|
||||
assertRunDispatchId('runId', candidate.runId);
|
||||
assertRunDispatchId('attemptId', candidate.attemptId);
|
||||
assertRunDispatchLeaseVersion('expiresAtMs', candidate.expiresAtMs);
|
||||
if (candidate.expiresAtMs > observedAtMs) {
|
||||
throw new TypeError('Expiry source returned a live Run lease');
|
||||
}
|
||||
return candidate;
|
||||
});
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,164 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import {
|
||||
RUN_DISPATCH_LEASE_EXPIRY_INDEX,
|
||||
RUN_DISPATCH_LEASE_TABLE,
|
||||
} from '../../../migrations/0009-run-dispatch-lease';
|
||||
import { WORKER_REGISTRY_TABLE } from '../../../migrations/0008-worker-registry';
|
||||
import type { RunDispatchCandidate } from '../../domain/runDispatchCandidate';
|
||||
import {
|
||||
MAX_RUN_DISPATCH_RECOVERY_PAGE_SIZE,
|
||||
assertRecoverableRunDispatch,
|
||||
assertRunDispatchRecoveryCursor,
|
||||
assertRunDispatchRecoveryPageSize,
|
||||
type RecoverableRunDispatch,
|
||||
} from '../../domain/runDispatchRecovery';
|
||||
import {
|
||||
assertRunDispatchLeaseVersion,
|
||||
type RunDispatchLeaseRecord,
|
||||
} from '../../domain/runDispatchLease';
|
||||
import type {
|
||||
ListRecoverableRunDispatchesOptions,
|
||||
RunDispatchRecoverySource,
|
||||
} from '../../ports/runDispatchRecoverySource';
|
||||
|
||||
interface RecoveryRow {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
projectId: string;
|
||||
taskId: string;
|
||||
taskRevision: string;
|
||||
priority: number | string;
|
||||
queuedAtMs: number | string;
|
||||
attemptCreatedAtMs: number | string;
|
||||
executorType: string;
|
||||
version: number | string;
|
||||
leaseGeneration: number | string;
|
||||
workerId: string;
|
||||
workerSessionId: string;
|
||||
workerGeneration: number | string;
|
||||
leaseToken: string;
|
||||
acquiredAtMs: number | string;
|
||||
renewedAtMs: number | string;
|
||||
expiresAtMs: number | string;
|
||||
updatedAtMs: number | string;
|
||||
}
|
||||
|
||||
export class LegacySequelizeRunDispatchRecoverySource
|
||||
implements RunDispatchRecoverySource
|
||||
{
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy Run dispatch recovery source is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async listRecoverable({
|
||||
observedAtMs,
|
||||
after,
|
||||
limit = MAX_RUN_DISPATCH_RECOVERY_PAGE_SIZE,
|
||||
}: ListRecoverableRunDispatchesOptions): Promise<RecoverableRunDispatch[]> {
|
||||
assertRunDispatchLeaseVersion('observedAtMs', observedAtMs);
|
||||
assertRunDispatchRecoveryPageSize(limit);
|
||||
if (after) assertRunDispatchRecoveryCursor(after);
|
||||
const cursorPredicate = after
|
||||
? `AND (
|
||||
l.expires_at_ms > :afterExpiresAtMs
|
||||
OR (
|
||||
l.expires_at_ms = :afterExpiresAtMs
|
||||
AND l.attempt_id > :afterAttemptId
|
||||
)
|
||||
)`
|
||||
: '';
|
||||
const rows = await this.database.query<RecoveryRow>(
|
||||
`SELECT
|
||||
r.id AS runId,
|
||||
a.id AS attemptId,
|
||||
r.project_id AS projectId,
|
||||
r.task_id AS taskId,
|
||||
r.task_revision AS taskRevision,
|
||||
r.priority AS priority,
|
||||
r.queued_at_ms AS queuedAtMs,
|
||||
a.created_at_ms AS attemptCreatedAtMs,
|
||||
a.executor_type AS executorType,
|
||||
l.version AS version,
|
||||
l.lease_generation AS leaseGeneration,
|
||||
l.worker_id AS workerId,
|
||||
l.worker_session_id AS workerSessionId,
|
||||
l.worker_generation AS workerGeneration,
|
||||
l.lease_token AS leaseToken,
|
||||
l.acquired_at_ms AS acquiredAtMs,
|
||||
l.renewed_at_ms AS renewedAtMs,
|
||||
l.expires_at_ms AS expiresAtMs,
|
||||
l.updated_at_ms AS updatedAtMs
|
||||
FROM ${RUN_DISPATCH_LEASE_TABLE} l INDEXED BY ${RUN_DISPATCH_LEASE_EXPIRY_INDEX}
|
||||
INNER JOIN ${RUN_TABLE} r ON r.id = l.run_id
|
||||
INNER JOIN ${RUN_ATTEMPT_TABLE} a ON a.id = l.attempt_id
|
||||
INNER JOIN ${WORKER_REGISTRY_TABLE} w ON w.id = l.worker_id
|
||||
WHERE l.status = 'leased'
|
||||
AND l.expires_at_ms > :observedAtMs
|
||||
AND r.execution_owner = 'runtime'
|
||||
AND r.status = 'dispatching'
|
||||
AND r.cancel_requested_at_ms IS NULL
|
||||
AND r.queued_at_ms IS NOT NULL
|
||||
AND a.run_id = r.id
|
||||
AND a.status = 'claimed'
|
||||
AND w.session_id = l.worker_session_id
|
||||
AND w.generation = l.worker_generation
|
||||
AND w.status IN ('online', 'draining')
|
||||
AND w.lease_expires_at_ms > :observedAtMs
|
||||
${cursorPredicate}
|
||||
ORDER BY l.expires_at_ms ASC, l.attempt_id ASC
|
||||
LIMIT :limit`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: {
|
||||
observedAtMs,
|
||||
limit,
|
||||
...(after
|
||||
? {
|
||||
afterExpiresAtMs: after.expiresAtMs,
|
||||
afterAttemptId: after.attemptId,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
);
|
||||
return rows.map((row) => {
|
||||
const candidate: RunDispatchCandidate = {
|
||||
runId: row.runId,
|
||||
attemptId: row.attemptId,
|
||||
projectId: row.projectId,
|
||||
taskId: row.taskId,
|
||||
taskRevision: row.taskRevision,
|
||||
priority: Number(row.priority),
|
||||
queuedAtMs: Number(row.queuedAtMs),
|
||||
attemptCreatedAtMs: Number(row.attemptCreatedAtMs),
|
||||
executorType: row.executorType,
|
||||
};
|
||||
const lease: RunDispatchLeaseRecord = {
|
||||
attemptId: row.attemptId,
|
||||
runId: row.runId,
|
||||
status: 'leased',
|
||||
version: Number(row.version),
|
||||
leaseGeneration: Number(row.leaseGeneration),
|
||||
workerId: row.workerId,
|
||||
workerSessionId: row.workerSessionId,
|
||||
workerGeneration: Number(row.workerGeneration),
|
||||
leaseToken: row.leaseToken,
|
||||
acquiredAtMs: Number(row.acquiredAtMs),
|
||||
renewedAtMs: Number(row.renewedAtMs),
|
||||
expiresAtMs: Number(row.expiresAtMs),
|
||||
updatedAtMs: Number(row.updatedAtMs),
|
||||
};
|
||||
const recovery = { candidate, lease };
|
||||
assertRecoverableRunDispatch(recovery);
|
||||
return recovery;
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import { RUN_TABLE } from '../../../migrations/0002-run-schema';
|
||||
import {
|
||||
RUN_LOST_RETRY_INDEX,
|
||||
RUN_RETRY_POLICY_DUE_INDEX,
|
||||
RUN_RETRY_POLICY_TABLE,
|
||||
} from '../../../migrations/0011-run-retry-policy';
|
||||
import {
|
||||
MAX_RUN_LOST_RETRY_PAGE_SIZE,
|
||||
type ListRunLostRetryCandidatesOptions,
|
||||
type RunLostRetryCandidate,
|
||||
type RunLostRetrySource,
|
||||
} from '../../ports/runLostRetrySource';
|
||||
|
||||
interface RunLostRetryCandidateRow {
|
||||
runId: string;
|
||||
phase: 'lost' | 'retry_wait';
|
||||
availableAtMs: number | string;
|
||||
}
|
||||
|
||||
export class LegacySequelizeRunLostRetrySource implements RunLostRetrySource {
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy lost retry source is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async listCandidates({
|
||||
observedAtMs,
|
||||
limit = 16,
|
||||
}: ListRunLostRetryCandidatesOptions): Promise<
|
||||
readonly RunLostRetryCandidate[]
|
||||
> {
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new RangeError('observedAtMs must be a non-negative safe integer');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_RUN_LOST_RETRY_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
`limit must be between 1 and ${MAX_RUN_LOST_RETRY_PAGE_SIZE}`,
|
||||
);
|
||||
}
|
||||
const rows = await this.database.query<RunLostRetryCandidateRow>(
|
||||
`SELECT runId, phase, availableAtMs
|
||||
FROM (
|
||||
SELECT
|
||||
r.id AS runId,
|
||||
'lost' AS phase,
|
||||
0 AS availableAtMs
|
||||
FROM ${RUN_TABLE} r INDEXED BY ${RUN_LOST_RETRY_INDEX}
|
||||
WHERE r.execution_owner = 'runtime'
|
||||
AND r.status = 'lost'
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
r.id AS runId,
|
||||
'retry_wait' AS phase,
|
||||
p.next_attempt_at_ms AS availableAtMs
|
||||
FROM ${RUN_RETRY_POLICY_TABLE} p INDEXED BY ${RUN_RETRY_POLICY_DUE_INDEX}
|
||||
INNER JOIN ${RUN_TABLE} r ON r.id = p.run_id
|
||||
WHERE p.next_attempt_at_ms IS NOT NULL
|
||||
AND p.next_attempt_at_ms <= :observedAtMs
|
||||
AND r.execution_owner = 'runtime'
|
||||
AND r.status = 'retry_wait'
|
||||
) candidates
|
||||
ORDER BY availableAtMs ASC, runId ASC
|
||||
LIMIT :limit`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { observedAtMs, limit },
|
||||
},
|
||||
);
|
||||
return rows.map((row) => ({
|
||||
runId: row.runId,
|
||||
phase: row.phase,
|
||||
availableAtMs: Number(row.availableAtMs),
|
||||
}));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,278 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Sequelize,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { TASK_EXECUTION_REVISION_TABLE } from '../../../migrations/0012-task-execution-revisions';
|
||||
import { EXECUTOR_TYPES, type ExecutorType } from '../../domain/execution';
|
||||
import type { PinnedTaskExecutionRevision } from '../../domain/taskExecutionRevision';
|
||||
import {
|
||||
createPinnedTaskExecutionRevisionRecord,
|
||||
normalizePinnedTaskExecutionRevision,
|
||||
taskExecutionRevisionDigest,
|
||||
TaskExecutionRevisionCorruptError,
|
||||
} from '../../domain/taskExecutionRevisionRecord';
|
||||
import type {
|
||||
InsertTaskExecutionRevisionResult,
|
||||
TaskExecutionRevisionRepository,
|
||||
} from '../../ports/taskExecutionRevisionRepository';
|
||||
import type { TaskExecutionRevisionRequest } from '../../ports/taskExecutionRevisionSource';
|
||||
|
||||
interface TaskExecutionRevisionRow {
|
||||
projectId: string;
|
||||
taskId: string;
|
||||
taskRevision: string;
|
||||
executorType: string;
|
||||
executionTemplate: string;
|
||||
contextRef: string;
|
||||
contentDigest: string;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface TaskExecutionRevisionInstance
|
||||
extends Model<TaskExecutionRevisionRow, TaskExecutionRevisionRow>,
|
||||
TaskExecutionRevisionRow {}
|
||||
|
||||
export class TaskExecutionRevisionConflictError extends Error {
|
||||
constructor(
|
||||
readonly projectId: string,
|
||||
readonly taskId: string,
|
||||
readonly taskRevision: string,
|
||||
) {
|
||||
super(
|
||||
`Task execution revision ${projectId}/${taskId}@${taskRevision} is immutable`,
|
||||
);
|
||||
this.name = 'TaskExecutionRevisionConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
function defineTaskExecutionRevisionModel(
|
||||
database: Sequelize,
|
||||
): ModelStatic<TaskExecutionRevisionInstance> {
|
||||
return database.define<TaskExecutionRevisionInstance>(
|
||||
'Ql3TaskExecutionRevision',
|
||||
{
|
||||
projectId: {
|
||||
field: 'project_id',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
taskId: {
|
||||
field: 'task_id',
|
||||
type: DataTypes.STRING(255),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
taskRevision: {
|
||||
field: 'task_revision',
|
||||
type: DataTypes.STRING(128),
|
||||
allowNull: false,
|
||||
primaryKey: true,
|
||||
},
|
||||
executorType: {
|
||||
field: 'executor_type',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
executionTemplate: {
|
||||
field: 'execution_template',
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: false,
|
||||
},
|
||||
contextRef: {
|
||||
field: 'context_ref',
|
||||
type: DataTypes.STRING(512),
|
||||
allowNull: false,
|
||||
},
|
||||
contentDigest: {
|
||||
field: 'content_digest',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
createdAtMs: {
|
||||
field: 'created_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: TASK_EXECUTION_REVISION_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function assertIdentity(name: string, value: string, maximum: number): void {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > maximum ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) {
|
||||
throw new TypeError(`${name} is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertRequest(request: Readonly<TaskExecutionRevisionRequest>): void {
|
||||
if (!request || typeof request !== 'object' || Array.isArray(request)) {
|
||||
throw new TypeError('Task execution revision request must be an object');
|
||||
}
|
||||
assertIdentity('projectId', request.projectId, 128);
|
||||
assertIdentity('taskId', request.taskId, 255);
|
||||
assertIdentity('taskRevision', request.taskRevision, 128);
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
function rowToRevision(
|
||||
row: TaskExecutionRevisionRow,
|
||||
): PinnedTaskExecutionRevision {
|
||||
if (!EXECUTOR_TYPES.includes(row.executorType as ExecutorType)) {
|
||||
throw new TaskExecutionRevisionCorruptError(
|
||||
'Stored Task execution revision has an invalid executor type',
|
||||
);
|
||||
}
|
||||
let execution: unknown;
|
||||
try {
|
||||
execution = JSON.parse(row.executionTemplate);
|
||||
} catch {
|
||||
throw new TaskExecutionRevisionCorruptError(
|
||||
'Stored Task execution revision template is not valid JSON',
|
||||
);
|
||||
}
|
||||
try {
|
||||
const normalized = normalizePinnedTaskExecutionRevision({
|
||||
projectId: row.projectId,
|
||||
taskId: row.taskId,
|
||||
taskRevision: row.taskRevision,
|
||||
executorType: row.executorType as ExecutorType,
|
||||
execution: execution as PinnedTaskExecutionRevision['execution'],
|
||||
contextRef: row.contextRef,
|
||||
});
|
||||
if (JSON.stringify(normalized.execution) !== row.executionTemplate) {
|
||||
throw new TaskExecutionRevisionCorruptError(
|
||||
'Stored Task execution revision template is not canonical',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!/^[0-9a-f]{64}$/.test(row.contentDigest) ||
|
||||
taskExecutionRevisionDigest(normalized) !== row.contentDigest
|
||||
) {
|
||||
throw new TaskExecutionRevisionCorruptError(
|
||||
'Stored Task execution revision digest does not match its content',
|
||||
);
|
||||
}
|
||||
const createdAtMs = Number(row.createdAtMs);
|
||||
return createPinnedTaskExecutionRevisionRecord(normalized, createdAtMs);
|
||||
} catch (error) {
|
||||
if (error instanceof TaskExecutionRevisionCorruptError) throw error;
|
||||
throw new TaskExecutionRevisionCorruptError(
|
||||
`Stored Task execution revision is invalid: ${
|
||||
error instanceof Error ? error.message : 'unknown validation error'
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class LegacySequelizeTaskExecutionRevisionRepository
|
||||
implements TaskExecutionRevisionRepository
|
||||
{
|
||||
private readonly revision: ModelStatic<TaskExecutionRevisionInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
if (database.getDialect() !== 'sqlite') {
|
||||
throw new TypeError(
|
||||
'Legacy Task execution revision repository is SQLite-only; cluster-control requires a PostgreSQL adapter',
|
||||
);
|
||||
}
|
||||
this.revision = defineTaskExecutionRevisionModel(database);
|
||||
}
|
||||
|
||||
async resolve(
|
||||
request: Readonly<TaskExecutionRevisionRequest>,
|
||||
): Promise<PinnedTaskExecutionRevision | null> {
|
||||
assertRequest(request);
|
||||
const row = (await this.revision.findOne({
|
||||
where: {
|
||||
projectId: request.projectId,
|
||||
taskId: request.taskId,
|
||||
taskRevision: request.taskRevision,
|
||||
},
|
||||
raw: true,
|
||||
})) as unknown as TaskExecutionRevisionRow | null;
|
||||
return row ? rowToRevision(row) : null;
|
||||
}
|
||||
|
||||
async insert(
|
||||
revision: PinnedTaskExecutionRevision,
|
||||
createdAtMs: number,
|
||||
): Promise<InsertTaskExecutionRevisionResult> {
|
||||
const record = createPinnedTaskExecutionRevisionRecord(
|
||||
revision,
|
||||
createdAtMs,
|
||||
);
|
||||
const values: TaskExecutionRevisionRow = {
|
||||
projectId: record.projectId,
|
||||
taskId: record.taskId,
|
||||
taskRevision: record.taskRevision,
|
||||
executorType: record.executorType,
|
||||
executionTemplate: JSON.stringify(record.execution),
|
||||
contextRef: record.contextRef,
|
||||
contentDigest: record.contentDigest,
|
||||
createdAtMs: record.createdAtMs,
|
||||
};
|
||||
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
try {
|
||||
await this.revision.create(values);
|
||||
return 'inserted';
|
||||
} catch (error) {
|
||||
if (error instanceof UniqueConstraintError) {
|
||||
const existing = await this.resolve(record);
|
||||
if (
|
||||
existing &&
|
||||
taskExecutionRevisionDigest(existing) === record.contentDigest
|
||||
) {
|
||||
return 'idempotent';
|
||||
}
|
||||
throw new TaskExecutionRevisionConflictError(
|
||||
record.projectId,
|
||||
record.taskId,
|
||||
record.taskRevision,
|
||||
);
|
||||
}
|
||||
if (errorCode(error) === 'SQLITE_BUSY' && attempt < 4) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new Error('Task execution revision insert retry budget exhausted');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,504 @@
|
||||
import {
|
||||
DataTypes,
|
||||
Model,
|
||||
ModelStatic,
|
||||
Op,
|
||||
Sequelize,
|
||||
Transaction,
|
||||
UniqueConstraintError,
|
||||
} from 'sequelize';
|
||||
import { WORKER_REGISTRY_TABLE } from '../../../migrations/0008-worker-registry';
|
||||
import {
|
||||
WORKER_STATUSES,
|
||||
WorkerFenceRejectedError,
|
||||
WorkerSessionConflictError,
|
||||
assertWorkerConcurrency,
|
||||
assertWorkerId,
|
||||
assertWorkerSessionId,
|
||||
hashWorkerCapabilities,
|
||||
parseWorkerCapabilities,
|
||||
type WorkerRecord,
|
||||
type WorkerStatus,
|
||||
} from '../../domain/worker';
|
||||
import {
|
||||
MAX_AVAILABLE_WORKER_PAGE_SIZE,
|
||||
type AvailableWorkerPage,
|
||||
type HeartbeatWorkerSessionCommand,
|
||||
type RegisterWorkerSessionCommand,
|
||||
type RegisterWorkerSessionResult,
|
||||
type TransitionWorkerSessionCommand,
|
||||
type WorkerRegistryRepository,
|
||||
} from '../../ports/workerRegistryRepository';
|
||||
|
||||
interface WorkerRow {
|
||||
id: string;
|
||||
sessionId: string;
|
||||
generation: number;
|
||||
status: string;
|
||||
version: number;
|
||||
capabilitiesJson: string;
|
||||
capabilitiesHash: string;
|
||||
maxConcurrentRuns: number;
|
||||
availableSlots: number;
|
||||
registeredAtMs: number;
|
||||
lastHeartbeatAtMs: number;
|
||||
leaseExpiresAtMs: number;
|
||||
updatedAtMs: number;
|
||||
}
|
||||
|
||||
interface WorkerInstance extends Model<WorkerRow, WorkerRow>, WorkerRow {}
|
||||
|
||||
function defineWorkerModel(database: Sequelize): ModelStatic<WorkerInstance> {
|
||||
return database.define<WorkerInstance>(
|
||||
'Ql3WorkerRegistry',
|
||||
{
|
||||
id: { type: DataTypes.STRING(128), primaryKey: true },
|
||||
sessionId: {
|
||||
field: 'session_id',
|
||||
type: DataTypes.STRING(36),
|
||||
allowNull: false,
|
||||
},
|
||||
generation: { type: DataTypes.INTEGER, allowNull: false },
|
||||
status: { type: DataTypes.STRING(16), allowNull: false },
|
||||
version: { type: DataTypes.INTEGER, allowNull: false },
|
||||
capabilitiesJson: {
|
||||
field: 'capabilities_json',
|
||||
type: DataTypes.TEXT,
|
||||
allowNull: false,
|
||||
},
|
||||
capabilitiesHash: {
|
||||
field: 'capabilities_hash',
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
},
|
||||
maxConcurrentRuns: {
|
||||
field: 'max_concurrent_runs',
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
},
|
||||
availableSlots: {
|
||||
field: 'available_slots',
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false,
|
||||
},
|
||||
registeredAtMs: {
|
||||
field: 'registered_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
lastHeartbeatAtMs: {
|
||||
field: 'last_heartbeat_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
leaseExpiresAtMs: {
|
||||
field: 'lease_expires_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
updatedAtMs: {
|
||||
field: 'updated_at_ms',
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
tableName: WORKER_REGISTRY_TABLE,
|
||||
timestamps: false,
|
||||
freezeTableName: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function nonNegativeTimestamp(value: number, name: string): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new RangeError(`${name} must be a non-negative safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function positiveInteger(value: number, name: string): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new RangeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertCapabilities(
|
||||
capabilitiesJson: string,
|
||||
capabilitiesHash: string,
|
||||
): void {
|
||||
parseWorkerCapabilities(capabilitiesJson);
|
||||
if (
|
||||
!/^[0-9a-f]{64}$/.test(capabilitiesHash) ||
|
||||
hashWorkerCapabilities(capabilitiesJson) !== capabilitiesHash
|
||||
) {
|
||||
throw new TypeError('capabilitiesHash does not match capabilitiesJson');
|
||||
}
|
||||
}
|
||||
|
||||
function toRecord(row: WorkerRow): WorkerRecord {
|
||||
if (!WORKER_STATUSES.includes(row.status as WorkerStatus)) {
|
||||
throw new Error(`Worker ${row.id} has an invalid status`);
|
||||
}
|
||||
assertWorkerId(row.id);
|
||||
assertWorkerSessionId(row.sessionId);
|
||||
positiveInteger(Number(row.generation), 'generation');
|
||||
nonNegativeTimestamp(Number(row.version), 'version');
|
||||
assertCapabilities(row.capabilitiesJson, row.capabilitiesHash);
|
||||
assertWorkerConcurrency(
|
||||
Number(row.maxConcurrentRuns),
|
||||
Number(row.availableSlots),
|
||||
);
|
||||
for (const [name, value] of [
|
||||
['registeredAtMs', row.registeredAtMs],
|
||||
['lastHeartbeatAtMs', row.lastHeartbeatAtMs],
|
||||
['leaseExpiresAtMs', row.leaseExpiresAtMs],
|
||||
['updatedAtMs', row.updatedAtMs],
|
||||
] as const) {
|
||||
nonNegativeTimestamp(Number(value), name);
|
||||
}
|
||||
if (
|
||||
Number(row.lastHeartbeatAtMs) < Number(row.registeredAtMs) ||
|
||||
Number(row.leaseExpiresAtMs) <= Number(row.lastHeartbeatAtMs) ||
|
||||
Number(row.updatedAtMs) < Number(row.lastHeartbeatAtMs)
|
||||
) {
|
||||
throw new Error(`Worker ${row.id} timestamps are corrupt`);
|
||||
}
|
||||
return {
|
||||
id: row.id,
|
||||
sessionId: row.sessionId,
|
||||
generation: Number(row.generation),
|
||||
status: row.status as WorkerStatus,
|
||||
version: Number(row.version),
|
||||
capabilities: parseWorkerCapabilities(row.capabilitiesJson),
|
||||
capabilitiesHash: row.capabilitiesHash,
|
||||
maxConcurrentRuns: Number(row.maxConcurrentRuns),
|
||||
availableSlots: Number(row.availableSlots),
|
||||
registeredAtMs: Number(row.registeredAtMs),
|
||||
lastHeartbeatAtMs: Number(row.lastHeartbeatAtMs),
|
||||
leaseExpiresAtMs: Number(row.leaseExpiresAtMs),
|
||||
updatedAtMs: Number(row.updatedAtMs),
|
||||
};
|
||||
}
|
||||
|
||||
function assertRegister(command: RegisterWorkerSessionCommand): void {
|
||||
assertWorkerId(command.workerId);
|
||||
assertWorkerSessionId(command.sessionId);
|
||||
assertCapabilities(command.capabilitiesJson, command.capabilitiesHash);
|
||||
assertWorkerConcurrency(command.maxConcurrentRuns, command.availableSlots);
|
||||
nonNegativeTimestamp(command.registeredAtMs, 'registeredAtMs');
|
||||
nonNegativeTimestamp(command.leaseExpiresAtMs, 'leaseExpiresAtMs');
|
||||
if (command.leaseExpiresAtMs <= command.registeredAtMs) {
|
||||
throw new RangeError('leaseExpiresAtMs must be after registeredAtMs');
|
||||
}
|
||||
}
|
||||
|
||||
function assertHeartbeat(command: HeartbeatWorkerSessionCommand): void {
|
||||
assertWorkerId(command.workerId);
|
||||
assertWorkerSessionId(command.sessionId);
|
||||
positiveInteger(command.generation, 'generation');
|
||||
nonNegativeTimestamp(command.expectedVersion, 'expectedVersion');
|
||||
nonNegativeTimestamp(command.availableSlots, 'availableSlots');
|
||||
nonNegativeTimestamp(command.heartbeatAtMs, 'heartbeatAtMs');
|
||||
nonNegativeTimestamp(command.leaseExpiresAtMs, 'leaseExpiresAtMs');
|
||||
if (command.leaseExpiresAtMs <= command.heartbeatAtMs) {
|
||||
throw new RangeError('leaseExpiresAtMs must be after heartbeatAtMs');
|
||||
}
|
||||
}
|
||||
|
||||
function assertTransition(command: TransitionWorkerSessionCommand): void {
|
||||
assertWorkerId(command.workerId);
|
||||
assertWorkerSessionId(command.sessionId);
|
||||
positiveInteger(command.generation, 'generation');
|
||||
nonNegativeTimestamp(command.expectedVersion, 'expectedVersion');
|
||||
nonNegativeTimestamp(command.transitionedAtMs, 'transitionedAtMs');
|
||||
if (command.status !== 'draining' && command.status !== 'offline') {
|
||||
throw new TypeError('Worker transition status is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function fenceReason(
|
||||
row: WorkerRow | null,
|
||||
command: {
|
||||
workerId: string;
|
||||
sessionId: string;
|
||||
generation: number;
|
||||
expectedVersion: number;
|
||||
},
|
||||
): WorkerFenceRejectedError['reason'] | undefined {
|
||||
if (!row) return 'missing';
|
||||
if (row.sessionId !== command.sessionId) return 'session_mismatch';
|
||||
if (Number(row.generation) !== command.generation) {
|
||||
return 'generation_mismatch';
|
||||
}
|
||||
if (Number(row.version) !== command.expectedVersion) {
|
||||
return 'version_mismatch';
|
||||
}
|
||||
if (row.status === 'offline') return 'offline';
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
for (const candidate of [
|
||||
error,
|
||||
'original' in error ? error.original : undefined,
|
||||
'parent' in error ? error.parent : undefined,
|
||||
]) {
|
||||
if (
|
||||
candidate &&
|
||||
typeof candidate === 'object' &&
|
||||
'code' in candidate &&
|
||||
typeof candidate.code === 'string'
|
||||
) {
|
||||
return candidate.code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function retryDelay(attempt: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 2 ** attempt));
|
||||
}
|
||||
|
||||
export class LegacySequelizeWorkerRegistryRepository
|
||||
implements WorkerRegistryRepository
|
||||
{
|
||||
private readonly worker: ModelStatic<WorkerInstance>;
|
||||
|
||||
constructor(private readonly database: Sequelize) {
|
||||
this.worker = defineWorkerModel(database);
|
||||
}
|
||||
|
||||
async findById(workerId: string): Promise<WorkerRecord | null> {
|
||||
assertWorkerId(workerId);
|
||||
const row = (await this.worker.findByPk(workerId, {
|
||||
raw: true,
|
||||
})) as unknown as WorkerRow | null;
|
||||
return row ? toRecord(row) : null;
|
||||
}
|
||||
|
||||
async register(
|
||||
command: RegisterWorkerSessionCommand,
|
||||
): Promise<RegisterWorkerSessionResult> {
|
||||
assertRegister(command);
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
try {
|
||||
return await this.database.transaction(
|
||||
this.database.getDialect() === 'sqlite'
|
||||
? { type: Transaction.TYPES.IMMEDIATE }
|
||||
: {},
|
||||
async (transaction) => {
|
||||
const current = await this.worker.findByPk(command.workerId, {
|
||||
transaction,
|
||||
lock: transaction.LOCK.UPDATE,
|
||||
});
|
||||
if (!current) {
|
||||
const created = await this.worker.create(
|
||||
{
|
||||
id: command.workerId,
|
||||
sessionId: command.sessionId,
|
||||
generation: 1,
|
||||
status: 'online',
|
||||
version: 0,
|
||||
capabilitiesJson: command.capabilitiesJson,
|
||||
capabilitiesHash: command.capabilitiesHash,
|
||||
maxConcurrentRuns: command.maxConcurrentRuns,
|
||||
availableSlots: command.availableSlots,
|
||||
registeredAtMs: command.registeredAtMs,
|
||||
lastHeartbeatAtMs: command.registeredAtMs,
|
||||
leaseExpiresAtMs: command.leaseExpiresAtMs,
|
||||
updatedAtMs: command.registeredAtMs,
|
||||
},
|
||||
{ transaction },
|
||||
);
|
||||
return {
|
||||
worker: toRecord(created.get()),
|
||||
replacedSession: false,
|
||||
};
|
||||
}
|
||||
|
||||
const row = current.get();
|
||||
if (row.sessionId === command.sessionId) {
|
||||
if (
|
||||
row.capabilitiesHash !== command.capabilitiesHash ||
|
||||
Number(row.maxConcurrentRuns) !== command.maxConcurrentRuns ||
|
||||
Number(row.availableSlots) !== command.availableSlots
|
||||
) {
|
||||
throw new WorkerSessionConflictError(command.workerId);
|
||||
}
|
||||
if (Number(row.leaseExpiresAtMs) <= command.registeredAtMs) {
|
||||
throw new WorkerFenceRejectedError(
|
||||
command.workerId,
|
||||
'lease_expired',
|
||||
);
|
||||
}
|
||||
return { worker: toRecord(row), replacedSession: false };
|
||||
}
|
||||
|
||||
const next: Partial<WorkerRow> = {
|
||||
sessionId: command.sessionId,
|
||||
generation: Number(row.generation) + 1,
|
||||
status: 'online',
|
||||
version: Number(row.version) + 1,
|
||||
capabilitiesJson: command.capabilitiesJson,
|
||||
capabilitiesHash: command.capabilitiesHash,
|
||||
maxConcurrentRuns: command.maxConcurrentRuns,
|
||||
availableSlots: command.availableSlots,
|
||||
registeredAtMs: command.registeredAtMs,
|
||||
lastHeartbeatAtMs: command.registeredAtMs,
|
||||
leaseExpiresAtMs: command.leaseExpiresAtMs,
|
||||
updatedAtMs: command.registeredAtMs,
|
||||
};
|
||||
await current.update(next, { transaction });
|
||||
return {
|
||||
worker: toRecord(current.get()),
|
||||
replacedSession: true,
|
||||
};
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
(error instanceof UniqueConstraintError ||
|
||||
errorCode(error) === 'SQLITE_BUSY') &&
|
||||
attempt < 4
|
||||
) {
|
||||
await retryDelay(attempt);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new Error('Worker registration retry budget exhausted');
|
||||
}
|
||||
|
||||
async heartbeat(
|
||||
command: HeartbeatWorkerSessionCommand,
|
||||
): Promise<WorkerRecord> {
|
||||
assertHeartbeat(command);
|
||||
return this.database.transaction(async (transaction) => {
|
||||
const current = await this.worker.findByPk(command.workerId, {
|
||||
transaction,
|
||||
lock: transaction.LOCK.UPDATE,
|
||||
});
|
||||
const row = current?.get() ?? null;
|
||||
const reason = fenceReason(row, command);
|
||||
if (reason) throw new WorkerFenceRejectedError(command.workerId, reason);
|
||||
if (!current || !row) {
|
||||
throw new WorkerFenceRejectedError(command.workerId, 'missing');
|
||||
}
|
||||
if (Number(row.leaseExpiresAtMs) <= command.heartbeatAtMs) {
|
||||
throw new WorkerFenceRejectedError(command.workerId, 'lease_expired');
|
||||
}
|
||||
if (command.heartbeatAtMs < Number(row.lastHeartbeatAtMs)) {
|
||||
throw new RangeError('heartbeatAtMs must not move backwards');
|
||||
}
|
||||
assertWorkerConcurrency(
|
||||
Number(row.maxConcurrentRuns),
|
||||
command.availableSlots,
|
||||
);
|
||||
await current.update(
|
||||
{
|
||||
version: Number(row.version) + 1,
|
||||
availableSlots:
|
||||
row.status === 'draining' ? 0 : command.availableSlots,
|
||||
lastHeartbeatAtMs: command.heartbeatAtMs,
|
||||
leaseExpiresAtMs: command.leaseExpiresAtMs,
|
||||
updatedAtMs: command.heartbeatAtMs,
|
||||
},
|
||||
{ transaction },
|
||||
);
|
||||
return toRecord(current.get());
|
||||
});
|
||||
}
|
||||
|
||||
async transition(
|
||||
command: TransitionWorkerSessionCommand,
|
||||
): Promise<WorkerRecord> {
|
||||
assertTransition(command);
|
||||
return this.database.transaction(async (transaction) => {
|
||||
const current = await this.worker.findByPk(command.workerId, {
|
||||
transaction,
|
||||
lock: transaction.LOCK.UPDATE,
|
||||
});
|
||||
const row = current?.get() ?? null;
|
||||
if (
|
||||
row &&
|
||||
row.sessionId === command.sessionId &&
|
||||
Number(row.generation) === command.generation &&
|
||||
row.status === command.status &&
|
||||
Number(row.version) === command.expectedVersion + 1 &&
|
||||
Number(row.updatedAtMs) === command.transitionedAtMs
|
||||
) {
|
||||
return toRecord(row);
|
||||
}
|
||||
const reason = fenceReason(row, command);
|
||||
if (reason) throw new WorkerFenceRejectedError(command.workerId, reason);
|
||||
if (!current || !row) {
|
||||
throw new WorkerFenceRejectedError(command.workerId, 'missing');
|
||||
}
|
||||
if (
|
||||
command.status === 'draining' &&
|
||||
Number(row.leaseExpiresAtMs) <= command.transitionedAtMs
|
||||
) {
|
||||
throw new WorkerFenceRejectedError(command.workerId, 'lease_expired');
|
||||
}
|
||||
if (command.transitionedAtMs < Number(row.lastHeartbeatAtMs)) {
|
||||
throw new RangeError('transitionedAtMs must not move backwards');
|
||||
}
|
||||
await current.update(
|
||||
{
|
||||
status: command.status,
|
||||
version: Number(row.version) + 1,
|
||||
availableSlots: 0,
|
||||
updatedAtMs: command.transitionedAtMs,
|
||||
},
|
||||
{ transaction },
|
||||
);
|
||||
return toRecord(current.get());
|
||||
});
|
||||
}
|
||||
|
||||
async listAvailable({
|
||||
observedAtMs,
|
||||
afterWorkerId,
|
||||
limit = 32,
|
||||
}: {
|
||||
observedAtMs: number;
|
||||
afterWorkerId?: string;
|
||||
limit?: number;
|
||||
}): Promise<AvailableWorkerPage> {
|
||||
nonNegativeTimestamp(observedAtMs, 'observedAtMs');
|
||||
if (afterWorkerId !== undefined) assertWorkerId(afterWorkerId);
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_AVAILABLE_WORKER_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_AVAILABLE_WORKER_PAGE_SIZE',
|
||||
);
|
||||
}
|
||||
const rows = (await this.worker.findAll({
|
||||
where: {
|
||||
status: 'online',
|
||||
availableSlots: { [Op.gt]: 0 },
|
||||
leaseExpiresAtMs: { [Op.gt]: observedAtMs },
|
||||
...(afterWorkerId === undefined
|
||||
? {}
|
||||
: { id: { [Op.gt]: afterWorkerId } }),
|
||||
},
|
||||
order: [['id', 'ASC']],
|
||||
limit: limit + 1,
|
||||
raw: true,
|
||||
})) as unknown as WorkerRow[];
|
||||
const truncated = rows.length > limit;
|
||||
const bounded = rows.slice(0, limit).map(toRecord);
|
||||
return {
|
||||
workers: bounded,
|
||||
truncated,
|
||||
...(bounded.length === 0
|
||||
? {}
|
||||
: { nextCursor: bounded[bounded.length - 1].id }),
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user