feat(ql3): terminalize copilot diagnosis failures

This commit is contained in:
whyour
2026-08-15 23:16:26 +08:00
parent 5fc70010f2
commit 0a31c3364b
26 changed files with 3217 additions and 127 deletions
@@ -5,6 +5,9 @@ const {
CopilotFailureDiagnosisApplicationService,
CopilotFailureDiagnosisApplicationUnavailableError,
} = require('@qinglong/ai/failure-diagnosis-application');
const {
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError,
} = require('@qinglong/ai/failure-diagnosis-pre-model-terminalization');
const {
BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION,
} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool');
@@ -80,9 +83,12 @@ function fixture(options = {}) {
let resolvedKeyCopies = [];
let toolCalls = 0;
let modelCalls = 0;
let terminalizationCalls = 0;
let releaseTool;
const toolGate = options.blockTool
? new Promise((resolve) => { releaseTool = resolve; })
? new Promise((resolve) => {
releaseTool = resolve;
})
: Promise.resolve();
const admissions = {
async findByRequestId(requestId) {
@@ -108,20 +114,30 @@ function fixture(options = {}) {
},
};
const snapshots = {
async findCurrent() { return { snapshot: snapshot(), committedAtMs: NOW }; },
async findCurrent() {
return { snapshot: snapshot(), committedAtMs: NOW };
},
};
const runs = {
async findRunById() {
return {
id: 'source-run-1', projectId: 'project-1', status: 'failed',
version: 8, eventSequence: 8,
id: 'source-run-1',
projectId: 'project-1',
status: 'failed',
version: 8,
eventSequence: 8,
};
},
async findLatestAttemptByRunId() {
return {
id: 'source-attempt-1', runId: 'source-run-1', attempt: 1,
status: 'failed', executorType: 'remote_worker', callbackSequence: 0,
createdAtMs: NOW - 3000, finishedAtMs: NOW - 1000,
id: 'source-attempt-1',
runId: 'source-run-1',
attempt: 1,
status: 'failed',
executorType: 'remote_worker',
callbackSequence: 0,
createdAtMs: NOW - 3000,
finishedAtMs: NOW - 1000,
logArtifactId: `wlog-${'d'.repeat(30)}`,
};
},
@@ -140,8 +156,12 @@ function fixture(options = {}) {
artifacts = { input, preview };
return { status: 'inserted' };
},
async findInput() { return artifacts?.input ?? null; },
async findPreview() { return artifacts?.preview ?? null; },
async findInput() {
return artifacts?.input ?? null;
},
async findPreview() {
return artifacts?.preview ?? null;
},
};
const invocationKeys = {
async active() {
@@ -156,23 +176,58 @@ function fixture(options = {}) {
return { keyId, key: copy };
},
};
const unlocks = { async findByRequestId() { return null; }, async commit() {} };
const unlocks = {
async findByRequestId() {
return null;
},
async commit() {},
};
const tool = {
admissions,
snapshots,
runs,
artifacts: artifactRepository,
invocationKeys,
resultKeys: { async resolve() { return null; } },
stepRuns: { async findById() { return null; } },
barriers: {}, completions: {}, failureCompletions: {},
resultKeyCatalog: {}, resultRekeys: {}, logs: {}, unlocks,
resultKeys: {
async resolve() {
return null;
},
},
stepRuns: {
async findById() {
return null;
},
},
barriers: {},
completions: {},
failureCompletions: {},
resultKeyCatalog: {},
resultRekeys: {},
logs: {},
unlocks,
};
const model = {
admissions,
unlocks,
toolResults: {}, modelInvocations: {}, outputs: {}, gateway: {},
successfulCompletion: {}, finalizations: {},
toolResults: {},
modelInvocations: {},
outputs: {},
gateway: {},
successfulCompletion: {},
finalizations: {},
};
const terminalizations = {
repository: {
async findByRequestId() {
return null;
},
async readAuthority() {
return {};
},
async commit() {
return {};
},
},
};
const service = new CopilotFailureDiagnosisApplicationService({
admissions,
@@ -183,13 +238,15 @@ function fixture(options = {}) {
authorizer: {
async authorize() {
return {
effect: 'allow', reasons: ['role_grant'],
effect: 'allow',
reasons: ['role_grant'],
fence: { projectVersion: 1, bindingVersion: 1 },
};
},
},
tool,
model,
terminalizations,
modelIntent: MODEL,
executionTimeoutMs: 60_000,
now: () => NOW,
@@ -197,10 +254,24 @@ function fixture(options = {}) {
toolCalls += 1;
await toolGate;
return options.toolFailure
? { outcome: 'failed', completionStatus: 'created', unlockStatus: null }
? {
outcome: 'failed',
completionStatus: 'created',
unlockStatus: null,
completion: { completionDigest: 'f'.repeat(64) },
}
: {
outcome: 'succeeded', completionStatus: 'created',
unlockStatus: 'created', completion: {}, unlock: {},
outcome: 'succeeded',
completionStatus: 'created',
unlockStatus: 'created',
completion: { completionDigest: 'e'.repeat(64) },
output: {
status: 'available',
runId: 'source-run-1',
attemptId: 'source-attempt-1',
profile: 'cluster-control',
},
unlock: {},
};
},
async executeModel() {
@@ -211,13 +282,36 @@ function fixture(options = {}) {
finalization: { requestId: 'diagnosis-request-1' },
};
},
async terminalizeBeforeModel(_requestId, trigger) {
terminalizationCalls += 1;
if (
trigger.kind === 'boundary' ||
(trigger.kind === 'tool_projection' &&
trigger.output.status === 'available')
) {
throw new CopilotFailureDiagnosisPreModelTerminalizationNotReadyError();
}
return {
status: 'created',
receipt: {
requestId: 'diagnosis-request-1',
reason: 'tool_failed',
outcome: 'failed',
},
};
},
});
return {
service,
releaseTool: () => releaseTool?.(),
state: () => ({
plan, artifacts, toolCalls, modelCalls,
activeKeyCopies, resolvedKeyCopies,
plan,
artifacts,
toolCalls,
modelCalls,
terminalizationCalls,
activeKeyCopies,
resolvedKeyCopies,
}),
};
}
@@ -228,6 +322,7 @@ test('application derives, admits and executes one server-owned diagnosis with e
assert.equal(first.admissionStatus, 'created');
assert.equal(first.tool.outcome, 'succeeded');
assert.equal(first.model.outcome, 'succeeded');
assert.equal(first.terminalization, null);
assert.equal(first.terminalizationRequired, false);
const second = await testFixture.service.execute(command());
assert.equal(second.admissionStatus, 'existing');
@@ -235,9 +330,18 @@ test('application derives, admits and executes one server-owned diagnosis with e
assert.equal(state.toolCalls, 2);
assert.equal(state.modelCalls, 2);
assert.equal(state.plan.source.attemptId, 'source-attempt-1');
assert.equal(state.plan.tool.invocationArtifact.artifactId.startsWith('cdia:'), true);
assert.equal(state.activeKeyCopies[0].every((value) => value === 0), true);
assert.equal(state.resolvedKeyCopies[0].every((value) => value === 0), true);
assert.equal(
state.plan.tool.invocationArtifact.artifactId.startsWith('cdia:'),
true,
);
assert.equal(
state.activeKeyCopies[0].every((value) => value === 0),
true,
);
assert.equal(
state.resolvedKeyCopies[0].every((value) => value === 0),
true,
);
});
test('application repairs the durable admission-to-Artifact crash window', async () => {
@@ -253,7 +357,7 @@ test('application repairs the durable admission-to-Artifact crash window', async
assert.ok(testFixture.state().artifacts);
});
test('application coalesces exact callers and exposes Tool terminalization debt', async () => {
test('application coalesces exact callers and terminalizes Tool failure', async () => {
const concurrent = fixture({ blockTool: true });
const first = concurrent.service.execute(command());
const second = concurrent.service.execute(command());
@@ -266,6 +370,7 @@ test('application coalesces exact callers and exposes Tool terminalization debt'
const result = await failed.service.execute(command());
assert.equal(result.tool.outcome, 'failed');
assert.equal(result.model, null);
assert.equal(result.terminalizationRequired, true);
assert.equal(result.terminalization.reason, 'tool_failed');
assert.equal(result.terminalizationRequired, false);
assert.equal(failed.state().modelCalls, 0);
});
@@ -0,0 +1,442 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
BUILTIN_RUN_LOG_EXCERPT_TOOL,
BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION,
createBuiltInRunLogExcerptToolHandlerBinding,
} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool');
const {
createPluginPackageResourceGenerationFromReferences,
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
const {
createProjectToolDefinitionSnapshot,
projectToolDefinitionRegistry,
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
const {
prepareToolInvocation,
} = require('@qinglong/runtime-core/tool-registry');
const {
TrustedToolHandlerBindingRegistry,
createTrustedToolInvocationPlan,
} = require('@qinglong/runtime-core/trusted-tool-invocation');
const {
transitionStepRunMutation,
} = require('@qinglong/runtime-core/step-run');
const {
createCopilotFailureDiagnosisAdmissionBundle,
prepareCopilotFailureDiagnosisExecution,
} = require('@qinglong/ai/failure-diagnosis-execution-admission');
const {
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError,
normalizeCopilotFailureDiagnosisPreModelTerminalizationCommand,
terminalizeCopilotFailureDiagnosisBeforeModel,
} = require('@qinglong/ai/failure-diagnosis-pre-model-terminalization');
const NOW = 4_000;
async function plan() {
const generation = createPluginPackageResourceGenerationFromReferences({
installationId: 'install-terminalization-test',
projectId: 'project-terminalization',
packageName: 'qinglong',
lockDigest: 'a'.repeat(64),
generation: 1,
previousActiveLockDigest: null,
contentDigest: 'b'.repeat(64),
resources: [],
});
const snapshot = createProjectToolDefinitionSnapshot({
projectId: 'project-terminalization',
contributions: [
{
generation,
revisionDigest: 'c'.repeat(64),
definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION],
},
],
});
const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [
'cluster-control',
]);
const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]);
const principal = {
subject: { type: 'user', id: 'owner-terminalization' },
authenticationId: 'auth-terminalization',
authenticatedAtMs: 1_000,
expiresAtMs: 10_000,
assurance: 'multi_factor',
};
const invocation = await prepareToolInvocation(
projectToolDefinitionRegistry(snapshot),
{
projectId: 'project-terminalization',
principal,
nowMs: 1_200,
tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
input: { runId: 'source-run-failed', attemptId: 'source-attempt-failed' },
},
{
async authorize() {
return {
effect: 'allow',
reasons: ['role_grant'],
fence: { projectVersion: 1, bindingVersion: 1 },
};
},
},
);
const tool = createTrustedToolInvocationPlan(bindings, invocation, {
actionRef: 'terminalization-log-tool',
inputArtifactId: 'terminalization-input',
previewArtifactId: 'terminalization-preview',
artifactKeyId: 'terminalization-key',
artifactKey: Buffer.alloc(32, 0x11),
artifactNonce: Buffer.alloc(12, 0x22),
profile: 'cluster-control',
preview: {
title: 'Read failed Run log',
summary: 'Read bounded evidence',
fields: [
{ kind: 'identifier', label: 'Run', value: 'source-run-failed' },
{
kind: 'identifier',
label: 'Attempt',
value: 'source-attempt-failed',
},
],
warnings: ['potentially_sensitive_output'],
},
sealedAtMs: 2_000,
});
return prepareCopilotFailureDiagnosisExecution({
requestId: 'terminalization-request-1',
traceId: 'terminalization-trace-1',
source: {
runId: 'source-run-failed',
runVersion: 4,
runStatus: 'failed',
attemptId: 'source-attempt-failed',
attemptStatus: 'failed',
attemptFinishedAtMs: 1_900,
logArtifactId: 'source-log-artifact',
},
toolPlan: tool.plan,
bindings,
model: {
provider: 'provider-primary',
model: 'model-diagnosis',
modelBoundary: 'external',
responseLanguage: 'zh-CN',
maxOutputTokens: 256,
egressPolicy: {
schema: 'qinglong/copilot-model-egress-policy@v1',
revision: 'terminalization-policy-v1',
potentiallySensitiveDataBoundaries: ['external'],
maxInputBytes: 64 * 1024,
maxOutputTokens: 512,
},
},
plannedAtMs: 2_100,
deadlineAtMs: 8_000,
});
}
function transition(step, runVersion, to, atMs, suffix, outputRef) {
return transitionStepRunMutation(
step,
{
expectedVersion: step.version,
expectedDigest: step.stepRunDigest,
mutationId: `fixture-mutation-${suffix}`,
to,
atMs,
...(outputRef ? { outputRef } : {}),
...(['failed', 'timed_out', 'cancelled'].includes(to)
? { resultCode: `fixture_${to}` }
: {}),
...(['failed', 'timed_out'].includes(to)
? { errorSummary: `Fixture ${to}` }
: {}),
},
{
expectedRunVersion: runVersion,
expectedRunEventSequence: runVersion,
eventId: `fixture-event-${suffix}`,
dedupeKey: `fixture-event-${suffix}`,
actor: { type: 'system' },
},
);
}
function repository(authority) {
let command = null;
return {
value: {
async findByRequestId() {
return null;
},
async readAuthority() {
return authority;
},
async commit(value) {
command =
normalizeCopilotFailureDiagnosisPreModelTerminalizationCommand(value);
return { status: 'created', receipt: command.receipt };
},
},
command: () => command,
};
}
test('terminalizes a durable Tool failure and cancels the pending Model Step', async () => {
const current = await plan();
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
const started = transition(
admitted.toolStepMutation.stepRun,
3,
'running',
2_500,
'tool-start',
);
const failed = transition(started.stepRun, 4, 'failed', 3_000, 'tool-failed');
const storage = repository({
plan: current,
run: {
id: current.runId,
projectId: current.projectId,
status: 'running',
version: 5,
eventSequence: 5,
},
toolStep: failed.stepRun,
modelStep: admitted.modelStepMutation.stepRun,
modelStartExists: false,
observedAtMs: NOW,
});
const result = await terminalizeCopilotFailureDiagnosisBeforeModel(
current.requestId,
{
kind: 'tool_failure',
completion: {
startId: 'tool-start-1',
runId: current.runId,
stepRunId: current.toolStepRunId,
outcome: 'failed',
completedStepRunDigest: failed.stepRun.stepRunDigest,
completionDigest: 'd'.repeat(64),
},
},
{ repository: storage.value },
);
assert.equal(result.receipt.reason, 'tool_failed');
assert.equal(result.receipt.outcome, 'failed');
assert.equal(result.receipt.terminalSteps.length, 1);
assert.equal(
result.receipt.terminalSteps[0].stepRunId,
current.modelStepRunId,
);
assert.equal(result.receipt.terminalSteps[0].status, 'cancelled');
assert.equal(storage.command().expectedRunVersion, 5);
assert.equal(result.receipt.finalRunVersion, 7);
});
test('terminalizes every non-available log status without exposing content', async () => {
for (const status of ['not_found', 'pending', 'missing', 'retired']) {
const current = await plan();
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
const toolStarted = transition(
admitted.toolStepMutation.stepRun,
3,
'running',
2_500,
`start-${status}`,
);
const toolSucceeded = transition(
toolStarted.stepRun,
4,
'succeeded',
3_000,
`success-${status}`,
'tool-result-artifact',
);
const modelReady = transition(
admitted.modelStepMutation.stepRun,
5,
'ready',
3_100,
`ready-${status}`,
);
const storage = repository({
plan: current,
run: {
id: current.runId,
projectId: current.projectId,
status: 'running',
version: 6,
eventSequence: 6,
},
toolStep: toolSucceeded.stepRun,
modelStep: modelReady.stepRun,
modelStartExists: false,
observedAtMs: NOW,
});
const result = await terminalizeCopilotFailureDiagnosisBeforeModel(
current.requestId,
{
kind: 'tool_projection',
completion: {
startId: 'tool-start-1',
runId: current.runId,
stepRunId: current.toolStepRunId,
completedStepRunDigest: toolSucceeded.stepRun.stepRunDigest,
completionDigest: 'e'.repeat(64),
},
output: {
status,
runId: current.source.runId,
attemptId: current.source.attemptId,
profile: 'cluster-control',
},
},
{ repository: storage.value },
);
assert.equal(result.receipt.reason, `log_${status}`);
assert.equal(result.receipt.outcome, 'failed');
assert.equal(result.receipt.terminalSteps[0].status, 'failed');
assert.equal(JSON.stringify(result.receipt).includes('log content'), false);
}
});
test('database-observed deadline terminalizes both unstarted Steps', async () => {
const current = await plan();
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
const storage = repository({
plan: current,
run: {
id: current.runId,
projectId: current.projectId,
status: 'running',
version: 3,
eventSequence: 3,
},
toolStep: admitted.toolStepMutation.stepRun,
modelStep: admitted.modelStepMutation.stepRun,
modelStartExists: false,
observedAtMs: current.deadlineAtMs,
});
const result = await terminalizeCopilotFailureDiagnosisBeforeModel(
current.requestId,
{ kind: 'boundary' },
{ repository: storage.value },
);
assert.equal(result.receipt.reason, 'deadline_exceeded');
assert.equal(result.receipt.outcome, 'timed_out');
assert.deepEqual(
result.receipt.terminalSteps.map(({ status }) => status),
['timed_out', 'cancelled'],
);
assert.equal(result.receipt.finalRunVersion, 6);
});
test('insufficient durable Tool budget terminalizes before starting the Tool', async () => {
const current = await plan();
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
const storage = repository({
plan: current,
run: {
id: current.runId,
projectId: current.projectId,
status: 'running',
version: 3,
eventSequence: 3,
},
toolStep: admitted.toolStepMutation.stepRun,
modelStep: admitted.modelStepMutation.stepRun,
modelStartExists: false,
observedAtMs: current.deadlineAtMs - 4_999,
});
const result = await terminalizeCopilotFailureDiagnosisBeforeModel(
current.requestId,
{ kind: 'boundary' },
{ repository: storage.value },
);
assert.equal(result.receipt.reason, 'tool_budget_exhausted');
assert.equal(result.receipt.outcome, 'timed_out');
assert.deepEqual(
result.receipt.terminalSteps.map(({ status }) => status),
['timed_out', 'cancelled'],
);
});
test('durable cancellation terminalizes both unstarted Steps', async () => {
const current = await plan();
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
const storage = repository({
plan: current,
run: {
id: current.runId,
projectId: current.projectId,
status: 'running',
version: 3,
eventSequence: 3,
cancelRequestedAtMs: NOW - 1,
cancelReason: 'user',
},
toolStep: admitted.toolStepMutation.stepRun,
modelStep: admitted.modelStepMutation.stepRun,
modelStartExists: false,
observedAtMs: NOW,
});
const result = await terminalizeCopilotFailureDiagnosisBeforeModel(
current.requestId,
{ kind: 'boundary' },
{ repository: storage.value },
);
assert.equal(result.receipt.reason, 'cancellation_requested');
assert.equal(result.receipt.outcome, 'cancelled');
assert.deepEqual(
result.receipt.terminalSteps.map(({ status }) => status),
['cancelled', 'cancelled'],
);
});
test('refuses early boundary settlement and any pre-Model lie after Model start', async () => {
const current = await plan();
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
const base = {
plan: current,
run: {
id: current.runId,
projectId: current.projectId,
status: 'running',
version: 3,
eventSequence: 3,
},
toolStep: admitted.toolStepMutation.stepRun,
modelStep: admitted.modelStepMutation.stepRun,
observedAtMs: current.deadlineAtMs - 5_001,
};
await assert.rejects(
terminalizeCopilotFailureDiagnosisBeforeModel(
current.requestId,
{ kind: 'boundary' },
{ repository: repository({ ...base, modelStartExists: false }).value },
),
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError,
);
await assert.rejects(
terminalizeCopilotFailureDiagnosisBeforeModel(
current.requestId,
{ kind: 'boundary' },
{
repository: repository({
...base,
modelStartExists: true,
observedAtMs: current.deadlineAtMs,
}).value,
},
),
/pre-Model authority is invalid/,
);
});
@@ -22,6 +22,7 @@ const {
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID,
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID,
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID,
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID,
POSTGRES_MODEL_INVOCATION_MIGRATION_ID,
POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID,
@@ -436,6 +437,10 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async ()
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID,
'pg-9020-ai-copilot-failure-diagnosis-model-executions',
);
assert.equal(
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID,
'pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations',
);
assert.equal(
POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
'ai_schema_migrations',
@@ -523,7 +528,7 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async ()
);
assert.equal(
postgresModelInvocationMigrationDefinition.migrations.length,
20,
21,
);
const diagnosisAdmissionStatements = [];
@@ -597,6 +602,29 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async ()
/GRANT[^;]*(?:UPDATE|DELETE)/,
);
const diagnosisPreModelTerminalizationStatements = [];
await postgresModelInvocationMigrationDefinition.migrations[20].up({
async query(statement) {
diagnosisPreModelTerminalizationStatements.push(statement);
return { rows: [] };
},
});
const diagnosisPreModelTerminalizationSql =
diagnosisPreModelTerminalizationStatements.join('\n');
assert.match(
diagnosisPreModelTerminalizationSql,
/CREATE TABLE "ql3_ai"\."copilot_failure_diagnosis_pre_model_terminalizations"/,
);
assert.match(
diagnosisPreModelTerminalizationSql,
/jsonb_array_length\(terminal_steps_json\) BETWEEN 1 AND 2/,
);
assert.match(diagnosisPreModelTerminalizationSql, /TO ql3_runtime/);
assert.doesNotMatch(
diagnosisPreModelTerminalizationSql,
/GRANT[^;]*(?:UPDATE|DELETE)/,
);
const retirementStatements = [];
await postgresModelInvocationMigrationDefinition.migrations[10].up({
async query(statement) {
@@ -175,7 +175,7 @@ test('readiness binds exact migration history and least-privilege primary author
assert.equal(report.ready, true);
assert.equal(
report.migrationIds.at(-1),
'pg-9020-ai-copilot-failure-diagnosis-model-executions',
'pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations',
);
assert.match(
queries[1],
@@ -277,7 +277,7 @@ test('tester readiness freezes migration history and least privilege', async ()
assert.equal(ready.ready, true);
assert.equal(
ready.migrationIds.at(-1),
'pg-9020-ai-copilot-failure-diagnosis-model-executions',
'pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations',
);
await assert.rejects(