mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 03:18:09 +08:00
feat(ql3): terminalize copilot diagnosis failures
This commit is contained in:
@@ -5,6 +5,9 @@ const {
|
||||
CopilotFailureDiagnosisApplicationService,
|
||||
CopilotFailureDiagnosisApplicationUnavailableError,
|
||||
} = require('@qinglong/ai/failure-diagnosis-application');
|
||||
const {
|
||||
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError,
|
||||
} = require('@qinglong/ai/failure-diagnosis-pre-model-terminalization');
|
||||
const {
|
||||
BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION,
|
||||
} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool');
|
||||
@@ -80,9 +83,12 @@ function fixture(options = {}) {
|
||||
let resolvedKeyCopies = [];
|
||||
let toolCalls = 0;
|
||||
let modelCalls = 0;
|
||||
let terminalizationCalls = 0;
|
||||
let releaseTool;
|
||||
const toolGate = options.blockTool
|
||||
? new Promise((resolve) => { releaseTool = resolve; })
|
||||
? new Promise((resolve) => {
|
||||
releaseTool = resolve;
|
||||
})
|
||||
: Promise.resolve();
|
||||
const admissions = {
|
||||
async findByRequestId(requestId) {
|
||||
@@ -108,20 +114,30 @@ function fixture(options = {}) {
|
||||
},
|
||||
};
|
||||
const snapshots = {
|
||||
async findCurrent() { return { snapshot: snapshot(), committedAtMs: NOW }; },
|
||||
async findCurrent() {
|
||||
return { snapshot: snapshot(), committedAtMs: NOW };
|
||||
},
|
||||
};
|
||||
const runs = {
|
||||
async findRunById() {
|
||||
return {
|
||||
id: 'source-run-1', projectId: 'project-1', status: 'failed',
|
||||
version: 8, eventSequence: 8,
|
||||
id: 'source-run-1',
|
||||
projectId: 'project-1',
|
||||
status: 'failed',
|
||||
version: 8,
|
||||
eventSequence: 8,
|
||||
};
|
||||
},
|
||||
async findLatestAttemptByRunId() {
|
||||
return {
|
||||
id: 'source-attempt-1', runId: 'source-run-1', attempt: 1,
|
||||
status: 'failed', executorType: 'remote_worker', callbackSequence: 0,
|
||||
createdAtMs: NOW - 3000, finishedAtMs: NOW - 1000,
|
||||
id: 'source-attempt-1',
|
||||
runId: 'source-run-1',
|
||||
attempt: 1,
|
||||
status: 'failed',
|
||||
executorType: 'remote_worker',
|
||||
callbackSequence: 0,
|
||||
createdAtMs: NOW - 3000,
|
||||
finishedAtMs: NOW - 1000,
|
||||
logArtifactId: `wlog-${'d'.repeat(30)}`,
|
||||
};
|
||||
},
|
||||
@@ -140,8 +156,12 @@ function fixture(options = {}) {
|
||||
artifacts = { input, preview };
|
||||
return { status: 'inserted' };
|
||||
},
|
||||
async findInput() { return artifacts?.input ?? null; },
|
||||
async findPreview() { return artifacts?.preview ?? null; },
|
||||
async findInput() {
|
||||
return artifacts?.input ?? null;
|
||||
},
|
||||
async findPreview() {
|
||||
return artifacts?.preview ?? null;
|
||||
},
|
||||
};
|
||||
const invocationKeys = {
|
||||
async active() {
|
||||
@@ -156,23 +176,58 @@ function fixture(options = {}) {
|
||||
return { keyId, key: copy };
|
||||
},
|
||||
};
|
||||
const unlocks = { async findByRequestId() { return null; }, async commit() {} };
|
||||
const unlocks = {
|
||||
async findByRequestId() {
|
||||
return null;
|
||||
},
|
||||
async commit() {},
|
||||
};
|
||||
const tool = {
|
||||
admissions,
|
||||
snapshots,
|
||||
runs,
|
||||
artifacts: artifactRepository,
|
||||
invocationKeys,
|
||||
resultKeys: { async resolve() { return null; } },
|
||||
stepRuns: { async findById() { return null; } },
|
||||
barriers: {}, completions: {}, failureCompletions: {},
|
||||
resultKeyCatalog: {}, resultRekeys: {}, logs: {}, unlocks,
|
||||
resultKeys: {
|
||||
async resolve() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
stepRuns: {
|
||||
async findById() {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
barriers: {},
|
||||
completions: {},
|
||||
failureCompletions: {},
|
||||
resultKeyCatalog: {},
|
||||
resultRekeys: {},
|
||||
logs: {},
|
||||
unlocks,
|
||||
};
|
||||
const model = {
|
||||
admissions,
|
||||
unlocks,
|
||||
toolResults: {}, modelInvocations: {}, outputs: {}, gateway: {},
|
||||
successfulCompletion: {}, finalizations: {},
|
||||
toolResults: {},
|
||||
modelInvocations: {},
|
||||
outputs: {},
|
||||
gateway: {},
|
||||
successfulCompletion: {},
|
||||
finalizations: {},
|
||||
};
|
||||
const terminalizations = {
|
||||
repository: {
|
||||
async findByRequestId() {
|
||||
return null;
|
||||
},
|
||||
async readAuthority() {
|
||||
return {};
|
||||
},
|
||||
async commit() {
|
||||
return {};
|
||||
},
|
||||
},
|
||||
};
|
||||
const service = new CopilotFailureDiagnosisApplicationService({
|
||||
admissions,
|
||||
@@ -183,13 +238,15 @@ function fixture(options = {}) {
|
||||
authorizer: {
|
||||
async authorize() {
|
||||
return {
|
||||
effect: 'allow', reasons: ['role_grant'],
|
||||
effect: 'allow',
|
||||
reasons: ['role_grant'],
|
||||
fence: { projectVersion: 1, bindingVersion: 1 },
|
||||
};
|
||||
},
|
||||
},
|
||||
tool,
|
||||
model,
|
||||
terminalizations,
|
||||
modelIntent: MODEL,
|
||||
executionTimeoutMs: 60_000,
|
||||
now: () => NOW,
|
||||
@@ -197,10 +254,24 @@ function fixture(options = {}) {
|
||||
toolCalls += 1;
|
||||
await toolGate;
|
||||
return options.toolFailure
|
||||
? { outcome: 'failed', completionStatus: 'created', unlockStatus: null }
|
||||
? {
|
||||
outcome: 'failed',
|
||||
completionStatus: 'created',
|
||||
unlockStatus: null,
|
||||
completion: { completionDigest: 'f'.repeat(64) },
|
||||
}
|
||||
: {
|
||||
outcome: 'succeeded', completionStatus: 'created',
|
||||
unlockStatus: 'created', completion: {}, unlock: {},
|
||||
outcome: 'succeeded',
|
||||
completionStatus: 'created',
|
||||
unlockStatus: 'created',
|
||||
completion: { completionDigest: 'e'.repeat(64) },
|
||||
output: {
|
||||
status: 'available',
|
||||
runId: 'source-run-1',
|
||||
attemptId: 'source-attempt-1',
|
||||
profile: 'cluster-control',
|
||||
},
|
||||
unlock: {},
|
||||
};
|
||||
},
|
||||
async executeModel() {
|
||||
@@ -211,13 +282,36 @@ function fixture(options = {}) {
|
||||
finalization: { requestId: 'diagnosis-request-1' },
|
||||
};
|
||||
},
|
||||
async terminalizeBeforeModel(_requestId, trigger) {
|
||||
terminalizationCalls += 1;
|
||||
if (
|
||||
trigger.kind === 'boundary' ||
|
||||
(trigger.kind === 'tool_projection' &&
|
||||
trigger.output.status === 'available')
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisPreModelTerminalizationNotReadyError();
|
||||
}
|
||||
return {
|
||||
status: 'created',
|
||||
receipt: {
|
||||
requestId: 'diagnosis-request-1',
|
||||
reason: 'tool_failed',
|
||||
outcome: 'failed',
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
return {
|
||||
service,
|
||||
releaseTool: () => releaseTool?.(),
|
||||
state: () => ({
|
||||
plan, artifacts, toolCalls, modelCalls,
|
||||
activeKeyCopies, resolvedKeyCopies,
|
||||
plan,
|
||||
artifacts,
|
||||
toolCalls,
|
||||
modelCalls,
|
||||
terminalizationCalls,
|
||||
activeKeyCopies,
|
||||
resolvedKeyCopies,
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -228,6 +322,7 @@ test('application derives, admits and executes one server-owned diagnosis with e
|
||||
assert.equal(first.admissionStatus, 'created');
|
||||
assert.equal(first.tool.outcome, 'succeeded');
|
||||
assert.equal(first.model.outcome, 'succeeded');
|
||||
assert.equal(first.terminalization, null);
|
||||
assert.equal(first.terminalizationRequired, false);
|
||||
const second = await testFixture.service.execute(command());
|
||||
assert.equal(second.admissionStatus, 'existing');
|
||||
@@ -235,9 +330,18 @@ test('application derives, admits and executes one server-owned diagnosis with e
|
||||
assert.equal(state.toolCalls, 2);
|
||||
assert.equal(state.modelCalls, 2);
|
||||
assert.equal(state.plan.source.attemptId, 'source-attempt-1');
|
||||
assert.equal(state.plan.tool.invocationArtifact.artifactId.startsWith('cdia:'), true);
|
||||
assert.equal(state.activeKeyCopies[0].every((value) => value === 0), true);
|
||||
assert.equal(state.resolvedKeyCopies[0].every((value) => value === 0), true);
|
||||
assert.equal(
|
||||
state.plan.tool.invocationArtifact.artifactId.startsWith('cdia:'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
state.activeKeyCopies[0].every((value) => value === 0),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
state.resolvedKeyCopies[0].every((value) => value === 0),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('application repairs the durable admission-to-Artifact crash window', async () => {
|
||||
@@ -253,7 +357,7 @@ test('application repairs the durable admission-to-Artifact crash window', async
|
||||
assert.ok(testFixture.state().artifacts);
|
||||
});
|
||||
|
||||
test('application coalesces exact callers and exposes Tool terminalization debt', async () => {
|
||||
test('application coalesces exact callers and terminalizes Tool failure', async () => {
|
||||
const concurrent = fixture({ blockTool: true });
|
||||
const first = concurrent.service.execute(command());
|
||||
const second = concurrent.service.execute(command());
|
||||
@@ -266,6 +370,7 @@ test('application coalesces exact callers and exposes Tool terminalization debt'
|
||||
const result = await failed.service.execute(command());
|
||||
assert.equal(result.tool.outcome, 'failed');
|
||||
assert.equal(result.model, null);
|
||||
assert.equal(result.terminalizationRequired, true);
|
||||
assert.equal(result.terminalization.reason, 'tool_failed');
|
||||
assert.equal(result.terminalizationRequired, false);
|
||||
assert.equal(failed.state().modelCalls, 0);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,442 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||
BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION,
|
||||
createBuiltInRunLogExcerptToolHandlerBinding,
|
||||
} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool');
|
||||
const {
|
||||
createPluginPackageResourceGenerationFromReferences,
|
||||
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
|
||||
const {
|
||||
createProjectToolDefinitionSnapshot,
|
||||
projectToolDefinitionRegistry,
|
||||
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
|
||||
const {
|
||||
prepareToolInvocation,
|
||||
} = require('@qinglong/runtime-core/tool-registry');
|
||||
const {
|
||||
TrustedToolHandlerBindingRegistry,
|
||||
createTrustedToolInvocationPlan,
|
||||
} = require('@qinglong/runtime-core/trusted-tool-invocation');
|
||||
const {
|
||||
transitionStepRunMutation,
|
||||
} = require('@qinglong/runtime-core/step-run');
|
||||
const {
|
||||
createCopilotFailureDiagnosisAdmissionBundle,
|
||||
prepareCopilotFailureDiagnosisExecution,
|
||||
} = require('@qinglong/ai/failure-diagnosis-execution-admission');
|
||||
const {
|
||||
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError,
|
||||
normalizeCopilotFailureDiagnosisPreModelTerminalizationCommand,
|
||||
terminalizeCopilotFailureDiagnosisBeforeModel,
|
||||
} = require('@qinglong/ai/failure-diagnosis-pre-model-terminalization');
|
||||
|
||||
const NOW = 4_000;
|
||||
|
||||
async function plan() {
|
||||
const generation = createPluginPackageResourceGenerationFromReferences({
|
||||
installationId: 'install-terminalization-test',
|
||||
projectId: 'project-terminalization',
|
||||
packageName: 'qinglong',
|
||||
lockDigest: 'a'.repeat(64),
|
||||
generation: 1,
|
||||
previousActiveLockDigest: null,
|
||||
contentDigest: 'b'.repeat(64),
|
||||
resources: [],
|
||||
});
|
||||
const snapshot = createProjectToolDefinitionSnapshot({
|
||||
projectId: 'project-terminalization',
|
||||
contributions: [
|
||||
{
|
||||
generation,
|
||||
revisionDigest: 'c'.repeat(64),
|
||||
definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION],
|
||||
},
|
||||
],
|
||||
});
|
||||
const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [
|
||||
'cluster-control',
|
||||
]);
|
||||
const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]);
|
||||
const principal = {
|
||||
subject: { type: 'user', id: 'owner-terminalization' },
|
||||
authenticationId: 'auth-terminalization',
|
||||
authenticatedAtMs: 1_000,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'multi_factor',
|
||||
};
|
||||
const invocation = await prepareToolInvocation(
|
||||
projectToolDefinitionRegistry(snapshot),
|
||||
{
|
||||
projectId: 'project-terminalization',
|
||||
principal,
|
||||
nowMs: 1_200,
|
||||
tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||
input: { runId: 'source-run-failed', attemptId: 'source-attempt-failed' },
|
||||
},
|
||||
{
|
||||
async authorize() {
|
||||
return {
|
||||
effect: 'allow',
|
||||
reasons: ['role_grant'],
|
||||
fence: { projectVersion: 1, bindingVersion: 1 },
|
||||
};
|
||||
},
|
||||
},
|
||||
);
|
||||
const tool = createTrustedToolInvocationPlan(bindings, invocation, {
|
||||
actionRef: 'terminalization-log-tool',
|
||||
inputArtifactId: 'terminalization-input',
|
||||
previewArtifactId: 'terminalization-preview',
|
||||
artifactKeyId: 'terminalization-key',
|
||||
artifactKey: Buffer.alloc(32, 0x11),
|
||||
artifactNonce: Buffer.alloc(12, 0x22),
|
||||
profile: 'cluster-control',
|
||||
preview: {
|
||||
title: 'Read failed Run log',
|
||||
summary: 'Read bounded evidence',
|
||||
fields: [
|
||||
{ kind: 'identifier', label: 'Run', value: 'source-run-failed' },
|
||||
{
|
||||
kind: 'identifier',
|
||||
label: 'Attempt',
|
||||
value: 'source-attempt-failed',
|
||||
},
|
||||
],
|
||||
warnings: ['potentially_sensitive_output'],
|
||||
},
|
||||
sealedAtMs: 2_000,
|
||||
});
|
||||
return prepareCopilotFailureDiagnosisExecution({
|
||||
requestId: 'terminalization-request-1',
|
||||
traceId: 'terminalization-trace-1',
|
||||
source: {
|
||||
runId: 'source-run-failed',
|
||||
runVersion: 4,
|
||||
runStatus: 'failed',
|
||||
attemptId: 'source-attempt-failed',
|
||||
attemptStatus: 'failed',
|
||||
attemptFinishedAtMs: 1_900,
|
||||
logArtifactId: 'source-log-artifact',
|
||||
},
|
||||
toolPlan: tool.plan,
|
||||
bindings,
|
||||
model: {
|
||||
provider: 'provider-primary',
|
||||
model: 'model-diagnosis',
|
||||
modelBoundary: 'external',
|
||||
responseLanguage: 'zh-CN',
|
||||
maxOutputTokens: 256,
|
||||
egressPolicy: {
|
||||
schema: 'qinglong/copilot-model-egress-policy@v1',
|
||||
revision: 'terminalization-policy-v1',
|
||||
potentiallySensitiveDataBoundaries: ['external'],
|
||||
maxInputBytes: 64 * 1024,
|
||||
maxOutputTokens: 512,
|
||||
},
|
||||
},
|
||||
plannedAtMs: 2_100,
|
||||
deadlineAtMs: 8_000,
|
||||
});
|
||||
}
|
||||
|
||||
function transition(step, runVersion, to, atMs, suffix, outputRef) {
|
||||
return transitionStepRunMutation(
|
||||
step,
|
||||
{
|
||||
expectedVersion: step.version,
|
||||
expectedDigest: step.stepRunDigest,
|
||||
mutationId: `fixture-mutation-${suffix}`,
|
||||
to,
|
||||
atMs,
|
||||
...(outputRef ? { outputRef } : {}),
|
||||
...(['failed', 'timed_out', 'cancelled'].includes(to)
|
||||
? { resultCode: `fixture_${to}` }
|
||||
: {}),
|
||||
...(['failed', 'timed_out'].includes(to)
|
||||
? { errorSummary: `Fixture ${to}` }
|
||||
: {}),
|
||||
},
|
||||
{
|
||||
expectedRunVersion: runVersion,
|
||||
expectedRunEventSequence: runVersion,
|
||||
eventId: `fixture-event-${suffix}`,
|
||||
dedupeKey: `fixture-event-${suffix}`,
|
||||
actor: { type: 'system' },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function repository(authority) {
|
||||
let command = null;
|
||||
return {
|
||||
value: {
|
||||
async findByRequestId() {
|
||||
return null;
|
||||
},
|
||||
async readAuthority() {
|
||||
return authority;
|
||||
},
|
||||
async commit(value) {
|
||||
command =
|
||||
normalizeCopilotFailureDiagnosisPreModelTerminalizationCommand(value);
|
||||
return { status: 'created', receipt: command.receipt };
|
||||
},
|
||||
},
|
||||
command: () => command,
|
||||
};
|
||||
}
|
||||
|
||||
test('terminalizes a durable Tool failure and cancels the pending Model Step', async () => {
|
||||
const current = await plan();
|
||||
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
|
||||
const started = transition(
|
||||
admitted.toolStepMutation.stepRun,
|
||||
3,
|
||||
'running',
|
||||
2_500,
|
||||
'tool-start',
|
||||
);
|
||||
const failed = transition(started.stepRun, 4, 'failed', 3_000, 'tool-failed');
|
||||
const storage = repository({
|
||||
plan: current,
|
||||
run: {
|
||||
id: current.runId,
|
||||
projectId: current.projectId,
|
||||
status: 'running',
|
||||
version: 5,
|
||||
eventSequence: 5,
|
||||
},
|
||||
toolStep: failed.stepRun,
|
||||
modelStep: admitted.modelStepMutation.stepRun,
|
||||
modelStartExists: false,
|
||||
observedAtMs: NOW,
|
||||
});
|
||||
const result = await terminalizeCopilotFailureDiagnosisBeforeModel(
|
||||
current.requestId,
|
||||
{
|
||||
kind: 'tool_failure',
|
||||
completion: {
|
||||
startId: 'tool-start-1',
|
||||
runId: current.runId,
|
||||
stepRunId: current.toolStepRunId,
|
||||
outcome: 'failed',
|
||||
completedStepRunDigest: failed.stepRun.stepRunDigest,
|
||||
completionDigest: 'd'.repeat(64),
|
||||
},
|
||||
},
|
||||
{ repository: storage.value },
|
||||
);
|
||||
assert.equal(result.receipt.reason, 'tool_failed');
|
||||
assert.equal(result.receipt.outcome, 'failed');
|
||||
assert.equal(result.receipt.terminalSteps.length, 1);
|
||||
assert.equal(
|
||||
result.receipt.terminalSteps[0].stepRunId,
|
||||
current.modelStepRunId,
|
||||
);
|
||||
assert.equal(result.receipt.terminalSteps[0].status, 'cancelled');
|
||||
assert.equal(storage.command().expectedRunVersion, 5);
|
||||
assert.equal(result.receipt.finalRunVersion, 7);
|
||||
});
|
||||
|
||||
test('terminalizes every non-available log status without exposing content', async () => {
|
||||
for (const status of ['not_found', 'pending', 'missing', 'retired']) {
|
||||
const current = await plan();
|
||||
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
|
||||
const toolStarted = transition(
|
||||
admitted.toolStepMutation.stepRun,
|
||||
3,
|
||||
'running',
|
||||
2_500,
|
||||
`start-${status}`,
|
||||
);
|
||||
const toolSucceeded = transition(
|
||||
toolStarted.stepRun,
|
||||
4,
|
||||
'succeeded',
|
||||
3_000,
|
||||
`success-${status}`,
|
||||
'tool-result-artifact',
|
||||
);
|
||||
const modelReady = transition(
|
||||
admitted.modelStepMutation.stepRun,
|
||||
5,
|
||||
'ready',
|
||||
3_100,
|
||||
`ready-${status}`,
|
||||
);
|
||||
const storage = repository({
|
||||
plan: current,
|
||||
run: {
|
||||
id: current.runId,
|
||||
projectId: current.projectId,
|
||||
status: 'running',
|
||||
version: 6,
|
||||
eventSequence: 6,
|
||||
},
|
||||
toolStep: toolSucceeded.stepRun,
|
||||
modelStep: modelReady.stepRun,
|
||||
modelStartExists: false,
|
||||
observedAtMs: NOW,
|
||||
});
|
||||
const result = await terminalizeCopilotFailureDiagnosisBeforeModel(
|
||||
current.requestId,
|
||||
{
|
||||
kind: 'tool_projection',
|
||||
completion: {
|
||||
startId: 'tool-start-1',
|
||||
runId: current.runId,
|
||||
stepRunId: current.toolStepRunId,
|
||||
completedStepRunDigest: toolSucceeded.stepRun.stepRunDigest,
|
||||
completionDigest: 'e'.repeat(64),
|
||||
},
|
||||
output: {
|
||||
status,
|
||||
runId: current.source.runId,
|
||||
attemptId: current.source.attemptId,
|
||||
profile: 'cluster-control',
|
||||
},
|
||||
},
|
||||
{ repository: storage.value },
|
||||
);
|
||||
assert.equal(result.receipt.reason, `log_${status}`);
|
||||
assert.equal(result.receipt.outcome, 'failed');
|
||||
assert.equal(result.receipt.terminalSteps[0].status, 'failed');
|
||||
assert.equal(JSON.stringify(result.receipt).includes('log content'), false);
|
||||
}
|
||||
});
|
||||
|
||||
test('database-observed deadline terminalizes both unstarted Steps', async () => {
|
||||
const current = await plan();
|
||||
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
|
||||
const storage = repository({
|
||||
plan: current,
|
||||
run: {
|
||||
id: current.runId,
|
||||
projectId: current.projectId,
|
||||
status: 'running',
|
||||
version: 3,
|
||||
eventSequence: 3,
|
||||
},
|
||||
toolStep: admitted.toolStepMutation.stepRun,
|
||||
modelStep: admitted.modelStepMutation.stepRun,
|
||||
modelStartExists: false,
|
||||
observedAtMs: current.deadlineAtMs,
|
||||
});
|
||||
const result = await terminalizeCopilotFailureDiagnosisBeforeModel(
|
||||
current.requestId,
|
||||
{ kind: 'boundary' },
|
||||
{ repository: storage.value },
|
||||
);
|
||||
assert.equal(result.receipt.reason, 'deadline_exceeded');
|
||||
assert.equal(result.receipt.outcome, 'timed_out');
|
||||
assert.deepEqual(
|
||||
result.receipt.terminalSteps.map(({ status }) => status),
|
||||
['timed_out', 'cancelled'],
|
||||
);
|
||||
assert.equal(result.receipt.finalRunVersion, 6);
|
||||
});
|
||||
|
||||
test('insufficient durable Tool budget terminalizes before starting the Tool', async () => {
|
||||
const current = await plan();
|
||||
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
|
||||
const storage = repository({
|
||||
plan: current,
|
||||
run: {
|
||||
id: current.runId,
|
||||
projectId: current.projectId,
|
||||
status: 'running',
|
||||
version: 3,
|
||||
eventSequence: 3,
|
||||
},
|
||||
toolStep: admitted.toolStepMutation.stepRun,
|
||||
modelStep: admitted.modelStepMutation.stepRun,
|
||||
modelStartExists: false,
|
||||
observedAtMs: current.deadlineAtMs - 4_999,
|
||||
});
|
||||
const result = await terminalizeCopilotFailureDiagnosisBeforeModel(
|
||||
current.requestId,
|
||||
{ kind: 'boundary' },
|
||||
{ repository: storage.value },
|
||||
);
|
||||
assert.equal(result.receipt.reason, 'tool_budget_exhausted');
|
||||
assert.equal(result.receipt.outcome, 'timed_out');
|
||||
assert.deepEqual(
|
||||
result.receipt.terminalSteps.map(({ status }) => status),
|
||||
['timed_out', 'cancelled'],
|
||||
);
|
||||
});
|
||||
|
||||
test('durable cancellation terminalizes both unstarted Steps', async () => {
|
||||
const current = await plan();
|
||||
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
|
||||
const storage = repository({
|
||||
plan: current,
|
||||
run: {
|
||||
id: current.runId,
|
||||
projectId: current.projectId,
|
||||
status: 'running',
|
||||
version: 3,
|
||||
eventSequence: 3,
|
||||
cancelRequestedAtMs: NOW - 1,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
toolStep: admitted.toolStepMutation.stepRun,
|
||||
modelStep: admitted.modelStepMutation.stepRun,
|
||||
modelStartExists: false,
|
||||
observedAtMs: NOW,
|
||||
});
|
||||
const result = await terminalizeCopilotFailureDiagnosisBeforeModel(
|
||||
current.requestId,
|
||||
{ kind: 'boundary' },
|
||||
{ repository: storage.value },
|
||||
);
|
||||
assert.equal(result.receipt.reason, 'cancellation_requested');
|
||||
assert.equal(result.receipt.outcome, 'cancelled');
|
||||
assert.deepEqual(
|
||||
result.receipt.terminalSteps.map(({ status }) => status),
|
||||
['cancelled', 'cancelled'],
|
||||
);
|
||||
});
|
||||
|
||||
test('refuses early boundary settlement and any pre-Model lie after Model start', async () => {
|
||||
const current = await plan();
|
||||
const admitted = createCopilotFailureDiagnosisAdmissionBundle(current);
|
||||
const base = {
|
||||
plan: current,
|
||||
run: {
|
||||
id: current.runId,
|
||||
projectId: current.projectId,
|
||||
status: 'running',
|
||||
version: 3,
|
||||
eventSequence: 3,
|
||||
},
|
||||
toolStep: admitted.toolStepMutation.stepRun,
|
||||
modelStep: admitted.modelStepMutation.stepRun,
|
||||
observedAtMs: current.deadlineAtMs - 5_001,
|
||||
};
|
||||
await assert.rejects(
|
||||
terminalizeCopilotFailureDiagnosisBeforeModel(
|
||||
current.requestId,
|
||||
{ kind: 'boundary' },
|
||||
{ repository: repository({ ...base, modelStartExists: false }).value },
|
||||
),
|
||||
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError,
|
||||
);
|
||||
await assert.rejects(
|
||||
terminalizeCopilotFailureDiagnosisBeforeModel(
|
||||
current.requestId,
|
||||
{ kind: 'boundary' },
|
||||
{
|
||||
repository: repository({
|
||||
...base,
|
||||
modelStartExists: true,
|
||||
observedAtMs: current.deadlineAtMs,
|
||||
}).value,
|
||||
},
|
||||
),
|
||||
/pre-Model authority is invalid/,
|
||||
);
|
||||
});
|
||||
@@ -22,6 +22,7 @@ const {
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID,
|
||||
@@ -436,6 +437,10 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async ()
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID,
|
||||
'pg-9020-ai-copilot-failure-diagnosis-model-executions',
|
||||
);
|
||||
assert.equal(
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID,
|
||||
'pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations',
|
||||
);
|
||||
assert.equal(
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
|
||||
'ai_schema_migrations',
|
||||
@@ -523,7 +528,7 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async ()
|
||||
);
|
||||
assert.equal(
|
||||
postgresModelInvocationMigrationDefinition.migrations.length,
|
||||
20,
|
||||
21,
|
||||
);
|
||||
|
||||
const diagnosisAdmissionStatements = [];
|
||||
@@ -597,6 +602,29 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async ()
|
||||
/GRANT[^;]*(?:UPDATE|DELETE)/,
|
||||
);
|
||||
|
||||
const diagnosisPreModelTerminalizationStatements = [];
|
||||
await postgresModelInvocationMigrationDefinition.migrations[20].up({
|
||||
async query(statement) {
|
||||
diagnosisPreModelTerminalizationStatements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const diagnosisPreModelTerminalizationSql =
|
||||
diagnosisPreModelTerminalizationStatements.join('\n');
|
||||
assert.match(
|
||||
diagnosisPreModelTerminalizationSql,
|
||||
/CREATE TABLE "ql3_ai"\."copilot_failure_diagnosis_pre_model_terminalizations"/,
|
||||
);
|
||||
assert.match(
|
||||
diagnosisPreModelTerminalizationSql,
|
||||
/jsonb_array_length\(terminal_steps_json\) BETWEEN 1 AND 2/,
|
||||
);
|
||||
assert.match(diagnosisPreModelTerminalizationSql, /TO ql3_runtime/);
|
||||
assert.doesNotMatch(
|
||||
diagnosisPreModelTerminalizationSql,
|
||||
/GRANT[^;]*(?:UPDATE|DELETE)/,
|
||||
);
|
||||
|
||||
const retirementStatements = [];
|
||||
await postgresModelInvocationMigrationDefinition.migrations[10].up({
|
||||
async query(statement) {
|
||||
|
||||
+1
-1
@@ -175,7 +175,7 @@ test('readiness binds exact migration history and least-privilege primary author
|
||||
assert.equal(report.ready, true);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-9020-ai-copilot-failure-diagnosis-model-executions',
|
||||
'pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations',
|
||||
);
|
||||
assert.match(
|
||||
queries[1],
|
||||
|
||||
@@ -277,7 +277,7 @@ test('tester readiness freezes migration history and least privilege', async ()
|
||||
assert.equal(ready.ready, true);
|
||||
assert.equal(
|
||||
ready.migrationIds.at(-1),
|
||||
'pg-9020-ai-copilot-failure-diagnosis-model-executions',
|
||||
'pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations',
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
|
||||
Reference in New Issue
Block a user