mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): reconcile local lost run retries
This commit is contained in:
whyour
@@ -250,6 +250,11 @@
|
||||
"require": "./dist/run/runAttemptLogRetentionRepository.js",
|
||||
"default": "./dist/run/runAttemptLogRetentionRepository.js"
|
||||
},
|
||||
"./run-lost-retry": {
|
||||
"types": "./dist/run/runLostRetryRepository.d.ts",
|
||||
"require": "./dist/run/runLostRetryRepository.js",
|
||||
"default": "./dist/run/runLostRetryRepository.js"
|
||||
},
|
||||
"./trigger-administration": {
|
||||
"types": "./dist/scheduling/triggerAdministration.d.ts",
|
||||
"require": "./dist/scheduling/triggerAdministration.js",
|
||||
|
||||
@@ -47,6 +47,7 @@ export type LocalProfileStorageBootstrapResult =
|
||||
readonly executionControl: LocalSqliteRuntimeDatabase['executionControl'];
|
||||
readonly completionReceipts: LocalSqliteRuntimeDatabase['completionReceipts'];
|
||||
readonly runAttemptLogRetention: LocalSqliteRuntimeDatabase['runAttemptLogRetention'];
|
||||
readonly runLostRetry: LocalSqliteRuntimeDatabase['runLostRetry'];
|
||||
readonly localSecrets: LocalSqliteRuntimeDatabase['localSecrets'];
|
||||
readonly localSecretAdministration: LocalSqliteRuntimeDatabase['localSecretAdministration'];
|
||||
readonly projectPolicy: LocalSqliteRuntimeDatabase['projectPolicy'];
|
||||
@@ -140,6 +141,7 @@ export async function bootstrapLocalProfileStorage(
|
||||
executionControl: database.executionControl,
|
||||
completionReceipts: database.completionReceipts,
|
||||
runAttemptLogRetention: database.runAttemptLogRetention,
|
||||
runLostRetry: database.runLostRetry,
|
||||
localSecrets: database.localSecrets,
|
||||
localSecretAdministration: database.localSecretAdministration,
|
||||
projectPolicy: database.projectPolicy,
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import {
|
||||
RunLostRetryUnavailableError,
|
||||
buildRunLostRetryTransition,
|
||||
normalizeRunLostRetryPageCommand,
|
||||
normalizeRunLostRetryPageResult,
|
||||
type RunLostRetryDisposition,
|
||||
type RunLostRetryPageCommand,
|
||||
type RunLostRetryPageResult,
|
||||
type RunLostRetryRepository,
|
||||
type RunLostRetryTransition,
|
||||
} from '@qinglong/runtime-core/run-lost-retry';
|
||||
import type {
|
||||
RunRecord,
|
||||
RunRepositoryTransaction,
|
||||
RunRetryPolicyRecord,
|
||||
} from '@qinglong/runtime-core/run-repository';
|
||||
|
||||
import { LocalSqliteOperationAuthority } from '../authority/operationAuthority';
|
||||
import { LocalSqliteRunRepository } from './runRepository';
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
interface Candidate {
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
}
|
||||
|
||||
function identifier(row: Row, key: string): string {
|
||||
const value = row[key];
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > 128 ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) {
|
||||
throw new TypeError(`Local SQLite lost retry ${key} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function generatedId(factory: () => string): string {
|
||||
const value = factory();
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > 36 ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) {
|
||||
throw new TypeError('Local SQLite lost retry generated ID is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Local Profiles share one SQLite authority. Candidate discovery and each
|
||||
* BEGIN IMMEDIATE aggregate mutation are bounded, serialized operations; this
|
||||
* repository owns no connection, timer, cursor, or background task.
|
||||
*/
|
||||
export class LocalSqliteRunLostRetryRepository
|
||||
implements RunLostRetryRepository
|
||||
{
|
||||
constructor(
|
||||
private readonly authority: LocalSqliteOperationAuthority,
|
||||
private readonly runs: LocalSqliteRunRepository,
|
||||
private readonly createId: () => string = randomUUID,
|
||||
private readonly clock: { now(): number } = { now: Date.now },
|
||||
) {
|
||||
if (
|
||||
!(authority instanceof LocalSqliteOperationAuthority) ||
|
||||
!(runs instanceof LocalSqliteRunRepository) ||
|
||||
typeof createId !== 'function' ||
|
||||
typeof clock?.now !== 'function'
|
||||
) {
|
||||
throw new TypeError('Local SQLite lost retry repository is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async reconcilePage(
|
||||
input: Readonly<RunLostRetryPageCommand>,
|
||||
): Promise<Readonly<RunLostRetryPageResult>> {
|
||||
const command = normalizeRunLostRetryPageCommand(input);
|
||||
try {
|
||||
const candidates = await this.listCandidates(command.limit + 1);
|
||||
const page = candidates.slice(0, command.limit);
|
||||
const counts: Record<RunLostRetryDisposition | 'raced', number> = {
|
||||
scheduled: 0,
|
||||
requeued: 0,
|
||||
failed_disabled: 0,
|
||||
failed_unsafe: 0,
|
||||
failed_exhausted: 0,
|
||||
raced: 0,
|
||||
};
|
||||
for (const candidate of page) {
|
||||
counts[await this.reconcileCandidate(candidate)] += 1;
|
||||
}
|
||||
return normalizeRunLostRetryPageResult(
|
||||
{
|
||||
scanned: page.length,
|
||||
scheduled: counts.scheduled,
|
||||
requeued: counts.requeued,
|
||||
failed:
|
||||
counts.failed_disabled +
|
||||
counts.failed_unsafe +
|
||||
counts.failed_exhausted,
|
||||
raced: counts.raced,
|
||||
hasMore: candidates.length > command.limit,
|
||||
},
|
||||
command.limit,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof RunLostRetryUnavailableError) throw error;
|
||||
throw new RunLostRetryUnavailableError({ cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
private listCandidates(limit: number): Promise<readonly Candidate[]> {
|
||||
return this.authority.enqueue(
|
||||
async () => {
|
||||
const observedAtMs = this.observedAtMs();
|
||||
const rows = this.authority.client
|
||||
.prepare(
|
||||
`SELECT run."id" AS "runId", attempt."id" AS "attemptId"
|
||||
FROM "Runs" AS run
|
||||
JOIN "RunAttempts" AS attempt
|
||||
ON attempt."run_id" = run."id"
|
||||
AND attempt."attempt" = (
|
||||
SELECT MAX(latest."attempt")
|
||||
FROM "RunAttempts" AS latest
|
||||
WHERE latest."run_id" = run."id"
|
||||
)
|
||||
LEFT JOIN "RunRetryPolicies" AS policy
|
||||
ON policy."run_id" = run."id"
|
||||
WHERE run."execution_owner" = 'runtime'
|
||||
AND run."trigger_type" <> 'plugin_package_workflow'
|
||||
AND run."cancel_requested_at_ms" IS NULL
|
||||
AND attempt."status" = 'lost'
|
||||
AND (
|
||||
run."status" = 'lost'
|
||||
OR (
|
||||
run."status" = 'retry_wait'
|
||||
AND policy."next_attempt_at_ms" IS NOT NULL
|
||||
AND policy."next_attempt_at_ms" <= ?
|
||||
)
|
||||
)
|
||||
ORDER BY
|
||||
CASE WHEN run."status" = 'lost' THEN 0
|
||||
ELSE policy."next_attempt_at_ms" END,
|
||||
run."id"
|
||||
LIMIT ?`,
|
||||
)
|
||||
.all(observedAtMs, limit) as Row[];
|
||||
if (rows.length > limit) {
|
||||
throw new TypeError('Local SQLite lost retry exceeded its page size');
|
||||
}
|
||||
return Object.freeze(
|
||||
rows.map((row) =>
|
||||
Object.freeze({
|
||||
runId: identifier(row, 'runId'),
|
||||
attemptId: identifier(row, 'attemptId'),
|
||||
}),
|
||||
),
|
||||
);
|
||||
},
|
||||
() => new RunLostRetryUnavailableError(),
|
||||
);
|
||||
}
|
||||
|
||||
private async reconcileCandidate(
|
||||
candidate: Readonly<Candidate>,
|
||||
): Promise<RunLostRetryDisposition | 'raced'> {
|
||||
return this.runs.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(candidate.runId);
|
||||
const attempt = await transaction.findLatestAttemptByRunId(
|
||||
candidate.runId,
|
||||
);
|
||||
const policy = await transaction.findRetryPolicyByRunId(candidate.runId);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
attempt.id !== candidate.attemptId ||
|
||||
attempt.status !== 'lost' ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
run.triggerType === 'plugin_package_workflow' ||
|
||||
run.cancelRequestedAtMs !== undefined ||
|
||||
(run.status !== 'lost' && run.status !== 'retry_wait') ||
|
||||
(run.status === 'retry_wait' &&
|
||||
(policy?.nextAttemptAtMs === undefined ||
|
||||
policy.nextAttemptAtMs > this.observedAtMs()))
|
||||
) {
|
||||
return 'raced';
|
||||
}
|
||||
const transition = buildRunLostRetryTransition({
|
||||
run,
|
||||
attempt,
|
||||
policy,
|
||||
observedAtMs: this.observedAtMs(),
|
||||
runEventId: generatedId(this.createId),
|
||||
...(run.status === 'retry_wait'
|
||||
? {
|
||||
attemptId: generatedId(this.createId),
|
||||
attemptEventId: generatedId(this.createId),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
await this.persistTransition(transaction, run, policy, transition);
|
||||
return transition.disposition;
|
||||
});
|
||||
}
|
||||
|
||||
private observedAtMs(): number {
|
||||
const value = this.clock.now();
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError('Local SQLite lost retry clock is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private async persistTransition(
|
||||
transaction: RunRepositoryTransaction,
|
||||
currentRun: Readonly<RunRecord>,
|
||||
currentPolicy: Readonly<RunRetryPolicyRecord> | null,
|
||||
transition: Readonly<RunLostRetryTransition>,
|
||||
): Promise<void> {
|
||||
let version = currentRun.version;
|
||||
for (const run of transition.runTransitions) {
|
||||
if (!(await transaction.compareAndSetRun(run, version))) {
|
||||
throw new TypeError('Local SQLite lost retry Run fence changed');
|
||||
}
|
||||
version = run.version;
|
||||
}
|
||||
if (transition.policy && transition.policy !== currentPolicy) {
|
||||
if (
|
||||
!currentPolicy ||
|
||||
!(await transaction.compareAndSetRetryPolicy(
|
||||
transition.policy,
|
||||
currentPolicy.version,
|
||||
))
|
||||
) {
|
||||
throw new TypeError('Local SQLite lost retry policy fence changed');
|
||||
}
|
||||
}
|
||||
if (transition.attempt) {
|
||||
await transaction.insertAttempt(transition.attempt);
|
||||
}
|
||||
for (const event of transition.events) {
|
||||
await transaction.appendEvent(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -63,6 +63,7 @@ import type { LocalSqlitePluginPackageWorkflowFrontierRepository } from '../plug
|
||||
import type { LocalSqlitePluginPackageWorkflowTaskAttemptAdmissionRepository } from '../plugin-package/workflow/pluginPackageWorkflowTaskAttemptAdmissionRepository';
|
||||
import type { LocalSqlitePluginPackageWorkflowCancellationConvergenceRepository } from '../plugin-package/workflow/pluginPackageWorkflowCancellationConvergenceRepository';
|
||||
import { LocalSqliteRunAttemptLogRetentionRepository } from '../run/runAttemptLogRetentionRepository';
|
||||
import { LocalSqliteRunLostRetryRepository } from '../run/runLostRetryRepository';
|
||||
|
||||
export interface LocalSqliteRuntimeDependencies {
|
||||
readonly taskSpecSemanticRegistry?: TaskSpecSemanticRegistry;
|
||||
@@ -99,6 +100,7 @@ export interface LocalSqliteRuntimeDatabase {
|
||||
readonly executionControl: LocalExecutionControlSource;
|
||||
readonly completionReceipts: LocalCompletionReceiptJournal;
|
||||
readonly runAttemptLogRetention: LocalSqliteRunAttemptLogRetentionRepository;
|
||||
readonly runLostRetry: LocalSqliteRunLostRetryRepository;
|
||||
readonly localSecrets: LocalSecretEnvelopeRepository;
|
||||
readonly localSecretAdministration: LocalSecretAdministrationRepository;
|
||||
readonly projectPolicy: ProjectPolicyRepository;
|
||||
@@ -183,6 +185,10 @@ export async function openLocalSqliteRuntimeDatabase(
|
||||
const ownerPepper = new LocalSqliteOwnerPepperRepository(authority);
|
||||
const runAttemptLogRetention =
|
||||
new LocalSqliteRunAttemptLogRetentionRepository(authority);
|
||||
const runLostRetry = new LocalSqliteRunLostRetryRepository(
|
||||
authority,
|
||||
runRepository,
|
||||
);
|
||||
let pluginPackageInstallsPromise:
|
||||
| Promise<PluginPackageInstallRepository>
|
||||
| undefined;
|
||||
@@ -237,6 +243,7 @@ export async function openLocalSqliteRuntimeDatabase(
|
||||
executionControl: runRuntimeCapabilities.executionControl,
|
||||
completionReceipts: runRuntimeCapabilities.completionReceipts,
|
||||
runAttemptLogRetention,
|
||||
runLostRetry,
|
||||
localSecrets: securityAuthority,
|
||||
localSecretAdministration: securityAuthority,
|
||||
projectPolicy,
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
migrateLocalSqlitePath,
|
||||
openLocalSqliteRuntimeDatabase,
|
||||
} = require('../dist');
|
||||
|
||||
async function fixture(t) {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-lost-retry-'));
|
||||
const databasePath = path.join(directory, 'qinglong3.sqlite');
|
||||
await migrateLocalSqlitePath({ databasePath, profile: 'edge' });
|
||||
const runtime = await openLocalSqliteRuntimeDatabase({
|
||||
databasePath,
|
||||
profile: 'edge',
|
||||
});
|
||||
t.after(async () => {
|
||||
await runtime.close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
return runtime;
|
||||
}
|
||||
|
||||
async function insertLostRun(runtime, suffix, retryPolicy) {
|
||||
const runId = `run-lost-${suffix}`;
|
||||
const attemptId = `attempt-lost-${suffix}`;
|
||||
await runtime.runRepository.transaction(async (transaction) => {
|
||||
await transaction.insertRun({
|
||||
id: runId,
|
||||
projectId: 'default',
|
||||
taskId: `task-${suffix}`,
|
||||
taskRevision: 'revision-1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
status: 'lost',
|
||||
version: 1,
|
||||
eventSequence: 1,
|
||||
priority: 0,
|
||||
createdAtMs: 100,
|
||||
queuedAtMs: 110,
|
||||
startedAtMs: 120,
|
||||
errorCode: 'LOCAL_RECOVERY_EXECUTION_NOT_RUNNING',
|
||||
errorSummary: 'lost',
|
||||
});
|
||||
await transaction.insertAttempt({
|
||||
id: attemptId,
|
||||
runId,
|
||||
attempt: 1,
|
||||
status: 'lost',
|
||||
executorType: 'local_process',
|
||||
callbackSequence: 0,
|
||||
createdAtMs: 110,
|
||||
startedAtMs: 120,
|
||||
finishedAtMs: 130,
|
||||
errorCode: 'LOCAL_RECOVERY_EXECUTION_NOT_RUNNING',
|
||||
errorSummary: 'lost',
|
||||
});
|
||||
if (retryPolicy) {
|
||||
await transaction.insertRetryPolicy({
|
||||
runId,
|
||||
maxAttempts: 3,
|
||||
retryOnLost: true,
|
||||
safety: 'idempotent',
|
||||
backoffBaseMs: 0,
|
||||
backoffMaxMs: 0,
|
||||
version: 0,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
});
|
||||
}
|
||||
});
|
||||
return { runId, attemptId };
|
||||
}
|
||||
|
||||
test('atomically schedules and requeues one safe Local lost Run exactly once', async (t) => {
|
||||
const runtime = await fixture(t);
|
||||
const ids = await insertLostRun(runtime, 'safe', true);
|
||||
|
||||
assert.deepEqual(await runtime.runLostRetry.reconcilePage({ limit: 1 }), {
|
||||
scanned: 1,
|
||||
scheduled: 1,
|
||||
requeued: 0,
|
||||
failed: 0,
|
||||
raced: 0,
|
||||
hasMore: false,
|
||||
});
|
||||
assert.equal(
|
||||
(await runtime.runRepository.findRunById(ids.runId)).status,
|
||||
'retry_wait',
|
||||
);
|
||||
|
||||
assert.deepEqual(await runtime.runLostRetry.reconcilePage({ limit: 1 }), {
|
||||
scanned: 1,
|
||||
scheduled: 0,
|
||||
requeued: 1,
|
||||
failed: 0,
|
||||
raced: 0,
|
||||
hasMore: false,
|
||||
});
|
||||
const run = await runtime.runRepository.findRunById(ids.runId);
|
||||
const attempt = await runtime.runRepository.findLatestAttemptByRunId(
|
||||
ids.runId,
|
||||
);
|
||||
assert.equal(run.status, 'queued');
|
||||
assert.equal(run.version, 4);
|
||||
assert.equal(attempt.attempt, 2);
|
||||
assert.equal(attempt.status, 'claimed');
|
||||
assert.equal(attempt.executorType, 'local_process');
|
||||
assert.deepEqual(
|
||||
(await runtime.runRepository.listEvents(ids.runId)).map(
|
||||
(event) => event.type,
|
||||
),
|
||||
['run.retry_wait', 'run.queued', 'attempt.claimed'],
|
||||
);
|
||||
assert.equal(
|
||||
(await runtime.runLostRetry.reconcilePage({ limit: 1 })).scanned,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed without an admitted retry policy and preserves a bounded page', async (t) => {
|
||||
const runtime = await fixture(t);
|
||||
const first = await insertLostRun(runtime, 'disabled-a', false);
|
||||
await insertLostRun(runtime, 'disabled-b', false);
|
||||
|
||||
const result = await runtime.runLostRetry.reconcilePage({ limit: 1 });
|
||||
assert.deepEqual(result, {
|
||||
scanned: 1,
|
||||
scheduled: 0,
|
||||
requeued: 0,
|
||||
failed: 1,
|
||||
raced: 0,
|
||||
hasMore: true,
|
||||
});
|
||||
assert.equal(
|
||||
(await runtime.runRepository.findRunById(first.runId)).errorCode,
|
||||
'RUN_LOST_RETRY_DISABLED',
|
||||
);
|
||||
await assert.rejects(
|
||||
runtime.runLostRetry.reconcilePage({ limit: 65 }),
|
||||
/page size/,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user