mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): observe system crond runs idempotently
This commit is contained in:
+16
-6
@@ -5,10 +5,7 @@ import CronService from '../services/cron';
|
|||||||
import CronViewService from '../services/cronView';
|
import CronViewService from '../services/cronView';
|
||||||
import { celebrate, Joi } from 'celebrate';
|
import { celebrate, Joi } from 'celebrate';
|
||||||
import { commonCronSchema } from '../validation/schedule';
|
import { commonCronSchema } from '../validation/schedule';
|
||||||
import {
|
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
|
||||||
RunningInstanceModel,
|
|
||||||
InstanceStatus,
|
|
||||||
} from '../data/runningInstance';
|
|
||||||
import { t } from '../shared/i18n';
|
import { t } from '../shared/i18n';
|
||||||
|
|
||||||
const route = Router();
|
const route = Router();
|
||||||
@@ -313,7 +310,11 @@ export default (app: Router) => {
|
|||||||
try {
|
try {
|
||||||
const cronService = Container.get(CronService);
|
const cronService = Container.get(CronService);
|
||||||
const result = await cronService.log(req.params.id);
|
const result = await cronService.log(req.params.id);
|
||||||
return res.send({ code: 200, data: result.content, logStatus: result.status });
|
return res.send({
|
||||||
|
code: 200,
|
||||||
|
data: result.content,
|
||||||
|
logStatus: result.status,
|
||||||
|
});
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return next(e);
|
return next(e);
|
||||||
}
|
}
|
||||||
@@ -435,6 +436,11 @@ export default (app: Router) => {
|
|||||||
last_running_time: Joi.number().optional().allow(null),
|
last_running_time: Joi.number().optional().allow(null),
|
||||||
last_execution_time: Joi.number().optional().allow(null),
|
last_execution_time: Joi.number().optional().allow(null),
|
||||||
exit_code: Joi.number().optional().allow(null),
|
exit_code: Joi.number().optional().allow(null),
|
||||||
|
execution_id: Joi.string()
|
||||||
|
.pattern(
|
||||||
|
/^legacy-system:[1-9][0-9]{0,12}:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u,
|
||||||
|
)
|
||||||
|
.optional(),
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
async (req: Request, res: Response, next: NextFunction) => {
|
async (req: Request, res: Response, next: NextFunction) => {
|
||||||
@@ -483,7 +489,11 @@ export default (app: Router) => {
|
|||||||
instanceId: Joi.number().required(),
|
instanceId: Joi.number().required(),
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
async (req: Request<{ id: number; instanceId: number }>, res: Response, next: NextFunction) => {
|
async (
|
||||||
|
req: Request<{ id: number; instanceId: number }>,
|
||||||
|
res: Response,
|
||||||
|
next: NextFunction,
|
||||||
|
) => {
|
||||||
try {
|
try {
|
||||||
const cronService = Container.get(CronService);
|
const cronService = Container.get(CronService);
|
||||||
const data = await cronService.stopInstance(req.params.instanceId);
|
const data = await cronService.stopInstance(req.params.instanceId);
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
import { v7 as uuidV7 } from 'uuid';
|
import { v7 as uuidV7 } from 'uuid';
|
||||||
import type {
|
import type {
|
||||||
RunAttemptRecord,
|
RunAttemptRecord,
|
||||||
@@ -43,6 +44,38 @@ export interface LegacyShadowRunReference {
|
|||||||
|
|
||||||
export type ShadowIdFactory = () => string;
|
export type ShadowIdFactory = () => string;
|
||||||
|
|
||||||
|
function deterministicShadowId(
|
||||||
|
requestId: string,
|
||||||
|
acceptedAtMs: number,
|
||||||
|
domain: 'run' | 'attempt',
|
||||||
|
): string {
|
||||||
|
if (
|
||||||
|
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u.test(requestId) ||
|
||||||
|
!Number.isSafeInteger(acceptedAtMs) ||
|
||||||
|
acceptedAtMs < 0 ||
|
||||||
|
acceptedAtMs > 0xffffffffffff
|
||||||
|
) {
|
||||||
|
throw new TypeError('Invalid deterministic Legacy Shadow identity');
|
||||||
|
}
|
||||||
|
const bytes = createHash('sha256')
|
||||||
|
.update(`qinglong:legacy-shadow:${domain}:`)
|
||||||
|
.update(requestId)
|
||||||
|
.digest()
|
||||||
|
.subarray(0, 16);
|
||||||
|
let timestamp = acceptedAtMs;
|
||||||
|
for (let index = 5; index >= 0; index -= 1) {
|
||||||
|
bytes[index] = timestamp & 0xff;
|
||||||
|
timestamp = Math.floor(timestamp / 256);
|
||||||
|
}
|
||||||
|
bytes[6] = (bytes[6] & 0x0f) | 0x70;
|
||||||
|
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
||||||
|
const value = bytes.toString('hex');
|
||||||
|
return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(
|
||||||
|
12,
|
||||||
|
16,
|
||||||
|
)}-${value.slice(16, 20)}-${value.slice(20)}`;
|
||||||
|
}
|
||||||
|
|
||||||
export class LegacyShadowRunWriter {
|
export class LegacyShadowRunWriter {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly repository: RunRepository,
|
private readonly repository: RunRepository,
|
||||||
@@ -52,10 +85,25 @@ export class LegacyShadowRunWriter {
|
|||||||
async accept(
|
async accept(
|
||||||
fact: LegacyExecutionAcceptedFact,
|
fact: LegacyExecutionAcceptedFact,
|
||||||
): Promise<LegacyShadowRunReference> {
|
): Promise<LegacyShadowRunReference> {
|
||||||
const reference = {
|
const reference =
|
||||||
runId: this.createId(),
|
fact.requestId === undefined
|
||||||
attemptId: this.createId(),
|
? { runId: this.createId(), attemptId: this.createId() }
|
||||||
|
: {
|
||||||
|
runId: deterministicShadowId(
|
||||||
|
fact.requestId,
|
||||||
|
fact.acceptedAtMs,
|
||||||
|
'run',
|
||||||
|
),
|
||||||
|
attemptId: deterministicShadowId(
|
||||||
|
fact.requestId,
|
||||||
|
fact.acceptedAtMs,
|
||||||
|
'attempt',
|
||||||
|
),
|
||||||
};
|
};
|
||||||
|
const idempotencyKey =
|
||||||
|
fact.requestId === undefined
|
||||||
|
? undefined
|
||||||
|
: `legacy-shadow:${fact.origin}:${fact.requestId}`;
|
||||||
const initialRun: RunRecord = {
|
const initialRun: RunRecord = {
|
||||||
id: reference.runId,
|
id: reference.runId,
|
||||||
projectId: fact.projectId,
|
projectId: fact.projectId,
|
||||||
@@ -79,6 +127,7 @@ export class LegacyShadowRunWriter {
|
|||||||
version: 0,
|
version: 0,
|
||||||
eventSequence: 0,
|
eventSequence: 0,
|
||||||
priority: 0,
|
priority: 0,
|
||||||
|
...(idempotencyKey === undefined ? {} : { idempotencyKey }),
|
||||||
createdAtMs: fact.acceptedAtMs,
|
createdAtMs: fact.acceptedAtMs,
|
||||||
};
|
};
|
||||||
const initialAttempt: RunAttemptRecord = {
|
const initialAttempt: RunAttemptRecord = {
|
||||||
@@ -91,15 +140,23 @@ export class LegacyShadowRunWriter {
|
|||||||
createdAtMs: fact.acceptedAtMs,
|
createdAtMs: fact.acceptedAtMs,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
await this.repository.transaction(async (transaction) => {
|
await this.repository.transaction(async (transaction) => {
|
||||||
await transaction.insertRun(initialRun);
|
await transaction.insertRun(initialRun);
|
||||||
await transaction.insertAttempt(initialAttempt);
|
await transaction.insertAttempt(initialAttempt);
|
||||||
|
|
||||||
const created = reserveRunEvent(initialRun, 0);
|
const created = reserveRunEvent(initialRun, 0);
|
||||||
const createdRun = created.run;
|
const createdRun = created.run;
|
||||||
const createdUpdated = await transaction.compareAndSetRun(createdRun, 0);
|
const createdUpdated = await transaction.compareAndSetRun(
|
||||||
|
createdRun,
|
||||||
|
0,
|
||||||
|
);
|
||||||
if (!createdUpdated) {
|
if (!createdUpdated) {
|
||||||
throw new RunVersionConflictError(initialRun.id, 0, initialRun.version);
|
throw new RunVersionConflictError(
|
||||||
|
initialRun.id,
|
||||||
|
0,
|
||||||
|
initialRun.version,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
await transaction.appendEvent(
|
await transaction.appendEvent(
|
||||||
this.event(
|
this.event(
|
||||||
@@ -130,9 +187,53 @@ export class LegacyShadowRunWriter {
|
|||||||
fact.acceptedAtMs,
|
fact.acceptedAtMs,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (
|
||||||
|
fact.requestId === undefined ||
|
||||||
|
!(await this.isExactReplay(
|
||||||
|
reference,
|
||||||
|
initialRun,
|
||||||
|
initialAttempt,
|
||||||
|
idempotencyKey!,
|
||||||
|
))
|
||||||
|
) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
return reference;
|
return reference;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async isExactReplay(
|
||||||
|
reference: LegacyShadowRunReference,
|
||||||
|
expectedRun: RunRecord,
|
||||||
|
expectedAttempt: RunAttemptRecord,
|
||||||
|
idempotencyKey: string,
|
||||||
|
): Promise<boolean> {
|
||||||
|
const [run, attempt] = await Promise.all([
|
||||||
|
this.repository.findRunById(reference.runId),
|
||||||
|
this.repository.findAttemptById(reference.attemptId),
|
||||||
|
]);
|
||||||
|
if (!run || !attempt) return false;
|
||||||
|
return (
|
||||||
|
run.projectId === expectedRun.projectId &&
|
||||||
|
run.taskId === expectedRun.taskId &&
|
||||||
|
run.taskRevision === expectedRun.taskRevision &&
|
||||||
|
run.taskName === expectedRun.taskName &&
|
||||||
|
run.legacyCronId === expectedRun.legacyCronId &&
|
||||||
|
run.triggerType === expectedRun.triggerType &&
|
||||||
|
run.executionOrigin === expectedRun.executionOrigin &&
|
||||||
|
run.executionOwner === 'legacy' &&
|
||||||
|
run.triggeredBy === expectedRun.triggeredBy &&
|
||||||
|
run.requestId === expectedRun.requestId &&
|
||||||
|
run.idempotencyKey === idempotencyKey &&
|
||||||
|
run.createdAtMs === expectedRun.createdAtMs &&
|
||||||
|
attempt.runId === run.id &&
|
||||||
|
attempt.attempt === 1 &&
|
||||||
|
attempt.executorType === 'legacy_local' &&
|
||||||
|
attempt.createdAtMs === expectedAttempt.createdAtMs
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
async spawned(
|
async spawned(
|
||||||
reference: LegacyShadowRunReference,
|
reference: LegacyShadowRunReference,
|
||||||
fact: LegacyExecutionSpawnedFact,
|
fact: LegacyExecutionSpawnedFact,
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ const SHADOW_ORIGINS_ENV = 'QL3_SHADOW_ORIGINS';
|
|||||||
const SUPPORTED_SHADOW_ORIGINS = new Set<ExecutionOrigin>([
|
const SUPPORTED_SHADOW_ORIGINS = new Set<ExecutionOrigin>([
|
||||||
'manual',
|
'manual',
|
||||||
'scheduled_node',
|
'scheduled_node',
|
||||||
|
'scheduled_system',
|
||||||
'script',
|
'script',
|
||||||
'subscription',
|
'subscription',
|
||||||
'system',
|
'system',
|
||||||
@@ -84,7 +85,7 @@ function readConfiguredOrigins(): ReadonlySet<ExecutionOrigin> {
|
|||||||
incrementFailure('configuration:unsupported_origin');
|
incrementFailure('configuration:unsupported_origin');
|
||||||
try {
|
try {
|
||||||
Logger.warn(
|
Logger.warn(
|
||||||
'[ql3-shadow] ignored unsupported origin; this slice supports manual,scheduled_node,script,subscription,system',
|
'[ql3-shadow] ignored unsupported origin; this slice supports manual,scheduled_node,scheduled_system,script,subscription,system',
|
||||||
);
|
);
|
||||||
} catch {
|
} catch {
|
||||||
// Invalid compatibility configuration must not affect legacy paths.
|
// Invalid compatibility configuration must not affect legacy paths.
|
||||||
@@ -269,6 +270,57 @@ export function observeLegacyExecution(
|
|||||||
: NOOP_OBSERVATION;
|
: NOOP_OBSERVATION;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface LegacyShellExecutionCallbackInput {
|
||||||
|
pid?: number;
|
||||||
|
logPath?: string;
|
||||||
|
atMs: number;
|
||||||
|
phase: 'running' | 'finished';
|
||||||
|
exitCode?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Observes one callback-owned execution without registering an in-memory
|
||||||
|
* ChildProcess correlation entry. The accepted fact must carry a durable
|
||||||
|
* requestId so callback replay can converge in the Shadow writer.
|
||||||
|
*/
|
||||||
|
export function observeLegacyShellExecutionCallback(
|
||||||
|
createFact: LegacyExecutionAcceptedFactFactory,
|
||||||
|
input: LegacyShellExecutionCallbackInput,
|
||||||
|
): LegacyExecutionObservation | undefined {
|
||||||
|
const origin: ExecutionOrigin = 'scheduled_system';
|
||||||
|
let observation: LegacyExecutionObservation;
|
||||||
|
if (override) {
|
||||||
|
if (!override.origins.has(origin)) return undefined;
|
||||||
|
const fact = createAcceptedFactFailOpen(origin, createFact);
|
||||||
|
observation = fact
|
||||||
|
? beginFailOpen(override.observer, fact)
|
||||||
|
: NOOP_OBSERVATION;
|
||||||
|
} else {
|
||||||
|
if (!readConfiguredOrigins().has(origin)) return undefined;
|
||||||
|
const fact = createAcceptedFactFailOpen(origin, createFact);
|
||||||
|
observation = fact
|
||||||
|
? deferredObservation(getDefaultObserver(), fact)
|
||||||
|
: NOOP_OBSERVATION;
|
||||||
|
}
|
||||||
|
|
||||||
|
observation.spawned({
|
||||||
|
atMs: input.atMs,
|
||||||
|
...(input.pid === undefined ? {} : { pid: input.pid }),
|
||||||
|
...(input.logPath === undefined
|
||||||
|
? {}
|
||||||
|
: { logArtifactId: createLegacyLogArtifactId(input.logPath) }),
|
||||||
|
});
|
||||||
|
if (input.phase === 'running') {
|
||||||
|
observation.running({ atMs: input.atMs });
|
||||||
|
} else {
|
||||||
|
observation.exited({
|
||||||
|
atMs: input.atMs,
|
||||||
|
exitCode: input.exitCode ?? 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return observation;
|
||||||
|
}
|
||||||
|
|
||||||
export interface LegacyExecutionCancellationInput {
|
export interface LegacyExecutionCancellationInput {
|
||||||
legacyCronId: number;
|
legacyCronId: number;
|
||||||
pid?: number;
|
pid?: number;
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import type { Crontab } from '../../data/cron';
|
||||||
|
import type { LegacyExecutionObservation } from '../ports/legacyExecutionObserver';
|
||||||
|
import { observeLegacyShellExecutionCallback } from './legacyExecutionBridge';
|
||||||
|
import { createLegacyTaskRevision } from './legacyTaskRevision';
|
||||||
|
|
||||||
|
const EXECUTION_ID_PATTERN =
|
||||||
|
/^legacy-system:([1-9][0-9]{0,12}):([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/u;
|
||||||
|
|
||||||
|
export interface LegacyScheduledSystemCallbackInput {
|
||||||
|
executionId: string;
|
||||||
|
phase: 'running' | 'finished';
|
||||||
|
observedAtMs: number;
|
||||||
|
pid?: number;
|
||||||
|
logPath?: string;
|
||||||
|
exitCode?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function decorateScheduledSystemCronCommand(
|
||||||
|
command: string,
|
||||||
|
systemScheduler: boolean,
|
||||||
|
): string {
|
||||||
|
return systemScheduler
|
||||||
|
? `QL_EXECUTION_ORIGIN=scheduled_system ${command}`
|
||||||
|
: command;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseLegacyScheduledSystemExecutionId(
|
||||||
|
value: string,
|
||||||
|
): { requestId: string; acceptedAtMs: number } | undefined {
|
||||||
|
const match = EXECUTION_ID_PATTERN.exec(value);
|
||||||
|
if (!match) return undefined;
|
||||||
|
const acceptedAtMs = Number(match[1]) * 1000;
|
||||||
|
if (!Number.isSafeInteger(acceptedAtMs) || acceptedAtMs < 1) return undefined;
|
||||||
|
return Object.freeze({ requestId: value, acceptedAtMs });
|
||||||
|
}
|
||||||
|
|
||||||
|
export function observeLegacyScheduledSystemExecution(
|
||||||
|
cron: Crontab,
|
||||||
|
input: LegacyScheduledSystemCallbackInput,
|
||||||
|
): LegacyExecutionObservation | undefined {
|
||||||
|
const identity = parseLegacyScheduledSystemExecutionId(input.executionId);
|
||||||
|
if (
|
||||||
|
!identity ||
|
||||||
|
cron.id === undefined ||
|
||||||
|
!Number.isSafeInteger(input.observedAtMs) ||
|
||||||
|
input.observedAtMs < identity.acceptedAtMs
|
||||||
|
) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return observeLegacyShellExecutionCallback(
|
||||||
|
() => ({
|
||||||
|
origin: 'scheduled_system',
|
||||||
|
projectId: 'default',
|
||||||
|
taskId: `legacy-cron:${cron.id}`,
|
||||||
|
taskRevision: createLegacyTaskRevision({
|
||||||
|
command: cron.command,
|
||||||
|
...(cron.schedule === undefined ? {} : { schedule: cron.schedule }),
|
||||||
|
extraSchedules:
|
||||||
|
cron.extra_schedules?.map((item) => item.schedule) ?? [],
|
||||||
|
...(cron.task_before === undefined
|
||||||
|
? {}
|
||||||
|
: { taskBefore: cron.task_before }),
|
||||||
|
...(cron.task_after === undefined
|
||||||
|
? {}
|
||||||
|
: { taskAfter: cron.task_after }),
|
||||||
|
...(cron.work_dir === undefined
|
||||||
|
? {}
|
||||||
|
: { workDirectory: cron.work_dir }),
|
||||||
|
...(cron.log_name === undefined ? {} : { logName: cron.log_name }),
|
||||||
|
}),
|
||||||
|
...(cron.name === undefined ? {} : { taskName: cron.name }),
|
||||||
|
legacyCronId: cron.id,
|
||||||
|
triggerType: 'scheduled_system',
|
||||||
|
triggeredBy: 'legacy:system-crond',
|
||||||
|
requestId: identity.requestId,
|
||||||
|
scheduledForMs: identity.acceptedAtMs,
|
||||||
|
acceptedAtMs: identity.acceptedAtMs,
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
phase: input.phase,
|
||||||
|
atMs: input.observedAtMs,
|
||||||
|
...(input.pid === undefined ? {} : { pid: input.pid }),
|
||||||
|
...(input.logPath === undefined ? {} : { logPath: input.logPath }),
|
||||||
|
...(input.exitCode === undefined ? {} : { exitCode: input.exitCode }),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
+29
-4
@@ -39,6 +39,10 @@ import {
|
|||||||
createLegacyLogArtifactId,
|
createLegacyLogArtifactId,
|
||||||
createLegacyTaskRevision,
|
createLegacyTaskRevision,
|
||||||
} from '../runtime/compatibility/legacyTaskRevision';
|
} from '../runtime/compatibility/legacyTaskRevision';
|
||||||
|
import {
|
||||||
|
decorateScheduledSystemCronCommand,
|
||||||
|
observeLegacyScheduledSystemExecution,
|
||||||
|
} from '../runtime/compatibility/legacyScheduledSystemExecution';
|
||||||
import {
|
import {
|
||||||
selectManualPrimaryExecutionRouter,
|
selectManualPrimaryExecutionRouter,
|
||||||
stopManualPrimaryAttempt,
|
stopManualPrimaryAttempt,
|
||||||
@@ -233,6 +237,7 @@ export default class CronService {
|
|||||||
last_running_time = 0,
|
last_running_time = 0,
|
||||||
last_execution_time = 0,
|
last_execution_time = 0,
|
||||||
exit_code,
|
exit_code,
|
||||||
|
execution_id,
|
||||||
}: {
|
}: {
|
||||||
ids: number[];
|
ids: number[];
|
||||||
status: CrontabStatus;
|
status: CrontabStatus;
|
||||||
@@ -241,6 +246,7 @@ export default class CronService {
|
|||||||
last_running_time: number;
|
last_running_time: number;
|
||||||
last_execution_time: number;
|
last_execution_time: number;
|
||||||
exit_code?: number;
|
exit_code?: number;
|
||||||
|
execution_id?: string;
|
||||||
}) {
|
}) {
|
||||||
let options: any = {
|
let options: any = {
|
||||||
status,
|
status,
|
||||||
@@ -261,15 +267,27 @@ export default class CronService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (status === CrontabStatus.running || status === CrontabStatus.idle) {
|
if (status === CrontabStatus.running || status === CrontabStatus.idle) {
|
||||||
|
const observedAtMs = Date.now();
|
||||||
|
if (execution_id) {
|
||||||
|
observeLegacyScheduledSystemExecution(cron, {
|
||||||
|
executionId: execution_id,
|
||||||
|
...(pid ? { pid } : {}),
|
||||||
|
...(log_path ? { logPath: log_path } : {}),
|
||||||
|
observedAtMs,
|
||||||
|
phase: status === CrontabStatus.running ? 'running' : 'finished',
|
||||||
|
...(exit_code === undefined ? {} : { exitCode: exit_code }),
|
||||||
|
});
|
||||||
|
} else {
|
||||||
observeLegacyExecutionCallback({
|
observeLegacyExecutionCallback({
|
||||||
legacyCronId: id,
|
legacyCronId: id,
|
||||||
...(pid ? { pid } : {}),
|
...(pid ? { pid } : {}),
|
||||||
...(log_path ? { logPath: log_path } : {}),
|
...(log_path ? { logPath: log_path } : {}),
|
||||||
atMs: Date.now(),
|
atMs: observedAtMs,
|
||||||
phase: status === CrontabStatus.running ? 'running' : 'finished',
|
phase: status === CrontabStatus.running ? 'running' : 'finished',
|
||||||
...(exit_code === undefined ? {} : { exitCode: exit_code }),
|
...(exit_code === undefined ? {} : { exitCode: exit_code }),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if (status === CrontabStatus.idle && log_path !== cron.log_path) {
|
if (status === CrontabStatus.idle && log_path !== cron.log_path) {
|
||||||
options = omit(options, ['status', 'log_path', 'pid']);
|
options = omit(options, ['status', 'log_path', 'pid']);
|
||||||
}
|
}
|
||||||
@@ -1046,6 +1064,7 @@ export default class CronService {
|
|||||||
|
|
||||||
private async setCrontab(data?: { data: Crontab[]; total: number }) {
|
private async setCrontab(data?: { data: Crontab[]; total: number }) {
|
||||||
const tabs = data ?? (await this.crontabs());
|
const tabs = data ?? (await this.crontabs());
|
||||||
|
const systemScheduler = this.schedulerMode === 'system';
|
||||||
var crontab_string = '';
|
var crontab_string = '';
|
||||||
tabs.data.forEach((tab) => {
|
tabs.data.forEach((tab) => {
|
||||||
if (
|
if (
|
||||||
@@ -1056,19 +1075,25 @@ export default class CronService {
|
|||||||
crontab_string += '# ';
|
crontab_string += '# ';
|
||||||
crontab_string += tab.schedule;
|
crontab_string += tab.schedule;
|
||||||
crontab_string += ' ';
|
crontab_string += ' ';
|
||||||
crontab_string += this.makeCommand(tab);
|
crontab_string += decorateScheduledSystemCronCommand(
|
||||||
|
this.makeCommand(tab),
|
||||||
|
systemScheduler,
|
||||||
|
);
|
||||||
crontab_string += '\n';
|
crontab_string += '\n';
|
||||||
} else {
|
} else {
|
||||||
crontab_string += tab.schedule;
|
crontab_string += tab.schedule;
|
||||||
crontab_string += ' ';
|
crontab_string += ' ';
|
||||||
crontab_string += this.makeCommand(tab);
|
crontab_string += decorateScheduledSystemCronCommand(
|
||||||
|
this.makeCommand(tab),
|
||||||
|
systemScheduler,
|
||||||
|
);
|
||||||
crontab_string += '\n';
|
crontab_string += '\n';
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
await writeFileWithLock(config.crontabFile, crontab_string);
|
await writeFileWithLock(config.crontabFile, crontab_string);
|
||||||
|
|
||||||
if (this.schedulerMode === 'system') {
|
if (systemScheduler) {
|
||||||
try {
|
try {
|
||||||
execSync(`crontab ${config.crontabFile}`);
|
execSync(`crontab ${config.crontabFile}`);
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
|
|||||||
@@ -11,6 +11,21 @@
|
|||||||
|
|
||||||
最新增量证据(2026-08-18):
|
最新增量证据(2026-08-18):
|
||||||
|
|
||||||
|
- D-354/ADR-0446(已接受):关闭 system crond Shadow 的最后一个准入身份缺口。只有实际 system scheduler 写出的
|
||||||
|
crontab 命令带 `QL_EXECUTION_ORIGIN=scheduled_system`;Node scheduler、manual/boot 与直接 Shell 调用不带标记,避免一次执行双记。Shell 每次
|
||||||
|
被标记执行只生成一个 `legacy-system:<start-seconds>:<uuid-v4>`,start/finish callback 原样复用;Linux 路由设备优先读取 kernel UUID,
|
||||||
|
`uuidgen`/Node 只作 fallback。后端严格校验 ID,使用专用 detached observer 从 finish-only callback 建立 accepted→spawned→exited,不伪造
|
||||||
|
ChildProcess。Shadow writer 以 ID/accepted time 派生稳定 UUIDv7 形态 Run/Attempt ID并复用既有 Project idempotency unique index;只有完整
|
||||||
|
task/Cron/origin/request/time fence 一致才接受重放,response loss、重复 finish 和迟到 start 不增加 Run/Event,定义漂移失败开放。默认仍关闭,
|
||||||
|
不新增 package、生产依赖、schema、migration、timer、watcher、网络重试、端口或部署对象。D-354 开始前还复核了 18 包边界:4 个小包分别是
|
||||||
|
六消费者文件协议叶子、高权限短生命周期维护进程、三消费者 POSIX adapter 与 Secret/keyring authority,合并会扩大依赖或权限闭包,因此保留;
|
||||||
|
当前没有单源码包或平铺实现包。阶段门已重跑:system crond/Bridge/Correlation 聚焦 38/38、`build:back` 与 4 个 Shell 文件语法检查、完整
|
||||||
|
backend 1,415 pass + 2 条条件 skip/0 fail、18-package clean build/test、14/14 静态审计与 14/14 artifact 档位全部通过;artifact
|
||||||
|
字节与 D-353 完全一致:基础 Edge/Standalone `2589998/2590076`、adopted `2809293/2809416`、application
|
||||||
|
`3632877/3632997`、application-api `3800430/3800574`、AI `3069251/3069341`、application+AI
|
||||||
|
`4493151/4493283`、MCP `7315930/7316038`。本阶段没有修改数据库 schema/adapter、容器或 Kubernetes 拓扑,因此没有重跑物理
|
||||||
|
PostgreSQL HA/K3s 门,也不把相邻阶段结果冒充 D-354 新证据。
|
||||||
|
|
||||||
- D-353/ADR-0445(已接受;`scheduled_system` 的 response-loss-safe 幂等准入待独立 Gate):3.0 Shadow Run 不再只覆盖
|
- D-353/ADR-0445(已接受;`scheduled_system` 的 response-loss-safe 幂等准入待独立 Gate):3.0 Shadow Run 不再只覆盖
|
||||||
`manual/scheduled_node`。现有 `ScheduleService.runTask` 在既有 task limit 选中且 `onBefore` 成功后,可按默认关闭的
|
`manual/scheduled_node`。现有 `ScheduleService.runTask` 在既有 task limit 选中且 `onBefore` 成功后,可按默认关闭的
|
||||||
`QL3_SHADOW_ORIGINS` 精确旁路观察 `subscription/system/script`;观察器只附着到 Legacy 已创建的同一个 ChildProcess,不调用
|
`QL3_SHADOW_ORIGINS` 精确旁路观察 `subscription/system/script`;观察器只附着到 Legacy 已创建的同一个 ChildProcess,不调用
|
||||||
@@ -9055,7 +9070,7 @@ flowchart LR
|
|||||||
| PR-1 Run Schema | Incubating | Run/RunAttempt/RunEvent schema、nullable cancel request 与 Attempt deadline 字段及恢复索引、CancellationDispatch 状态/version/lease/backoff schema、Repository port、临时 Sequelize adapter、统一事件大小/分页上限、跨 adapter RunRepository contract suite(原子事务、回滚、Run/Attempt/RetryPolicy CAS、唯一错误、分页与取消恢复);ADR-0041 的 `pg-0003-run-retry-policy`、capability v2、driver-neutral PostgreSQL Run Repository 与真实 `pg.Pool` 上的共享 Repository/rollback/SQLSTATE contract;ADR-0063/0069/0071/0073/0074/0076 的独立 Node 24 local-sqlite typed schema、十二条 reviewed migration、capability v6、共享 operation authority、readiness/RunRepository/API credential repository/receipt journal/dispatch plan/encrypted Secret envelope/Project Policy/security audit/authorized mutation/stable Identity catalog、Drizzle↔真实 catalog table/column/index/CHECK/FK lockstep、base/adopted/application edge/standalone 产物门禁;ADR-0064 的 legacy baseline/plan digest、Online Backup recovery、side-by-side target migration、staged manifest、双库栅栏 activation、source 生命周期写栅栏、target stable identity 和重启语义;ADR-0065 的独立 cutover authority、外部副作用停机 evidence、append-only journal、start/restart/stop barrier 与 unknown→manual_required 收敛;ADR-0066 的 adopted storage→Run reconciliation→receipt maintenance→domain recovery→lifecycle→admission application gate、严格有界 recovery summary 与 admission-first reverse stop;ADR-0067 的 SQLite 事实驱动 Run 候选源、256 条硬上限、截断失败关闭和唯一 Repository authority;ADR-0068 的 receipt-first Reconciler、callback token/sequence fence、exact local-process identity、Attempt/Run/双 Event 原子终态推进和最终 verifier;ADR-0069 的 local-process 单向包边界、pre-spawn journal、受审 POSIX launcher、immutable receipt、exact identity 和 Profile-aware cleanup lifecycle;ADR-0070 的独立 local-execution、spawn 前后双 transaction CAS、callback digest、exact stop 补偿与 fail-closed starting 保留;ADR-0071 的独立 local-dispatch、不可变 revision/context、Secret-first materializer、Profile Artifact admission、4/64 MiB output hard quota 和窄 application facade;ADR-0073/0074 的 Project-bound SecretRef、AES-256-GCM、外置 keyring 生命周期、双 SQLite authority CAS、application preflight、强 Principal/Policy 和 envelope+audit 原子提交;ADR-0086 的本机 Owner provisioning/challenge/claim/delivery acknowledgement/credential recovery CLI;ADR-0377 的 Local/Cluster 同构、Profile-aware、Project-scoped Artifact range read | fresh database/pepper setup、credential rotation/GC 运维编排与 Secret/Project/Role/Approval 管理 CLI/API/UI、备份/rekey、2.x/target process controller、人工 recovery、target 写后 reconciliation 与完整 cutover/rollback 演练;retry 产品策略、Artifact retention/tombstone stack、具体本机 lifecycle 和 target executable;Linux x64/arm64、PID namespace、断电与固定路由设备门禁;PostgreSQL 16/18 双连接并发与 failover integration;Task revision/context 跨方言 contract/并发压力与引用感知 retention、Keyv 数据迁移 |
|
| PR-1 Run Schema | Incubating | Run/RunAttempt/RunEvent schema、nullable cancel request 与 Attempt deadline 字段及恢复索引、CancellationDispatch 状态/version/lease/backoff schema、Repository port、临时 Sequelize adapter、统一事件大小/分页上限、跨 adapter RunRepository contract suite(原子事务、回滚、Run/Attempt/RetryPolicy CAS、唯一错误、分页与取消恢复);ADR-0041 的 `pg-0003-run-retry-policy`、capability v2、driver-neutral PostgreSQL Run Repository 与真实 `pg.Pool` 上的共享 Repository/rollback/SQLSTATE contract;ADR-0063/0069/0071/0073/0074/0076 的独立 Node 24 local-sqlite typed schema、十二条 reviewed migration、capability v6、共享 operation authority、readiness/RunRepository/API credential repository/receipt journal/dispatch plan/encrypted Secret envelope/Project Policy/security audit/authorized mutation/stable Identity catalog、Drizzle↔真实 catalog table/column/index/CHECK/FK lockstep、base/adopted/application edge/standalone 产物门禁;ADR-0064 的 legacy baseline/plan digest、Online Backup recovery、side-by-side target migration、staged manifest、双库栅栏 activation、source 生命周期写栅栏、target stable identity 和重启语义;ADR-0065 的独立 cutover authority、外部副作用停机 evidence、append-only journal、start/restart/stop barrier 与 unknown→manual_required 收敛;ADR-0066 的 adopted storage→Run reconciliation→receipt maintenance→domain recovery→lifecycle→admission application gate、严格有界 recovery summary 与 admission-first reverse stop;ADR-0067 的 SQLite 事实驱动 Run 候选源、256 条硬上限、截断失败关闭和唯一 Repository authority;ADR-0068 的 receipt-first Reconciler、callback token/sequence fence、exact local-process identity、Attempt/Run/双 Event 原子终态推进和最终 verifier;ADR-0069 的 local-process 单向包边界、pre-spawn journal、受审 POSIX launcher、immutable receipt、exact identity 和 Profile-aware cleanup lifecycle;ADR-0070 的独立 local-execution、spawn 前后双 transaction CAS、callback digest、exact stop 补偿与 fail-closed starting 保留;ADR-0071 的独立 local-dispatch、不可变 revision/context、Secret-first materializer、Profile Artifact admission、4/64 MiB output hard quota 和窄 application facade;ADR-0073/0074 的 Project-bound SecretRef、AES-256-GCM、外置 keyring 生命周期、双 SQLite authority CAS、application preflight、强 Principal/Policy 和 envelope+audit 原子提交;ADR-0086 的本机 Owner provisioning/challenge/claim/delivery acknowledgement/credential recovery CLI;ADR-0377 的 Local/Cluster 同构、Profile-aware、Project-scoped Artifact range read | fresh database/pepper setup、credential rotation/GC 运维编排与 Secret/Project/Role/Approval 管理 CLI/API/UI、备份/rekey、2.x/target process controller、人工 recovery、target 写后 reconciliation 与完整 cutover/rollback 演练;retry 产品策略、Artifact retention/tombstone stack、具体本机 lifecycle 和 target executable;Linux x64/arm64、PID namespace、断电与固定路由设备门禁;PostgreSQL 16/18 双连接并发与 failover integration;Task revision/context 跨方言 contract/并发压力与引用感知 retention、Keyv 数据迁移 |
|
||||||
| PR-2 Run 状态机 | Incubating | 纯转换表、终态/时间/错误/执行器元数据规则、Run version 与 event sequence CAS、事务性 RunCommandService、回滚测试 | 重复 Worker callback/fencing、并发数据库压力测试、Primary 执行链接入 |
|
| PR-2 Run 状态机 | Incubating | 纯转换表、终态/时间/错误/执行器元数据规则、Run version 与 event sequence CAS、事务性 RunCommandService、回滚测试 | 重复 Worker callback/fencing、并发数据库压力测试、Primary 执行链接入 |
|
||||||
| PR-3 Executor 端口 | Incubating | ADR-0003、ExecutionSpec/Context/Handle/Result、Executor port、LocalProcessExecutor、进程组取消/超时升级、流式背压、Legacy Cron spec builder、真实进程 contract tests、可复现 edge 基准入口 | 固定 edge/多架构设备基线、Legacy builder 与 makeCommand 差异审计、Primary 生产流量接入 |
|
| PR-3 Executor 端口 | Incubating | ADR-0003、ExecutionSpec/Context/Handle/Result、Executor port、LocalProcessExecutor、进程组取消/超时升级、流式背压、Legacy Cron spec builder、真实进程 contract tests、可复现 edge 基准入口 | 固定 edge/多架构设备基线、Legacy builder 与 makeCommand 差异审计、Primary 生产流量接入 |
|
||||||
| PR-4 Shadow Run | Incubating | origin 三态策略;默认关闭的 `QL3_SHADOW_ORIGINS`;manual、scheduled_node、subscription、system 与 script 现有 ChildProcess 旁路观察;每个 worker 懒加载;Run/Attempt/Event 影子生命周期;稳定且不复制 caller 原文的 task identity/revision 与有界日志引用;同 worker 有界注册表和跨 worker 持久化候选关联;stop all/stop instance、Shell callback、乱序/迟到/歧义处理;失败开放和契约测试 | system crond `scheduled_system` 的 response-loss-safe accept identity/幂等准入;once/boot/grpc 独立裁决;启动后 Reconciler、差异报表、可采集指标、资源压力、回滚演练和 Primary 门禁 |
|
| PR-4 Shadow Run | Incubating | origin 三态策略;默认关闭的 `QL3_SHADOW_ORIGINS`;manual、scheduled_node、subscription、system 与 script 现有 ChildProcess 旁路观察;system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay;每个 worker 懒加载;Run/Attempt/Event 影子生命周期;稳定且不复制 caller 原文的 task identity/revision 与有界日志引用;同 worker 有界注册表和跨 worker 持久化候选关联;stop all/stop instance、Shell callback、乱序/迟到/歧义处理;失败开放和契约测试 | once/boot/grpc 独立裁决;启动后 Reconciler、差异报表、可采集指标、资源压力、回滚演练和 Primary 门禁 |
|
||||||
| PR-5 Primary LocalExecutor | Incubating(默认不激活,仅 manifest-gated manual) | runtime-owned Run 创建器;持久化先于 spawn;Run/Attempt 完整成功、失败、取消、超时与 lost 闭环;Executor handle 身份校验;spawn 后激活写失败的 stop+lost 补偿;completion rejection 安全收敛;独立 Primary 幂等查询与唯一索引竞态裁决;durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询;最多 64 条一页的 cross-worker cancellation source;独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent;最多 64 页的单周期 cancel supervisor;显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner;Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller;完整有界分页且 fail-closed 的 startup Reconcile supervisor;RunningInstance nullable `run_id/attempt_id` 关联;Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance,失败整体回滚;有界且防穿越的 legacy log output ref;manual owner seam、真实本机装配、单 spawn/fail-closed;严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计;HTTP worker 已接轻量 lazy bootstrap,accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活,失败撤销,监听失败和 shutdown 有界停止;Primary timeout 在 spawn 前持久化绝对 deadline,有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch;代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限,cluster-control/worker 拒绝误装本机 SQLite Primary;统一 CompletionService 原子提交 Attempt/Run/双 Event,spawn 前保存 callback token hash、终态推进 sequence,实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window;manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理;Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace;`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描,使终态残留继续可发现;确定无效的已知 Attempt receipt 先持久化隔离状态,再进入确定性私有分片 quarantine;终态 missing 与 quarantine 按 edge/standalone retention 有界清理;非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI;扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数;ENOSPC 与 launcher receipt 存储失败有代码门禁 | 部署配置写入/审批入口与用户可见状态;PostgreSQL CancellationDispatch adapter;cluster-control 生产启动拓扑;固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 |
|
| PR-5 Primary LocalExecutor | Incubating(默认不激活,仅 manifest-gated manual) | runtime-owned Run 创建器;持久化先于 spawn;Run/Attempt 完整成功、失败、取消、超时与 lost 闭环;Executor handle 身份校验;spawn 后激活写失败的 stop+lost 补偿;completion rejection 安全收敛;独立 Primary 幂等查询与唯一索引竞态裁决;durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询;最多 64 条一页的 cross-worker cancellation source;独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent;最多 64 页的单周期 cancel supervisor;显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner;Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller;完整有界分页且 fail-closed 的 startup Reconcile supervisor;RunningInstance nullable `run_id/attempt_id` 关联;Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance,失败整体回滚;有界且防穿越的 legacy log output ref;manual owner seam、真实本机装配、单 spawn/fail-closed;严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计;HTTP worker 已接轻量 lazy bootstrap,accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活,失败撤销,监听失败和 shutdown 有界停止;Primary timeout 在 spawn 前持久化绝对 deadline,有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch;代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限,cluster-control/worker 拒绝误装本机 SQLite Primary;统一 CompletionService 原子提交 Attempt/Run/双 Event,spawn 前保存 callback token hash、终态推进 sequence,实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window;manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理;Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace;`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描,使终态残留继续可发现;确定无效的已知 Attempt receipt 先持久化隔离状态,再进入确定性私有分片 quarantine;终态 missing 与 quarantine 按 edge/standalone retention 有界清理;非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI;扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数;ENOSPC 与 launcher receipt 存储失败有代码门禁 | 部署配置写入/审批入口与用户可见状态;PostgreSQL CancellationDispatch adapter;cluster-control 生产启动拓扑;固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 |
|
||||||
| PR-7 Worker Session、Run Lease 与启动协议基础 | Incubating(默认关闭,独立入口显式 opt-in) | ADR-0012/0013/0014/0021/0057–0061/0108–0121/0231–0239/0377;有界 capability/Placement/Dispatcher;SQLite 协议孵化与 PostgreSQL v9 Session/Run Lease/credential/attestation authority;immutable revision Placement、数据库时钟 keyset candidate、认证 Worker Pull、digest-only offer recovery;versioned capability-free ExecutionSpec response、stable claim 跨重启退避、单 owner 原子 inbox 准入与 TLS 1.3 mTLS/`ql3w` HTTPS client;同一 package journal 上 revision-fenced starting/spawn/started/running/completion 状态、callback digest、tagged no-spawn 与 ambiguous recovery;PostgreSQL starting/running/start-failure/completion 数据库权威事务、精确重放与 cancellation/timeout 优先终态;batch Secret delivery 在 Attempt advisory lock 下复验 Session/Lease/revision 完整围栏并复用单 Agent,Secret-before-Artifact materializer 将同一 log ID 交给 Executor/journal/running ACK;offer-scoped `wlog-*` 私有文件 spool、Edge/Node 容量策略、append/quota/path 防护、barrier 后 output ownership、受审 POSIX Executor、truncation fact、固定内存流式 source、认证 Artifact stream、共享 immutable store port、S3-compatible SSE/checksum/条件 promotion adapter、upload-before-completion 协调,以及 Local/Cluster 同构、Profile-aware、ETag-fenced range read;用户取消 run.stop mutation 以数据库时间写 intent/Event 并在事务内复验 Project/RoleBinding fence;非执行取消 convergence lifecycle、运行期 expiry 与安全 lost retry 已接入 cluster-control 单一全局 cadence;完整 generation/version/token/Attempt fencing;独立最小权限 Worker ingress、CA/CRL 与连接 generation 热重载;offer journal、spawn barrier、receipt-first recovery;独立 `@qinglong/worker-runtime` 的本地 P-256 CSR、key/chain/trust 验证、generation + active pointer 安装和持久退避;默认关闭的 production process 已装配具体 execution graph、完整 Session heartbeat/drain/offline、direct-file bootstrap、单 Agent/单 cadence、startup reconciliation、证书 maintenance、transport fail-close/recovery 与 Edge/Node 有界预算;真实 PostgreSQL 18 + Linux Node 合约已覆盖 Run completion、credential 和 CA 双轮换且保持同一 Session;真实 K3s 合约已覆盖 TLS/credential Secret 分权、双对象 CAS、Recreate 顺序、identity generation 与单节点 PVC recovery;所有能力默认不可达且受 edge/cluster import audit 约束 | 具体 cert-manager/Vault/SPIFFE/离线 CA adapter 与模板、ingress reload controller、生产 RBAC、证书到期告警和 `ql3w` credential recovery 产品面;具体 KMS/Vault Secret provider、对象存储 credential/temporary lifecycle 与 retention/tombstone;Worker 管理 API;真实 Kubernetes 多节点 CSI/node-loss/production 360 秒 drain 与固定 edge 文件系统 suspend/时钟/断电、x64/arm64 资源门禁 |
|
| PR-7 Worker Session、Run Lease 与启动协议基础 | Incubating(默认关闭,独立入口显式 opt-in) | ADR-0012/0013/0014/0021/0057–0061/0108–0121/0231–0239/0377;有界 capability/Placement/Dispatcher;SQLite 协议孵化与 PostgreSQL v9 Session/Run Lease/credential/attestation authority;immutable revision Placement、数据库时钟 keyset candidate、认证 Worker Pull、digest-only offer recovery;versioned capability-free ExecutionSpec response、stable claim 跨重启退避、单 owner 原子 inbox 准入与 TLS 1.3 mTLS/`ql3w` HTTPS client;同一 package journal 上 revision-fenced starting/spawn/started/running/completion 状态、callback digest、tagged no-spawn 与 ambiguous recovery;PostgreSQL starting/running/start-failure/completion 数据库权威事务、精确重放与 cancellation/timeout 优先终态;batch Secret delivery 在 Attempt advisory lock 下复验 Session/Lease/revision 完整围栏并复用单 Agent,Secret-before-Artifact materializer 将同一 log ID 交给 Executor/journal/running ACK;offer-scoped `wlog-*` 私有文件 spool、Edge/Node 容量策略、append/quota/path 防护、barrier 后 output ownership、受审 POSIX Executor、truncation fact、固定内存流式 source、认证 Artifact stream、共享 immutable store port、S3-compatible SSE/checksum/条件 promotion adapter、upload-before-completion 协调,以及 Local/Cluster 同构、Profile-aware、ETag-fenced range read;用户取消 run.stop mutation 以数据库时间写 intent/Event 并在事务内复验 Project/RoleBinding fence;非执行取消 convergence lifecycle、运行期 expiry 与安全 lost retry 已接入 cluster-control 单一全局 cadence;完整 generation/version/token/Attempt fencing;独立最小权限 Worker ingress、CA/CRL 与连接 generation 热重载;offer journal、spawn barrier、receipt-first recovery;独立 `@qinglong/worker-runtime` 的本地 P-256 CSR、key/chain/trust 验证、generation + active pointer 安装和持久退避;默认关闭的 production process 已装配具体 execution graph、完整 Session heartbeat/drain/offline、direct-file bootstrap、单 Agent/单 cadence、startup reconciliation、证书 maintenance、transport fail-close/recovery 与 Edge/Node 有界预算;真实 PostgreSQL 18 + Linux Node 合约已覆盖 Run completion、credential 和 CA 双轮换且保持同一 Session;真实 K3s 合约已覆盖 TLS/credential Secret 分权、双对象 CAS、Recreate 顺序、identity generation 与单节点 PVC recovery;所有能力默认不可达且受 edge/cluster import audit 约束 | 具体 cert-manager/Vault/SPIFFE/离线 CA adapter 与模板、ingress reload controller、生产 RBAC、证书到期告警和 `ql3w` credential recovery 产品面;具体 KMS/Vault Secret provider、对象存储 credential/temporary lifecycle 与 retention/tombstone;Worker 管理 API;真实 Kubernetes 多节点 CSI/node-loss/production 360 秒 drain 与固定 edge 文件系统 suspend/时钟/断电、x64/arm64 资源门禁 |
|
||||||
| PR-8 Project/Policy/Approval Core | Incubating(默认拒绝、无生产业务执行入口) | ADR-0028;统一六类 ActorRef 与 exact-shape 校验;`0017` ownerless default Project 和 append-only versioned RoleBinding;owner/admin/operator/viewer 固定矩阵;Project 内 mutation 幂等、expected-version CAS、双 SQLite 连接竞争门禁;archived read-only、revocation、存储损坏 fail-closed;Agent 写/Secret/Tool `require_approval`;ADR-0047 把六类 subject、role/permission matrix 与 fence 抽到 runtime-core,`pg-0004-project-policy`/capability v3 建立 ownerless PostgreSQL baseline、严格 role/state CHECK、append-only runtime 权限、SERIALIZABLE Project lock、mutation replay、双连接单 winner 和 cluster admission authorizer;ADR-0049/`pg-0005` capability v4 建立 stable IdentitySubject、append-only digest-only API credential、真实 cluster bearer authenticator、write-only durable security audit 与最小权限 runtime role,且已验证 HTTP→credential→Policy→audit→handler 纵向链路;ADR-0051 建立 `/api/v3` 认证前 peer/global 双预算、transport-peer-only、无 timer 且有界内存的 overload shield;ADR-0027 Artifact authorizer adapter;ADR-0029 `AuthenticatedPrincipal` contract、`0018` digest-only versioned challenge、CSPRNG/TTL、同事务消费 challenge + 写首 owner、精确重放与双连接竞争/崩溃回滚门禁;ADR-0030 `0019` stable identity/binding、legacy HS384 + current-session membership、logout/platform/revoke/disable、single-factor 与损坏 fail-closed 门禁;ADR-0031 `0020` digest-bound ApprovalRequest、User-only decision、Project/Role version fence、精确 expiry/重放/并发裁决及同事务 immutable dispatch;ADR-0032 `0021` execution backfill、三表原子 consume、稳定 due keyset、claim/renew/start/result fencing、pre-start takeover/post-start recovery-required、attempt budget、handler inspect/digest barrier 和 bounded dispatcher;ADR-0033/`0022` control/resolution backfill、start/renew/completion 原子联动、稳定 recovery keyset、双 resolver claim/takeover、finding/result 精确重放、自动/人工终结、迟到 completion 单 winner 和 evidence-only bounded reconciler;ADR-0034/`0023` 首个 `run.create` canonical plan、Run/Attempt/Event/receipt 同事务、幂等 collision fail-closed、renew/终态 fence、真实 SQLite handler 与 automatic evidence provider;ADR-0035/`0024` 独立 `approval.recover` 矩阵、稳定 User + 五分钟强认证、Project/RoleBinding fence、human resolution + authorization fact 原子提交、撤权竞态与回滚门禁;ADR-0036 recovery-first 单 timer lifecycle、edge/standalone 独立 cadence/页预算、跨周期 cursor、非重叠与有界 stop;ADR-0074 以新的 Node 24 SQLite v5 ownerless Project/RoleBinding/audit authority 和独立 local-secret-admin 提供强 Principal、`secret.manage`、撤权 fence、envelope+allowed audit 原子提交及不回显语义;ADR-0086 以可信 POSIX console 和 staged delivery 完成本机首 Owner 产品 ceremony | fresh database/pepper setup 与安全迁移向导;`shareStore`/Express 到 authentication core 的 production migration;credential rotation/revocation API、mTLS/Worker enrollment、恢复码;Project/Role/Approval/Secret 管理 CLI/API/UI、audit retention/query/export/alert、preview Artifact/digest/immutable plan builder、真实 MFA/hardware adapter、人工 recovery API/UI/独立 rate limit 与审计事件、handler/provider registry、lifecycle startup/shutdown/指标/admission gate;PostgreSQL action/receipt/provider/recovery-authorization 与 OPA adapter、缓存 version 失效;Tool/Package/Secret/Shell 各自的 handler/evidence contract;Secret/Run/Tool/Workflow waiting_approval 全入口装配;完整回滚演练 |
|
| PR-8 Project/Policy/Approval Core | Incubating(默认拒绝、无生产业务执行入口) | ADR-0028;统一六类 ActorRef 与 exact-shape 校验;`0017` ownerless default Project 和 append-only versioned RoleBinding;owner/admin/operator/viewer 固定矩阵;Project 内 mutation 幂等、expected-version CAS、双 SQLite 连接竞争门禁;archived read-only、revocation、存储损坏 fail-closed;Agent 写/Secret/Tool `require_approval`;ADR-0047 把六类 subject、role/permission matrix 与 fence 抽到 runtime-core,`pg-0004-project-policy`/capability v3 建立 ownerless PostgreSQL baseline、严格 role/state CHECK、append-only runtime 权限、SERIALIZABLE Project lock、mutation replay、双连接单 winner 和 cluster admission authorizer;ADR-0049/`pg-0005` capability v4 建立 stable IdentitySubject、append-only digest-only API credential、真实 cluster bearer authenticator、write-only durable security audit 与最小权限 runtime role,且已验证 HTTP→credential→Policy→audit→handler 纵向链路;ADR-0051 建立 `/api/v3` 认证前 peer/global 双预算、transport-peer-only、无 timer 且有界内存的 overload shield;ADR-0027 Artifact authorizer adapter;ADR-0029 `AuthenticatedPrincipal` contract、`0018` digest-only versioned challenge、CSPRNG/TTL、同事务消费 challenge + 写首 owner、精确重放与双连接竞争/崩溃回滚门禁;ADR-0030 `0019` stable identity/binding、legacy HS384 + current-session membership、logout/platform/revoke/disable、single-factor 与损坏 fail-closed 门禁;ADR-0031 `0020` digest-bound ApprovalRequest、User-only decision、Project/Role version fence、精确 expiry/重放/并发裁决及同事务 immutable dispatch;ADR-0032 `0021` execution backfill、三表原子 consume、稳定 due keyset、claim/renew/start/result fencing、pre-start takeover/post-start recovery-required、attempt budget、handler inspect/digest barrier 和 bounded dispatcher;ADR-0033/`0022` control/resolution backfill、start/renew/completion 原子联动、稳定 recovery keyset、双 resolver claim/takeover、finding/result 精确重放、自动/人工终结、迟到 completion 单 winner 和 evidence-only bounded reconciler;ADR-0034/`0023` 首个 `run.create` canonical plan、Run/Attempt/Event/receipt 同事务、幂等 collision fail-closed、renew/终态 fence、真实 SQLite handler 与 automatic evidence provider;ADR-0035/`0024` 独立 `approval.recover` 矩阵、稳定 User + 五分钟强认证、Project/RoleBinding fence、human resolution + authorization fact 原子提交、撤权竞态与回滚门禁;ADR-0036 recovery-first 单 timer lifecycle、edge/standalone 独立 cadence/页预算、跨周期 cursor、非重叠与有界 stop;ADR-0074 以新的 Node 24 SQLite v5 ownerless Project/RoleBinding/audit authority 和独立 local-secret-admin 提供强 Principal、`secret.manage`、撤权 fence、envelope+allowed audit 原子提交及不回显语义;ADR-0086 以可信 POSIX console 和 staged delivery 完成本机首 Owner 产品 ceremony | fresh database/pepper setup 与安全迁移向导;`shareStore`/Express 到 authentication core 的 production migration;credential rotation/revocation API、mTLS/Worker enrollment、恢复码;Project/Role/Approval/Secret 管理 CLI/API/UI、audit retention/query/export/alert、preview Artifact/digest/immutable plan builder、真实 MFA/hardware adapter、人工 recovery API/UI/独立 rate limit 与审计事件、handler/provider registry、lifecycle startup/shutdown/指标/admission gate;PostgreSQL action/receipt/provider/recovery-authorization 与 OPA adapter、缓存 version 失效;Tool/Package/Secret/Shell 各自的 handler/evidence contract;Secret/Run/Tool/Workflow waiting_approval 全入口装配;完整回滚演练 |
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
- 决策者:QingLong Maintainers
|
- 决策者:QingLong Maintainers
|
||||||
- 关联 RFC:[QL-RFC-0001](../QINGLONG_3_0_ARCHITECTURE_RFC.md)
|
- 关联 RFC:[QL-RFC-0001](../QINGLONG_3_0_ARCHITECTURE_RFC.md)
|
||||||
- 前置决策:[ADR-0001](./ADR-0001-run-state-and-transaction-boundaries.md)
|
- 前置决策:[ADR-0001](./ADR-0001-run-state-and-transaction-boundaries.md)
|
||||||
- Amended by:[ADR-0445](./ADR-0445-schedule-service-origin-shadow-run-coverage.md)
|
- Amended by:[ADR-0445](./ADR-0445-schedule-service-origin-shadow-run-coverage.md)、[ADR-0446](./ADR-0446-system-crond-stable-shadow-admission.md)
|
||||||
|
|
||||||
## 1. 决策摘要
|
## 1. 决策摘要
|
||||||
|
|
||||||
@@ -145,15 +145,16 @@ owner 在接受触发时写入执行上下文,并贯穿日志、指标和回
|
|||||||
|
|
||||||
当前孵化实现只开放观察型 Shadow,不通过该环境变量提供 primary:
|
当前孵化实现只开放观察型 Shadow,不通过该环境变量提供 primary:
|
||||||
|
|
||||||
QL3_SHADOW_ORIGINS=manual,scheduled_node,subscription,system,script
|
QL3_SHADOW_ORIGINS=manual,scheduled_node,scheduled_system,subscription,system,script
|
||||||
|
|
||||||
- 未设置或设置为空时全部为 off。
|
- 未设置或设置为空时全部为 off。
|
||||||
- ADR-0445 后当前只接受 `manual`、`scheduled_node`、`subscription`、`system` 与 `script`;未知 origin 被忽略并记录有界配置告警,
|
- ADR-0446 后当前接受 `manual`、`scheduled_node`、`scheduled_system`、`subscription`、`system` 与 `script`;未知 origin 被忽略并记录
|
||||||
`scheduled_system`、`once`、`boot` 与 `grpc` 仍不开放。
|
有界配置告警,`once`、`boot` 与 `grpc` 仍不开放。
|
||||||
- 配置在进程内首次使用时读取;edge 不启动 watcher,变更后需要通过既有进程重启或未来的显式 reload 生效。
|
- 配置在进程内首次使用时读取;edge 不启动 watcher,变更后需要通过既有进程重启或未来的显式 reload 生效。
|
||||||
- 兼容观察器在实际 HTTP/gRPC worker 中按需加载;关闭时不构造 Shadow 事实或任务摘要、不增加 ChildProcess 监听器、不初始化 Repository、不创建后台任务,也不引入额外数据库写入。
|
- 兼容观察器在实际 HTTP/gRPC worker 中按需加载;关闭时不构造 Shadow 事实或任务摘要、不增加 ChildProcess 监听器、不初始化 Repository、不创建后台任务,也不引入额外数据库写入。
|
||||||
- 所有已开放 origin 都只监听 Legacy 已创建的同一个 ChildProcess。Shadow 代码不得调用 Executor 或第二次 spawn;
|
- `manual/scheduled_node/subscription/system/script` 只监听 Legacy 已创建的同一个 ChildProcess。`scheduled_system` 不持有 Node ChildProcess,
|
||||||
`subscription/system/script` 仅在 `ScheduleService` 已选中执行且 `onBefore` 成功后 accepted。
|
只接受 system crond 显式标记后由 Shell start/finish 共用的稳定 execution ID;finish-only 回调可以幂等补齐 accepted→terminal 聚合。Shadow
|
||||||
|
代码不得调用 Executor 或第二次 spawn;`subscription/system/script` 仅在 `ScheduleService` 已选中执行且 `onBefore` 成功后 accepted。
|
||||||
- 任意初始化、接受或后续写入失败都退化为 no-op,只记录不含命令、环境变量和 Secret 的稳定错误类型与有界计数。
|
- 任意初始化、接受或后续写入失败都退化为 no-op,只记录不含命令、环境变量和 Secret 的稳定错误类型与有界计数。
|
||||||
- `boot` 虽复用 `runSingle`,仍携带独立 origin,当前不在允许列表中,不能被误记为 manual。
|
- `boot` 虽复用 `runSingle`,仍携带独立 origin,当前不在允许列表中,不能被误记为 manual。
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
- 关联 RFC:QL-RFC-0001 D-02、D-353、PR-4
|
- 关联 RFC:QL-RFC-0001 D-02、D-353、PR-4
|
||||||
- 关联 ADR:ADR-0001、ADR-0002、ADR-0003
|
- 关联 ADR:ADR-0001、ADR-0002、ADR-0003
|
||||||
- Amends:ADR-0002 的当前 Alpha Shadow origin allowlist,不改变 Legacy owner 或 Primary 门禁
|
- Amends:ADR-0002 的当前 Alpha Shadow origin allowlist,不改变 Legacy owner 或 Primary 门禁
|
||||||
|
- Follow-up:[ADR-0446](./ADR-0446-system-crond-stable-shadow-admission.md) 已完成本文保留的 `scheduled_system` 稳定准入 Gate
|
||||||
|
|
||||||
## 上下文
|
## 上下文
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
# ADR-0446:System Crond 稳定 Shadow 准入与回调重放
|
||||||
|
|
||||||
|
- 状态:Accepted
|
||||||
|
- 日期:2026-08-18
|
||||||
|
- 关联 RFC:QL-RFC-0001 D-02、D-354、PR-4
|
||||||
|
- 关联 ADR:ADR-0001、ADR-0002、ADR-0445
|
||||||
|
- Amends:ADR-0002 的 Alpha Shadow origin allowlist 与 system crond callback 关联规则
|
||||||
|
|
||||||
|
## 上下文
|
||||||
|
|
||||||
|
system crond 不由 Node worker spawn。它执行 `crontab.list` 中的 Shell 命令,`task.sh/share.sh` 再分别向
|
||||||
|
`/open/crons/status` 发送 running 与 idle callback。旧 callback 只有 Cron ID、PID、log path 和秒级开始时间:running callback 丢失时,finish
|
||||||
|
无法证明 accepted identity;重复、乱序或 HTTP response loss 又可能把同一次执行创建为多个 Shadow Run。因此 D-353 明确拒绝从结束事实直接伪造
|
||||||
|
`scheduled_system` Run。
|
||||||
|
|
||||||
|
这条边界还必须区分 Node scheduler 和面板手动执行。三者最终都可能进入同一 Shell 脚本;仅凭 `ID`、PID 或 `real_log_path` 猜测来源会把同一次手动
|
||||||
|
执行同时记为 `manual` 与 `scheduled_system`。
|
||||||
|
|
||||||
|
## 决策
|
||||||
|
|
||||||
|
1. 只有 `CronService.setCrontab` 在实际 system scheduler 模式写出的命令增加
|
||||||
|
`QL_EXECUTION_ORIGIN=scheduled_system`。Node scheduler 注册命令、manual/boot `runSingle` 和直接 Shell 调用不带该标记。
|
||||||
|
2. Shell 仅在标记存在时生成一次 `legacy-system:<start-seconds>:<uuid-v4>` execution ID,并在 start/finish callback 复用。UUID 优先读取 Linux
|
||||||
|
kernel random UUID,其次使用 `uuidgen`,最后使用已存在的 Node runtime;三者都不可用或输出非法时不发送 ID,Legacy 执行仍继续且不创建
|
||||||
|
`scheduled_system` Shadow Run。
|
||||||
|
3. `/open/crons/status` 只接受严格小写 UUIDv4 和正整数秒时间的可选 `execution_id`。旧客户端没有该字段时继续走原 Cron ID/PID/log correlation,
|
||||||
|
不改变 2.x 请求兼容性。
|
||||||
|
4. 带稳定 ID 的 callback 使用专用 detached observation,不注册虚构 ChildProcess,也不进入易歧义的本机 registry。running 映射为
|
||||||
|
accepted→spawned→running;finish 映射为 accepted→spawned→exited,因此 start request/response 丢失后,finish-only 仍能形成完整的终态聚合。
|
||||||
|
5. Shadow Run 固定 `executionOwner=legacy`、origin/trigger type `scheduled_system`、`triggeredBy=legacy:system-crond`、Project `default` 和
|
||||||
|
`legacy-cron:<id>` task identity。accepted/scheduled 时间从 execution ID 内的开始秒派生,task revision 摘要与 manual/node Cron 使用相同字段集合。
|
||||||
|
6. 带 request ID 的 `LegacyShadowRunWriter` 使用 request ID 和 accepted time 派生稳定 UUIDv7 形态的 Run/Attempt ID,并写既有
|
||||||
|
`(project_id,idempotency_key)` 唯一键。重放只有在 Run、Attempt、task revision、Cron ID、origin、request ID 与创建时间全部一致时才复用;任一
|
||||||
|
漂移都失败开放,不创建第二个 Run,也不改 Legacy callback 结果。
|
||||||
|
7. `QL3_SHADOW_ORIGINS` 增加 `scheduled_system`,但默认仍为 off;本决定不开放 Primary,不调用 Executor,不增加网络重试、timer、watcher 或后台
|
||||||
|
reconciler。
|
||||||
|
|
||||||
|
## Response-loss 与乱序语义
|
||||||
|
|
||||||
|
- start 成功且 response 丢失:finish 使用同一 execution ID,复用既有 Run 并终结。
|
||||||
|
- start request 未到达:finish-only 创建同一确定性 Run/Attempt 后直接终结。
|
||||||
|
- finish 成功且 response 丢失后重放:唯一键和确定性 ID 命中 exact replay,终态与 Event 数不增加。
|
||||||
|
- finish 先于迟到 start:迟到 start 命中已终态聚合,writer 的幂等状态推进保持终态不变。
|
||||||
|
- 相同 ID 携带不同任务定义:exact replay 校验失败,Shadow 记录有界 accept failure;Legacy 状态更新与任务结果不受影响。
|
||||||
|
- 无 ID、ID 非法或来源未显式标记:不创建 `scheduled_system` Run;无 ID 的既有 callback correlation 保持原行为。
|
||||||
|
|
||||||
|
## 部署与资源影响
|
||||||
|
|
||||||
|
- 不新增 workspace package、生产依赖、schema、migration、表、索引、端口、Kubernetes object 或常驻进程。
|
||||||
|
- 复用 Run 表既有 idempotency unique index;SQLite 与 PostgreSQL Repository contract 不变。
|
||||||
|
- 默认关闭时后端只多一次缓存 Set 查询;system crond 命令仍可执行,Shell 只在显式标记下读取一个 UUID。
|
||||||
|
- Edge/路由设备优先读取 `/proc/sys/kernel/random/uuid`,不额外启动 Node;只有缺少 kernel UUID 与 `uuidgen` 时才使用已有 Node runtime 作为
|
||||||
|
兼容 fallback。
|
||||||
|
- 不增加 callback retry,避免低配设备网络阻塞扩大;本决定保证重放安全和 finish-only 收敛,不把“最终一定送达”伪装成已解决问题。
|
||||||
|
|
||||||
|
## 被拒绝的替代方案
|
||||||
|
|
||||||
|
### 继续使用 Cron ID、PID 与 log path
|
||||||
|
|
||||||
|
拒绝。它们在并发、PID 复用、`/dev/null` 日志和进程重启后都不能证明一次 execution identity。
|
||||||
|
|
||||||
|
### 每个 callback 在后端生成新 ID
|
||||||
|
|
||||||
|
拒绝。start/finish 以及 response-loss 重放会产生不同 Run,无法幂等收敛。
|
||||||
|
|
||||||
|
### 为 Shell callback 注册虚构 ChildProcess
|
||||||
|
|
||||||
|
拒绝。Node 不拥有 system crond 子进程;伪造 handle 会污染取消、恢复和 owner 语义。
|
||||||
|
|
||||||
|
### 默认启用 scheduled_system Shadow
|
||||||
|
|
||||||
|
拒绝。低写入寿命设备必须显式选择迁移观测成本,且 Primary 与对账门仍未完成。
|
||||||
|
|
||||||
|
## 验证
|
||||||
|
|
||||||
|
- Shell contract 覆盖显式 origin 才生成 ID、UUID 格式和 callback JSON 原样复用;
|
||||||
|
- SQLite 集成覆盖 running→finish、start response-loss replay、finish-only、重复终态和 task revision drift;
|
||||||
|
- Bridge/registry/correlation/ChildProcess/ScheduleService/rollout 聚焦回归 38/38 通过,包含真实隔离 crontab 文件写入与 system/node 模式差异;
|
||||||
|
- `build:back`、4 个 Shell 文件语法检查与完整 backend 回归通过(1,415 pass、2 条条件 skip、0 fail);
|
||||||
|
- 18-package clean build/test 退出 0,14/14 静态审计与 14/14 artifact 档位均 compatible,artifact 字节与 D-353 相同;
|
||||||
|
- 物理 PostgreSQL HA/K3s 只在数据库或部署面变化时重跑,本决定不以相邻阶段证据冒充新运行。
|
||||||
+2
-1
@@ -448,7 +448,8 @@
|
|||||||
| [ADR-0442](./ADR-0442-catalog-ready-terminal-release-tag-publication.md) | Catalog-ready 的终态 Release Tag 发布与闭合收据 | Superseded by ADR-0443(bounded promotion/closure 机制保留) |
|
| [ADR-0442](./ADR-0442-catalog-ready-terminal-release-tag-publication.md) | Catalog-ready 的终态 Release Tag 发布与闭合收据 | Superseded by ADR-0443(bounded promotion/closure 机制保留) |
|
||||||
| [ADR-0443](./ADR-0443-deployment-ready-terminal-release-finalization.md) | Deployment-ready 的终态 Release Finalization | Accepted(首份真实 GHCR deployment-ready finalization 待实际 release tag) |
|
| [ADR-0443](./ADR-0443-deployment-ready-terminal-release-finalization.md) | Deployment-ready 的终态 Release Finalization | Accepted(首份真实 GHCR deployment-ready finalization 待实际 release tag) |
|
||||||
| [ADR-0444](./ADR-0444-fail-closed-release-tag-finalizer-and-replay-rehearsal.md) | Fail-closed Release Tag Finalizer 与重放演练 | Accepted(首份真实 GHCR response-loss 重放待实际 release tag) |
|
| [ADR-0444](./ADR-0444-fail-closed-release-tag-finalizer-and-replay-rehearsal.md) | Fail-closed Release Tag Finalizer 与重放演练 | Accepted(首份真实 GHCR response-loss 重放待实际 release tag) |
|
||||||
| [ADR-0445](./ADR-0445-schedule-service-origin-shadow-run-coverage.md) | ScheduleService 执行来源的 Shadow Run 覆盖 | Accepted(`scheduled_system` 幂等准入待独立 Gate) |
|
| [ADR-0445](./ADR-0445-schedule-service-origin-shadow-run-coverage.md) | ScheduleService 执行来源的 Shadow Run 覆盖 | Accepted(`scheduled_system` 后续由 ADR-0446 完成) |
|
||||||
|
| [ADR-0446](./ADR-0446-system-crond-stable-shadow-admission.md) | System Crond 稳定 Shadow 准入与回调重放 | Accepted |
|
||||||
|
|
||||||
## 规则
|
## 规则
|
||||||
|
|
||||||
|
|||||||
@@ -142,11 +142,15 @@ update_cron() {
|
|||||||
local lastExecutingTime="${5:-0}"
|
local lastExecutingTime="${5:-0}"
|
||||||
local runningTime="${6:-0}"
|
local runningTime="${6:-0}"
|
||||||
local exitCode="${7:-}"
|
local exitCode="${7:-}"
|
||||||
|
local executionId="${8:-}"
|
||||||
local currentTimeStamp=$(date +%s)
|
local currentTimeStamp=$(date +%s)
|
||||||
local dataRaw="{\"ids\":[$ids],\"status\":\"$status\",\"pid\":\"$pid\",\"log_path\":\"$logPath\",\"last_execution_time\":$lastExecutingTime,\"last_running_time\":$runningTime"
|
local dataRaw="{\"ids\":[$ids],\"status\":\"$status\",\"pid\":\"$pid\",\"log_path\":\"$logPath\",\"last_execution_time\":$lastExecutingTime,\"last_running_time\":$runningTime"
|
||||||
if [[ -n $exitCode ]]; then
|
if [[ -n $exitCode ]]; then
|
||||||
dataRaw="${dataRaw},\"exit_code\":$exitCode"
|
dataRaw="${dataRaw},\"exit_code\":$exitCode"
|
||||||
fi
|
fi
|
||||||
|
if [[ -n $executionId ]]; then
|
||||||
|
dataRaw="${dataRaw},\"execution_id\":\"$executionId\""
|
||||||
|
fi
|
||||||
dataRaw="${dataRaw}}"
|
dataRaw="${dataRaw}}"
|
||||||
local api=$(
|
local api=$(
|
||||||
curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons/status?t=$currentTimeStamp" \
|
curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons/status?t=$currentTimeStamp" \
|
||||||
@@ -166,6 +170,27 @@ update_cron() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
create_legacy_system_execution_id() {
|
||||||
|
local startedAt="$1"
|
||||||
|
local uuid=""
|
||||||
|
if [[ "${QL_EXECUTION_ORIGIN:-}" != "scheduled_system" ]] ||
|
||||||
|
[[ ! "$startedAt" =~ ^[0-9]{1,13}$ ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [[ -r /proc/sys/kernel/random/uuid ]]; then
|
||||||
|
IFS= read -r uuid </proc/sys/kernel/random/uuid
|
||||||
|
elif command -v uuidgen >/dev/null 2>&1; then
|
||||||
|
uuid=$(uuidgen)
|
||||||
|
elif command -v node >/dev/null 2>&1; then
|
||||||
|
uuid=$(node -e "process.stdout.write(require('node:crypto').randomUUID())")
|
||||||
|
fi
|
||||||
|
uuid=$(printf '%s' "$uuid" | tr '[:upper:]' '[:lower:]')
|
||||||
|
if [[ ! "$uuid" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
printf 'legacy-system:%s:%s' "$startedAt" "$uuid"
|
||||||
|
}
|
||||||
|
|
||||||
notify_api() {
|
notify_api() {
|
||||||
local title="$1"
|
local title="$1"
|
||||||
local content="$2"
|
local content="$2"
|
||||||
|
|||||||
+2
-2
@@ -377,7 +377,7 @@ clear_env() {
|
|||||||
handle_task_start() {
|
handle_task_start() {
|
||||||
local error_message=""
|
local error_message=""
|
||||||
if [[ $ID ]]; then
|
if [[ $ID ]]; then
|
||||||
local error=$(update_cron "\"$ID\"" "0" "$$" "$log_path" "$begin_timestamp")
|
local error=$(update_cron "\"$ID\"" "0" "$$" "$log_path" "$begin_timestamp" "" "" "$execution_id")
|
||||||
if [[ $error ]]; then
|
if [[ $error ]]; then
|
||||||
error_message=", 任务状态更新失败(${error})"
|
error_message=", 任务状态更新失败(${error})"
|
||||||
fi
|
fi
|
||||||
@@ -414,7 +414,7 @@ handle_task_end() {
|
|||||||
[[ "$diff_time" == 0 ]] && diff_time=1
|
[[ "$diff_time" == 0 ]] && diff_time=1
|
||||||
|
|
||||||
if [[ $ID ]]; then
|
if [[ $ID ]]; then
|
||||||
local error=$(update_cron "\"$ID\"" "1" "$$" "$log_path" "$begin_timestamp" "$diff_time" "$exit_code")
|
local error=$(update_cron "\"$ID\"" "1" "$$" "$log_path" "$begin_timestamp" "$diff_time" "$exit_code" "$execution_id")
|
||||||
if [[ $error ]]; then
|
if [[ $error ]]; then
|
||||||
error_message=", 状态更新失败(${error})"
|
error_message=", 状态更新失败(${error})"
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -126,6 +126,7 @@ format_params() {
|
|||||||
init_begin_time() {
|
init_begin_time() {
|
||||||
begin_time=$(format_time "$time_format" "$time")
|
begin_time=$(format_time "$time_format" "$time")
|
||||||
begin_timestamp=$(format_timestamp "$time_format" "$time")
|
begin_timestamp=$(format_timestamp "$time_format" "$time")
|
||||||
|
execution_id=$(create_legacy_system_execution_id "$begin_timestamp")
|
||||||
}
|
}
|
||||||
|
|
||||||
import_config "$@"
|
import_config "$@"
|
||||||
|
|||||||
+3
-2
@@ -488,6 +488,7 @@ main() {
|
|||||||
local time_format="%Y-%m-%d %H:%M:%S"
|
local time_format="%Y-%m-%d %H:%M:%S"
|
||||||
local time=$(date "+$time_format")
|
local time=$(date "+$time_format")
|
||||||
local begin_timestamp=$(format_timestamp "$time_format" "$time")
|
local begin_timestamp=$(format_timestamp "$time_format" "$time")
|
||||||
|
local execution_id=$(create_legacy_system_execution_id "$begin_timestamp")
|
||||||
|
|
||||||
local begin_time=$(format_time "$time_format" "$time")
|
local begin_time=$(format_time "$time_format" "$time")
|
||||||
|
|
||||||
@@ -495,7 +496,7 @@ main() {
|
|||||||
eval echo -e "\#\# 开始执行... $begin_time\\\n" $cmd
|
eval echo -e "\#\# 开始执行... $begin_time\\\n" $cmd
|
||||||
fi
|
fi
|
||||||
|
|
||||||
[[ $ID ]] && update_cron "\"$ID\"" "0" "$$" "$log_path" "$begin_timestamp"
|
[[ $ID ]] && update_cron "\"$ID\"" "0" "$$" "$log_path" "$begin_timestamp" "" "" "$execution_id"
|
||||||
|
|
||||||
case $p1 in
|
case $p1 in
|
||||||
update)
|
update)
|
||||||
@@ -558,7 +559,7 @@ main() {
|
|||||||
local end_time=$(format_time "$time_format" "$etime")
|
local end_time=$(format_time "$time_format" "$etime")
|
||||||
local end_timestamp=$(format_timestamp "$time_format" "$etime")
|
local end_timestamp=$(format_timestamp "$time_format" "$etime")
|
||||||
local diff_time=$(($end_timestamp - $begin_timestamp))
|
local diff_time=$(($end_timestamp - $begin_timestamp))
|
||||||
[[ $ID ]] && update_cron "\"$ID\"" "1" "$$" "$log_path" "$begin_timestamp" "$diff_time"
|
[[ $ID ]] && update_cron "\"$ID\"" "1" "$$" "$log_path" "$begin_timestamp" "$diff_time" "" "$execution_id"
|
||||||
|
|
||||||
if [[ "$p1" != "repo" ]] && [[ "$p1" != "raw" ]]; then
|
if [[ "$p1" != "repo" ]] && [[ "$p1" != "raw" ]]; then
|
||||||
eval echo -e "\\\n\#\# 执行结束... $end_time 耗时 $diff_time 秒 " $cmd
|
eval echo -e "\\\n\#\# 执行结束... $end_time 耗时 $diff_time 秒 " $cmd
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ function successfulCommand() {
|
|||||||
return `${JSON.stringify(process.execPath)} -e "process.exit(0)"`;
|
return `${JSON.stringify(process.execPath)} -e "process.exit(0)"`;
|
||||||
}
|
}
|
||||||
|
|
||||||
test('admits the three reviewed origins through the default environment boundary', () => {
|
test('admits the reviewed schedule and system-crond origins through the environment boundary', () => {
|
||||||
const source = `
|
const source = `
|
||||||
const bridge = require('./back/runtime/compatibility/legacyExecutionBridge');
|
const bridge = require('./back/runtime/compatibility/legacyExecutionBridge');
|
||||||
const fact = (origin) => ({
|
const fact = (origin) => ({
|
||||||
@@ -56,11 +56,11 @@ test('admits the three reviewed origins through the default environment boundary
|
|||||||
triggerType: origin,
|
triggerType: origin,
|
||||||
acceptedAtMs: 1,
|
acceptedAtMs: 1,
|
||||||
});
|
});
|
||||||
const result = ['subscription', 'system', 'script', 'boot'].map((origin) =>
|
const result = ['subscription', 'system', 'script', 'scheduled_system', 'boot'].map((origin) =>
|
||||||
Boolean(bridge.observeLegacyExecution(origin, () => fact(origin))),
|
Boolean(bridge.observeLegacyExecution(origin, () => fact(origin))),
|
||||||
);
|
);
|
||||||
process.stdout.write(JSON.stringify(result));
|
process.stdout.write(JSON.stringify(result));
|
||||||
process.exit(result.join(',') === 'true,true,true,false' ? 0 : 1);
|
process.exit(result.join(',') === 'true,true,true,true,false' ? 0 : 1);
|
||||||
`;
|
`;
|
||||||
const child = spawnSync(
|
const child = spawnSync(
|
||||||
process.execPath,
|
process.execPath,
|
||||||
@@ -69,7 +69,7 @@ test('admits the three reviewed origins through the default environment boundary
|
|||||||
cwd: path.resolve(__dirname, '../..'),
|
cwd: path.resolve(__dirname, '../..'),
|
||||||
env: {
|
env: {
|
||||||
...process.env,
|
...process.env,
|
||||||
QL3_SHADOW_ORIGINS: 'subscription,system,script',
|
QL3_SHADOW_ORIGINS: 'subscription,system,script,scheduled_system',
|
||||||
},
|
},
|
||||||
encoding: 'utf8',
|
encoding: 'utf8',
|
||||||
timeout: 10_000,
|
timeout: 10_000,
|
||||||
@@ -77,7 +77,7 @@ test('admits the three reviewed origins through the default environment boundary
|
|||||||
);
|
);
|
||||||
|
|
||||||
assert.equal(child.status, 0, child.stderr);
|
assert.equal(child.status, 0, child.stderr);
|
||||||
assert.equal(child.stdout, '[true,true,true,false]');
|
assert.equal(child.stdout, '[true,true,true,true,false]');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('observes subscription, system and script children without replacing legacy execution', async () => {
|
test('observes subscription, system and script children without replacing legacy execution', async () => {
|
||||||
|
|||||||
@@ -0,0 +1,384 @@
|
|||||||
|
require('ts-node/register/transpile-only');
|
||||||
|
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const fs = require('node:fs/promises');
|
||||||
|
const os = require('node:os');
|
||||||
|
const path = require('node:path');
|
||||||
|
const { spawnSync } = require('node:child_process');
|
||||||
|
const { test } = require('node:test');
|
||||||
|
const { Sequelize } = require('sequelize');
|
||||||
|
const config = require('../../back/config').default;
|
||||||
|
const { CrontabModel } = require('../../back/data/cron');
|
||||||
|
const {
|
||||||
|
defineSchemaMigrationModel,
|
||||||
|
} = require('../../back/data/schemaMigration');
|
||||||
|
const CronService = require('../../back/services/cron').default;
|
||||||
|
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||||
|
const {
|
||||||
|
runCancellationRequestMigration,
|
||||||
|
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||||
|
const {
|
||||||
|
runAttemptDeadlineMigration,
|
||||||
|
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||||
|
const { runMigrations } = require('../../back/migrations/runner');
|
||||||
|
const {
|
||||||
|
LegacySequelizeRunRepository,
|
||||||
|
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||||
|
const {
|
||||||
|
LegacyShadowRunObserver,
|
||||||
|
} = require('../../back/runtime/application/legacyShadowRunObserver');
|
||||||
|
const {
|
||||||
|
LegacyShadowRunWriter,
|
||||||
|
} = require('../../back/runtime/application/legacyShadowRunWriter');
|
||||||
|
const {
|
||||||
|
installLegacyExecutionObserver,
|
||||||
|
} = require('../../back/runtime/compatibility/legacyExecutionBridge');
|
||||||
|
const {
|
||||||
|
decorateScheduledSystemCronCommand,
|
||||||
|
observeLegacyScheduledSystemExecution,
|
||||||
|
parseLegacyScheduledSystemExecutionId,
|
||||||
|
} = require('../../back/runtime/compatibility/legacyScheduledSystemExecution');
|
||||||
|
const {
|
||||||
|
shadowOnlyRollout,
|
||||||
|
} = require('../../back/runtime/domain/runtimeRollout');
|
||||||
|
|
||||||
|
const EXECUTION_ID =
|
||||||
|
'legacy-system:1787004000:123e4567-e89b-42d3-a456-426614174000';
|
||||||
|
const SECOND_EXECUTION_ID =
|
||||||
|
'legacy-system:1787004060:123e4567-e89b-42d3-b456-426614174001';
|
||||||
|
|
||||||
|
function cron(overrides = {}) {
|
||||||
|
return {
|
||||||
|
id: 37,
|
||||||
|
name: 'system scheduled task',
|
||||||
|
command: 'task scripts/job.js',
|
||||||
|
schedule: '*/5 * * * *',
|
||||||
|
extra_schedules: [],
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function databaseFixture() {
|
||||||
|
const database = new Sequelize({
|
||||||
|
dialect: 'sqlite',
|
||||||
|
storage: ':memory:',
|
||||||
|
logging: false,
|
||||||
|
});
|
||||||
|
await runMigrations({
|
||||||
|
database,
|
||||||
|
migrationModel: defineSchemaMigrationModel(database),
|
||||||
|
migrations: [
|
||||||
|
runSchemaMigration,
|
||||||
|
runCancellationRequestMigration,
|
||||||
|
runAttemptDeadlineMigration,
|
||||||
|
],
|
||||||
|
logger: { info() {} },
|
||||||
|
});
|
||||||
|
return database;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runRows(database) {
|
||||||
|
const [rows] = await database.query(
|
||||||
|
'SELECT id FROM "Runs" ORDER BY created_at_ms, id',
|
||||||
|
);
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
function logger() {
|
||||||
|
return { info() {}, warn() {}, error() {} };
|
||||||
|
}
|
||||||
|
|
||||||
|
test('marks only system-crond commands and parses one bounded execution identity', () => {
|
||||||
|
const command = 'real_time=false no_tee=true ID=37 task scripts/job.js';
|
||||||
|
assert.equal(
|
||||||
|
decorateScheduledSystemCronCommand(command, true),
|
||||||
|
`QL_EXECUTION_ORIGIN=scheduled_system ${command}`,
|
||||||
|
);
|
||||||
|
assert.equal(decorateScheduledSystemCronCommand(command, false), command);
|
||||||
|
assert.deepEqual(parseLegacyScheduledSystemExecutionId(EXECUTION_ID), {
|
||||||
|
requestId: EXECUTION_ID,
|
||||||
|
acceptedAtMs: 1_787_004_000_000,
|
||||||
|
});
|
||||||
|
for (const invalid of [
|
||||||
|
'',
|
||||||
|
'legacy-system:0:123e4567-e89b-42d3-a456-426614174000',
|
||||||
|
'legacy-system:1787004000:123e4567-e89b-12d3-a456-426614174000',
|
||||||
|
'manual:1787004000:123e4567-e89b-42d3-a456-426614174000',
|
||||||
|
]) {
|
||||||
|
assert.equal(parseLegacyScheduledSystemExecutionId(invalid), undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Shell creates the identity only for an explicitly marked system execution', () => {
|
||||||
|
const script = `
|
||||||
|
. ./shell/api.sh
|
||||||
|
first=$(create_legacy_system_execution_id 1787004000)
|
||||||
|
QL_EXECUTION_ORIGIN=scheduled_system
|
||||||
|
second=$(create_legacy_system_execution_id 1787004000)
|
||||||
|
printf '%s\\n%s' "$first" "$second"
|
||||||
|
`;
|
||||||
|
const child = spawnSync('bash', ['-c', script], {
|
||||||
|
cwd: path.resolve(__dirname, '../..'),
|
||||||
|
encoding: 'utf8',
|
||||||
|
timeout: 10_000,
|
||||||
|
});
|
||||||
|
assert.equal(child.status, 0, child.stderr);
|
||||||
|
const [unmarked, marked] = child.stdout.split('\n');
|
||||||
|
assert.equal(unmarked, '');
|
||||||
|
assert.match(
|
||||||
|
marked,
|
||||||
|
/^legacy-system:1787004000:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Shell sends the same execution identity as bounded callback JSON', async () => {
|
||||||
|
const temporary = await fs.mkdtemp(
|
||||||
|
path.join(os.tmpdir(), 'ql3-system-callback-'),
|
||||||
|
);
|
||||||
|
const capture = path.join(temporary, 'curl-arguments');
|
||||||
|
const script = `
|
||||||
|
. ./shell/api.sh
|
||||||
|
curl() {
|
||||||
|
printf '%s\\n' "$@" > "$CALLBACK_CAPTURE"
|
||||||
|
printf '{"code":200}'
|
||||||
|
}
|
||||||
|
jq() {
|
||||||
|
if [[ "$*" == *'.code'* ]]; then printf '200'; fi
|
||||||
|
}
|
||||||
|
update_cron '"37"' '0' '4321' 'task/run.log' '1787004000' '' '' "$EXECUTION_ID"
|
||||||
|
`;
|
||||||
|
try {
|
||||||
|
const child = spawnSync('bash', ['-c', script], {
|
||||||
|
cwd: path.resolve(__dirname, '../..'),
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
CALLBACK_CAPTURE: capture,
|
||||||
|
EXECUTION_ID,
|
||||||
|
},
|
||||||
|
encoding: 'utf8',
|
||||||
|
timeout: 10_000,
|
||||||
|
});
|
||||||
|
assert.equal(child.status, 0, child.stderr);
|
||||||
|
const args = (await fs.readFile(capture, 'utf8')).split('\n');
|
||||||
|
const dataIndex = args.indexOf('--data-raw');
|
||||||
|
assert.equal(dataIndex >= 0, true);
|
||||||
|
assert.deepEqual(JSON.parse(args[dataIndex + 1]), {
|
||||||
|
ids: ['37'],
|
||||||
|
status: '0',
|
||||||
|
pid: '4321',
|
||||||
|
log_path: 'task/run.log',
|
||||||
|
last_execution_time: 1_787_004_000,
|
||||||
|
last_running_time: 0,
|
||||||
|
execution_id: EXECUTION_ID,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
await fs.rm(temporary, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('CronService marks the real system crontab file but not the node scheduler file', async () => {
|
||||||
|
const temporary = await fs.mkdtemp(
|
||||||
|
path.join(os.tmpdir(), 'ql3-system-crontab-'),
|
||||||
|
);
|
||||||
|
const executableDirectory = path.join(temporary, 'bin');
|
||||||
|
const crontabExecutable = path.join(executableDirectory, 'crontab');
|
||||||
|
const capture = path.join(temporary, 'crontab-call');
|
||||||
|
const crontabFile = path.join(temporary, 'crontab.list');
|
||||||
|
const previous = {
|
||||||
|
crontabFile: config.crontabFile,
|
||||||
|
path: process.env.PATH,
|
||||||
|
scheduler: process.env.QL_SCHEDULER,
|
||||||
|
update: CrontabModel.update,
|
||||||
|
};
|
||||||
|
await fs.mkdir(executableDirectory);
|
||||||
|
await fs.writeFile(
|
||||||
|
crontabExecutable,
|
||||||
|
'#!/bin/sh\nprintf "%s" "$1" > "$CRONTAB_CAPTURE"\n',
|
||||||
|
{ mode: 0o755 },
|
||||||
|
);
|
||||||
|
config.crontabFile = crontabFile;
|
||||||
|
process.env.PATH = `${executableDirectory}:${process.env.PATH ?? ''}`;
|
||||||
|
process.env.CRONTAB_CAPTURE = capture;
|
||||||
|
CrontabModel.update = async () => [0];
|
||||||
|
try {
|
||||||
|
const service = new CronService(logger());
|
||||||
|
process.env.QL_SCHEDULER = 'system';
|
||||||
|
await service.setCrontab({ data: [cron()], total: 1 });
|
||||||
|
const systemContent = await fs.readFile(crontabFile, 'utf8');
|
||||||
|
assert.match(
|
||||||
|
systemContent,
|
||||||
|
/^\*\/5 \* \* \* \* QL_EXECUTION_ORIGIN=scheduled_system /u,
|
||||||
|
);
|
||||||
|
assert.equal(await fs.readFile(capture, 'utf8'), crontabFile);
|
||||||
|
|
||||||
|
process.env.QL_SCHEDULER = 'node';
|
||||||
|
await service.setCrontab({ data: [cron()], total: 1 });
|
||||||
|
const nodeContent = await fs.readFile(crontabFile, 'utf8');
|
||||||
|
assert.equal(nodeContent.includes('QL_EXECUTION_ORIGIN='), false);
|
||||||
|
} finally {
|
||||||
|
config.crontabFile = previous.crontabFile;
|
||||||
|
process.env.PATH = previous.path;
|
||||||
|
CrontabModel.update = previous.update;
|
||||||
|
delete process.env.CRONTAB_CAPTURE;
|
||||||
|
if (previous.scheduler === undefined) {
|
||||||
|
delete process.env.QL_SCHEDULER;
|
||||||
|
} else {
|
||||||
|
process.env.QL_SCHEDULER = previous.scheduler;
|
||||||
|
}
|
||||||
|
await fs.rm(temporary, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('running, finish and response-loss replay converge to one durable aggregate', async () => {
|
||||||
|
const database = await databaseFixture();
|
||||||
|
const repository = new LegacySequelizeRunRepository(database);
|
||||||
|
const failures = [];
|
||||||
|
const observer = new LegacyShadowRunObserver(
|
||||||
|
shadowOnlyRollout(['scheduled_system']),
|
||||||
|
new LegacyShadowRunWriter(repository),
|
||||||
|
{ failure: (failure) => failures.push(failure) },
|
||||||
|
);
|
||||||
|
const restore = installLegacyExecutionObserver(observer, [
|
||||||
|
'scheduled_system',
|
||||||
|
]);
|
||||||
|
try {
|
||||||
|
const running = observeLegacyScheduledSystemExecution(cron(), {
|
||||||
|
executionId: EXECUTION_ID,
|
||||||
|
phase: 'running',
|
||||||
|
observedAtMs: 1_787_004_001_000,
|
||||||
|
pid: 4321,
|
||||||
|
logPath: 'task/run.log',
|
||||||
|
});
|
||||||
|
await running.settled();
|
||||||
|
const finished = observeLegacyScheduledSystemExecution(cron(), {
|
||||||
|
executionId: EXECUTION_ID,
|
||||||
|
phase: 'finished',
|
||||||
|
observedAtMs: 1_787_004_005_000,
|
||||||
|
pid: 4321,
|
||||||
|
logPath: 'task/run.log',
|
||||||
|
exitCode: 0,
|
||||||
|
});
|
||||||
|
await finished.settled();
|
||||||
|
const replay = observeLegacyScheduledSystemExecution(cron(), {
|
||||||
|
executionId: EXECUTION_ID,
|
||||||
|
phase: 'finished',
|
||||||
|
observedAtMs: 1_787_004_005_000,
|
||||||
|
pid: 4321,
|
||||||
|
logPath: 'task/run.log',
|
||||||
|
exitCode: 0,
|
||||||
|
});
|
||||||
|
await replay.settled();
|
||||||
|
|
||||||
|
const rows = await runRows(database);
|
||||||
|
assert.equal(rows.length, 1);
|
||||||
|
assert.match(
|
||||||
|
rows[0].id,
|
||||||
|
/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u,
|
||||||
|
);
|
||||||
|
const run = await repository.findRunById(rows[0].id);
|
||||||
|
const attempt = await repository.findLatestAttemptByRunId(rows[0].id);
|
||||||
|
const events = await repository.listEvents(rows[0].id);
|
||||||
|
assert.deepEqual(failures, []);
|
||||||
|
assert.equal(run.executionOrigin, 'scheduled_system');
|
||||||
|
assert.equal(run.executionOwner, 'legacy');
|
||||||
|
assert.equal(run.requestId, EXECUTION_ID);
|
||||||
|
assert.equal(
|
||||||
|
run.idempotencyKey,
|
||||||
|
`legacy-shadow:scheduled_system:${EXECUTION_ID}`,
|
||||||
|
);
|
||||||
|
assert.equal(run.createdAtMs, 1_787_004_000_000);
|
||||||
|
assert.equal(run.status, 'succeeded');
|
||||||
|
assert.equal(attempt.status, 'succeeded');
|
||||||
|
assert.equal(attempt.pid, 4321);
|
||||||
|
assert.equal(events.length, 8);
|
||||||
|
assert.deepEqual(
|
||||||
|
events.map((event) => event.sequence),
|
||||||
|
[1, 2, 3, 4, 5, 6, 7, 8],
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
restore();
|
||||||
|
await database.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('finish-only delivery creates one terminal Run and exact replay stays idempotent', async () => {
|
||||||
|
const database = await databaseFixture();
|
||||||
|
const repository = new LegacySequelizeRunRepository(database);
|
||||||
|
const failures = [];
|
||||||
|
const restore = installLegacyExecutionObserver(
|
||||||
|
new LegacyShadowRunObserver(
|
||||||
|
shadowOnlyRollout(['scheduled_system']),
|
||||||
|
new LegacyShadowRunWriter(repository),
|
||||||
|
{ failure: (failure) => failures.push(failure) },
|
||||||
|
),
|
||||||
|
['scheduled_system'],
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
for (let replay = 0; replay < 2; replay += 1) {
|
||||||
|
const observation = observeLegacyScheduledSystemExecution(cron(), {
|
||||||
|
executionId: SECOND_EXECUTION_ID,
|
||||||
|
phase: 'finished',
|
||||||
|
observedAtMs: 1_787_004_065_000,
|
||||||
|
pid: 4322,
|
||||||
|
exitCode: 17,
|
||||||
|
});
|
||||||
|
await observation.settled();
|
||||||
|
}
|
||||||
|
const rows = await runRows(database);
|
||||||
|
assert.equal(rows.length, 1);
|
||||||
|
const run = await repository.findRunById(rows[0].id);
|
||||||
|
const attempt = await repository.findLatestAttemptByRunId(rows[0].id);
|
||||||
|
assert.equal(run.status, 'failed');
|
||||||
|
assert.equal(attempt.status, 'failed');
|
||||||
|
assert.equal(attempt.exitCode, 17);
|
||||||
|
assert.deepEqual(failures, []);
|
||||||
|
} finally {
|
||||||
|
restore();
|
||||||
|
await database.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('same execution identity with changed task facts fails open without a second Run', async () => {
|
||||||
|
const database = await databaseFixture();
|
||||||
|
const repository = new LegacySequelizeRunRepository(database);
|
||||||
|
const failures = [];
|
||||||
|
const restore = installLegacyExecutionObserver(
|
||||||
|
new LegacyShadowRunObserver(
|
||||||
|
shadowOnlyRollout(['scheduled_system']),
|
||||||
|
new LegacyShadowRunWriter(repository),
|
||||||
|
{ failure: (failure) => failures.push(failure) },
|
||||||
|
),
|
||||||
|
['scheduled_system'],
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
const first = observeLegacyScheduledSystemExecution(cron(), {
|
||||||
|
executionId: EXECUTION_ID,
|
||||||
|
phase: 'finished',
|
||||||
|
observedAtMs: 1_787_004_005_000,
|
||||||
|
exitCode: 0,
|
||||||
|
});
|
||||||
|
await first.settled();
|
||||||
|
const drifted = observeLegacyScheduledSystemExecution(
|
||||||
|
cron({ command: 'task scripts/changed.js' }),
|
||||||
|
{
|
||||||
|
executionId: EXECUTION_ID,
|
||||||
|
phase: 'finished',
|
||||||
|
observedAtMs: 1_787_004_005_000,
|
||||||
|
exitCode: 0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
await drifted.settled();
|
||||||
|
assert.equal((await runRows(database)).length, 1);
|
||||||
|
assert.equal(
|
||||||
|
failures.some(
|
||||||
|
(failure) =>
|
||||||
|
failure.origin === 'scheduled_system' &&
|
||||||
|
failure.operation === 'accept',
|
||||||
|
),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
restore();
|
||||||
|
await database.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user