feat(ql3): observe system crond runs idempotently

2026-09-20 16:07:11 +08:00 · 2026-08-18 06:33:23 +08:00
parent 6831ea3de5
commit 0ad96d38a6
16 changed files with 858 additions and 73 deletions
@@ -5,10 +5,7 @@ import CronService from '../services/cron';
 import CronViewService from '../services/cronView';
 import { celebrate, Joi } from 'celebrate';
 import { commonCronSchema } from '../validation/schedule';
-import {
-  RunningInstanceModel,
-  InstanceStatus,
-} from '../data/runningInstance';
+import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
 import { t } from '../shared/i18n';

 const route = Router();
@@ -313,7 +310,11 @@ export default (app: Router) => {
      try {
        const cronService = Container.get(CronService);
        const result = await cronService.log(req.params.id);
-        return res.send({ code: 200, data: result.content, logStatus: result.status });
+        return res.send({
+          code: 200,
+          data: result.content,
+          logStatus: result.status,
+        });
      } catch (e) {
        return next(e);
      }
@@ -435,6 +436,11 @@ export default (app: Router) => {
        last_running_time: Joi.number().optional().allow(null),
        last_execution_time: Joi.number().optional().allow(null),
        exit_code: Joi.number().optional().allow(null),
+        execution_id: Joi.string()
+          .pattern(
+            /^legacy-system:[1-9][0-9]{0,12}:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u,
+          )
+          .optional(),
      }),
    }),
    async (req: Request, res: Response, next: NextFunction) => {
@@ -483,7 +489,11 @@ export default (app: Router) => {
        instanceId: Joi.number().required(),
      }),
    }),
-    async (req: Request<{ id: number; instanceId: number }>, res: Response, next: NextFunction) => {
+    async (
+      req: Request<{ id: number; instanceId: number }>,
+      res: Response,
+      next: NextFunction,
+    ) => {
      try {
        const cronService = Container.get(CronService);
        const data = await cronService.stopInstance(req.params.instanceId);
@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import { v7 as uuidV7 } from 'uuid';
 import type {
  RunAttemptRecord,
@@ -43,6 +44,38 @@ export interface LegacyShadowRunReference {

 export type ShadowIdFactory = () => string;

+function deterministicShadowId(
+  requestId: string,
+  acceptedAtMs: number,
+  domain: 'run' | 'attempt',
+): string {
+  if (
+    !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u.test(requestId) ||
+    !Number.isSafeInteger(acceptedAtMs) ||
+    acceptedAtMs < 0 ||
+    acceptedAtMs > 0xffffffffffff
+  ) {
+    throw new TypeError('Invalid deterministic Legacy Shadow identity');
+  }
+  const bytes = createHash('sha256')
+    .update(`qinglong:legacy-shadow:${domain}:`)
+    .update(requestId)
+    .digest()
+    .subarray(0, 16);
+  let timestamp = acceptedAtMs;
+  for (let index = 5; index >= 0; index -= 1) {
+    bytes[index] = timestamp & 0xff;
+    timestamp = Math.floor(timestamp / 256);
+  }
+  bytes[6] = (bytes[6] & 0x0f) | 0x70;
+  bytes[8] = (bytes[8] & 0x3f) | 0x80;
+  const value = bytes.toString('hex');
+  return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(
+    12,
+    16,
+  )}-${value.slice(16, 20)}-${value.slice(20)}`;
+}
+
 export class LegacyShadowRunWriter {
  constructor(
    private readonly repository: RunRepository,
@@ -52,10 +85,25 @@ export class LegacyShadowRunWriter {
  async accept(
    fact: LegacyExecutionAcceptedFact,
  ): Promise<LegacyShadowRunReference> {
-    const reference = {
-      runId: this.createId(),
-      attemptId: this.createId(),
-    };
+    const reference =
+      fact.requestId === undefined
+        ? { runId: this.createId(), attemptId: this.createId() }
+        : {
+            runId: deterministicShadowId(
+              fact.requestId,
+              fact.acceptedAtMs,
+              'run',
+            ),
+            attemptId: deterministicShadowId(
+              fact.requestId,
+              fact.acceptedAtMs,
+              'attempt',
+            ),
+          };
+    const idempotencyKey =
+      fact.requestId === undefined
+        ? undefined
+        : `legacy-shadow:${fact.origin}:${fact.requestId}`;
    const initialRun: RunRecord = {
      id: reference.runId,
      projectId: fact.projectId,
@@ -79,6 +127,7 @@ export class LegacyShadowRunWriter {
      version: 0,
      eventSequence: 0,
      priority: 0,
+      ...(idempotencyKey === undefined ? {} : { idempotencyKey }),
      createdAtMs: fact.acceptedAtMs,
    };
    const initialAttempt: RunAttemptRecord = {
@@ -91,48 +140,100 @@ export class LegacyShadowRunWriter {
      createdAtMs: fact.acceptedAtMs,
    };

-    await this.repository.transaction(async (transaction) => {
-      await transaction.insertRun(initialRun);
-      await transaction.insertAttempt(initialAttempt);
+    try {
+      await this.repository.transaction(async (transaction) => {
+        await transaction.insertRun(initialRun);
+        await transaction.insertAttempt(initialAttempt);

-      const created = reserveRunEvent(initialRun, 0);
-      const createdRun = created.run;
-      const createdUpdated = await transaction.compareAndSetRun(createdRun, 0);
-      if (!createdUpdated) {
-        throw new RunVersionConflictError(initialRun.id, 0, initialRun.version);
-      }
-      await transaction.appendEvent(
-        this.event(
+        const created = reserveRunEvent(initialRun, 0);
+        const createdRun = created.run;
+        const createdUpdated = await transaction.compareAndSetRun(
          createdRun,
-          {
-            sequence: created.sequence,
-            type: 'run.created',
-            payload: {
-              status: 'created',
-              version: createdRun.version,
-              execution_owner: 'legacy',
-              shadow: true,
+          0,
+        );
+        if (!createdUpdated) {
+          throw new RunVersionConflictError(
+            initialRun.id,
+            0,
+            initialRun.version,
+          );
+        }
+        await transaction.appendEvent(
+          this.event(
+            createdRun,
+            {
+              sequence: created.sequence,
+              type: 'run.created',
+              payload: {
+                status: 'created',
+                version: createdRun.version,
+                execution_owner: 'legacy',
+                shadow: true,
+              },
            },
-          },
-          fact.acceptedAtMs,
-        ),
-      );
+            fact.acceptedAtMs,
+          ),
+        );

-      const queued = transitionRun(createdRun, {
-        to: 'queued',
-        expectedVersion: createdRun.version,
-        atMs: fact.acceptedAtMs,
+        const queued = transitionRun(createdRun, {
+          to: 'queued',
+          expectedVersion: createdRun.version,
+          atMs: fact.acceptedAtMs,
+        });
+        await this.persistRunDecision(
+          transaction,
+          createdRun,
+          queued,
+          fact.acceptedAtMs,
+        );
      });
-      await this.persistRunDecision(
-        transaction,
-        createdRun,
-        queued,
-        fact.acceptedAtMs,
-      );
-    });
+    } catch (error) {
+      if (
+        fact.requestId === undefined ||
+        !(await this.isExactReplay(
+          reference,
+          initialRun,
+          initialAttempt,
+          idempotencyKey!,
+        ))
+      ) {
+        throw error;
+      }
+    }
    return reference;
  }

+  private async isExactReplay(
+    reference: LegacyShadowRunReference,
+    expectedRun: RunRecord,
+    expectedAttempt: RunAttemptRecord,
+    idempotencyKey: string,
+  ): Promise<boolean> {
+    const [run, attempt] = await Promise.all([
+      this.repository.findRunById(reference.runId),
+      this.repository.findAttemptById(reference.attemptId),
+    ]);
+    if (!run || !attempt) return false;
+    return (
+      run.projectId === expectedRun.projectId &&
+      run.taskId === expectedRun.taskId &&
+      run.taskRevision === expectedRun.taskRevision &&
+      run.taskName === expectedRun.taskName &&
+      run.legacyCronId === expectedRun.legacyCronId &&
+      run.triggerType === expectedRun.triggerType &&
+      run.executionOrigin === expectedRun.executionOrigin &&
+      run.executionOwner === 'legacy' &&
+      run.triggeredBy === expectedRun.triggeredBy &&
+      run.requestId === expectedRun.requestId &&
+      run.idempotencyKey === idempotencyKey &&
+      run.createdAtMs === expectedRun.createdAtMs &&
+      attempt.runId === run.id &&
+      attempt.attempt === 1 &&
+      attempt.executorType === 'legacy_local' &&
+      attempt.createdAtMs === expectedAttempt.createdAtMs
+    );
+  }
+
  async spawned(
    reference: LegacyShadowRunReference,
    fact: LegacyExecutionSpawnedFact,
@@ -22,6 +22,7 @@ const SHADOW_ORIGINS_ENV = 'QL3_SHADOW_ORIGINS';
 const SUPPORTED_SHADOW_ORIGINS = new Set<ExecutionOrigin>([
  'manual',
  'scheduled_node',
+  'scheduled_system',
  'script',
  'subscription',
  'system',
@@ -84,7 +85,7 @@ function readConfiguredOrigins(): ReadonlySet<ExecutionOrigin> {
        incrementFailure('configuration:unsupported_origin');
        try {
          Logger.warn(
-            '[ql3-shadow] ignored unsupported origin; this slice supports manual,scheduled_node,script,subscription,system',
+            '[ql3-shadow] ignored unsupported origin; this slice supports manual,scheduled_node,scheduled_system,script,subscription,system',
          );
        } catch {
          // Invalid compatibility configuration must not affect legacy paths.
@@ -269,6 +270,57 @@ export function observeLegacyExecution(
    : NOOP_OBSERVATION;
 }

+export interface LegacyShellExecutionCallbackInput {
+  pid?: number;
+  logPath?: string;
+  atMs: number;
+  phase: 'running' | 'finished';
+  exitCode?: number;
+}
+
+/**
+ * Observes one callback-owned execution without registering an in-memory
+ * ChildProcess correlation entry. The accepted fact must carry a durable
+ * requestId so callback replay can converge in the Shadow writer.
+ */
+export function observeLegacyShellExecutionCallback(
+  createFact: LegacyExecutionAcceptedFactFactory,
+  input: LegacyShellExecutionCallbackInput,
+): LegacyExecutionObservation | undefined {
+  const origin: ExecutionOrigin = 'scheduled_system';
+  let observation: LegacyExecutionObservation;
+  if (override) {
+    if (!override.origins.has(origin)) return undefined;
+    const fact = createAcceptedFactFailOpen(origin, createFact);
+    observation = fact
+      ? beginFailOpen(override.observer, fact)
+      : NOOP_OBSERVATION;
+  } else {
+    if (!readConfiguredOrigins().has(origin)) return undefined;
+    const fact = createAcceptedFactFailOpen(origin, createFact);
+    observation = fact
+      ? deferredObservation(getDefaultObserver(), fact)
+      : NOOP_OBSERVATION;
+  }
+
+  observation.spawned({
+    atMs: input.atMs,
+    ...(input.pid === undefined ? {} : { pid: input.pid }),
+    ...(input.logPath === undefined
+      ? {}
+      : { logArtifactId: createLegacyLogArtifactId(input.logPath) }),
+  });
+  if (input.phase === 'running') {
+    observation.running({ atMs: input.atMs });
+  } else {
+    observation.exited({
+      atMs: input.atMs,
+      exitCode: input.exitCode ?? 0,
+    });
+  }
+  return observation;
+}
+
 export interface LegacyExecutionCancellationInput {
  legacyCronId: number;
  pid?: number;
@@ -0,0 +1,87 @@
+import type { Crontab } from '../../data/cron';
+import type { LegacyExecutionObservation } from '../ports/legacyExecutionObserver';
+import { observeLegacyShellExecutionCallback } from './legacyExecutionBridge';
+import { createLegacyTaskRevision } from './legacyTaskRevision';
+
+const EXECUTION_ID_PATTERN =
+  /^legacy-system:([1-9][0-9]{0,12}):([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/u;
+
+export interface LegacyScheduledSystemCallbackInput {
+  executionId: string;
+  phase: 'running' | 'finished';
+  observedAtMs: number;
+  pid?: number;
+  logPath?: string;
+  exitCode?: number;
+}
+
+export function decorateScheduledSystemCronCommand(
+  command: string,
+  systemScheduler: boolean,
+): string {
+  return systemScheduler
+    ? `QL_EXECUTION_ORIGIN=scheduled_system ${command}`
+    : command;
+}
+
+export function parseLegacyScheduledSystemExecutionId(
+  value: string,
+): { requestId: string; acceptedAtMs: number } | undefined {
+  const match = EXECUTION_ID_PATTERN.exec(value);
+  if (!match) return undefined;
+  const acceptedAtMs = Number(match[1]) * 1000;
+  if (!Number.isSafeInteger(acceptedAtMs) || acceptedAtMs < 1) return undefined;
+  return Object.freeze({ requestId: value, acceptedAtMs });
+}
+
+export function observeLegacyScheduledSystemExecution(
+  cron: Crontab,
+  input: LegacyScheduledSystemCallbackInput,
+): LegacyExecutionObservation | undefined {
+  const identity = parseLegacyScheduledSystemExecutionId(input.executionId);
+  if (
+    !identity ||
+    cron.id === undefined ||
+    !Number.isSafeInteger(input.observedAtMs) ||
+    input.observedAtMs < identity.acceptedAtMs
+  ) {
+    return undefined;
+  }
+  return observeLegacyShellExecutionCallback(
+    () => ({
+      origin: 'scheduled_system',
+      projectId: 'default',
+      taskId: `legacy-cron:${cron.id}`,
+      taskRevision: createLegacyTaskRevision({
+        command: cron.command,
+        ...(cron.schedule === undefined ? {} : { schedule: cron.schedule }),
+        extraSchedules:
+          cron.extra_schedules?.map((item) => item.schedule) ?? [],
+        ...(cron.task_before === undefined
+          ? {}
+          : { taskBefore: cron.task_before }),
+        ...(cron.task_after === undefined
+          ? {}
+          : { taskAfter: cron.task_after }),
+        ...(cron.work_dir === undefined
+          ? {}
+          : { workDirectory: cron.work_dir }),
+        ...(cron.log_name === undefined ? {} : { logName: cron.log_name }),
+      }),
+      ...(cron.name === undefined ? {} : { taskName: cron.name }),
+      legacyCronId: cron.id,
+      triggerType: 'scheduled_system',
+      triggeredBy: 'legacy:system-crond',
+      requestId: identity.requestId,
+      scheduledForMs: identity.acceptedAtMs,
+      acceptedAtMs: identity.acceptedAtMs,
+    }),
+    {
+      phase: input.phase,
+      atMs: input.observedAtMs,
+      ...(input.pid === undefined ? {} : { pid: input.pid }),
+      ...(input.logPath === undefined ? {} : { logPath: input.logPath }),
+      ...(input.exitCode === undefined ? {} : { exitCode: input.exitCode }),
+    },
+  );
+}
@@ -39,6 +39,10 @@ import {
  createLegacyLogArtifactId,
  createLegacyTaskRevision,
 } from '../runtime/compatibility/legacyTaskRevision';
+import {
+  decorateScheduledSystemCronCommand,
+  observeLegacyScheduledSystemExecution,
+} from '../runtime/compatibility/legacyScheduledSystemExecution';
 import {
  selectManualPrimaryExecutionRouter,
  stopManualPrimaryAttempt,
@@ -233,6 +237,7 @@ export default class CronService {
    last_running_time = 0,
    last_execution_time = 0,
    exit_code,
+    execution_id,
  }: {
    ids: number[];
    status: CrontabStatus;
@@ -241,6 +246,7 @@ export default class CronService {
    last_running_time: number;
    last_execution_time: number;
    exit_code?: number;
+    execution_id?: string;
  }) {
    let options: any = {
      status,
@@ -261,14 +267,26 @@ export default class CronService {
        continue;
      }
      if (status === CrontabStatus.running || status === CrontabStatus.idle) {
-        observeLegacyExecutionCallback({
-          legacyCronId: id,
-          ...(pid ? { pid } : {}),
-          ...(log_path ? { logPath: log_path } : {}),
-          atMs: Date.now(),
-          phase: status === CrontabStatus.running ? 'running' : 'finished',
-          ...(exit_code === undefined ? {} : { exitCode: exit_code }),
-        });
+        const observedAtMs = Date.now();
+        if (execution_id) {
+          observeLegacyScheduledSystemExecution(cron, {
+            executionId: execution_id,
+            ...(pid ? { pid } : {}),
+            ...(log_path ? { logPath: log_path } : {}),
+            observedAtMs,
+            phase: status === CrontabStatus.running ? 'running' : 'finished',
+            ...(exit_code === undefined ? {} : { exitCode: exit_code }),
+          });
+        } else {
+          observeLegacyExecutionCallback({
+            legacyCronId: id,
+            ...(pid ? { pid } : {}),
+            ...(log_path ? { logPath: log_path } : {}),
+            atMs: observedAtMs,
+            phase: status === CrontabStatus.running ? 'running' : 'finished',
+            ...(exit_code === undefined ? {} : { exitCode: exit_code }),
+          });
+        }
      }
      if (status === CrontabStatus.idle && log_path !== cron.log_path) {
        options = omit(options, ['status', 'log_path', 'pid']);
@@ -1046,6 +1064,7 @@ export default class CronService {

  private async setCrontab(data?: { data: Crontab[]; total: number }) {
    const tabs = data ?? (await this.crontabs());
+    const systemScheduler = this.schedulerMode === 'system';
    var crontab_string = '';
    tabs.data.forEach((tab) => {
      if (
@@ -1056,19 +1075,25 @@ export default class CronService {
        crontab_string += '# ';
        crontab_string += tab.schedule;
        crontab_string += ' ';
-        crontab_string += this.makeCommand(tab);
+        crontab_string += decorateScheduledSystemCronCommand(
+          this.makeCommand(tab),
+          systemScheduler,
+        );
        crontab_string += '\n';
      } else {
        crontab_string += tab.schedule;
        crontab_string += ' ';
-        crontab_string += this.makeCommand(tab);
+        crontab_string += decorateScheduledSystemCronCommand(
+          this.makeCommand(tab),
+          systemScheduler,
+        );
        crontab_string += '\n';
      }
    });

    await writeFileWithLock(config.crontabFile, crontab_string);

-    if (this.schedulerMode === 'system') {
+    if (systemScheduler) {
      try {
        execSync(`crontab ${config.crontabFile}`);
      } catch (error: any) {