From 6831ea3de5a7c09fd5eb754ae702b338a8228aab Mon Sep 17 00:00:00 2001 From: whyour Date: Tue, 18 Aug 2026 05:57:19 +0800 Subject: [PATCH] feat(ql3): observe legacy schedule service runs --- .../compatibility/legacyExecutionBridge.ts | 5 +- back/services/schedule.ts | 31 ++ docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 21 +- ...-crontab-compatibility-and-shadow-write.md | 9 +- ...dule-service-origin-shadow-run-coverage.md | 80 +++++ docs/adr/README.md | 1 + .../src/plugin-package/pluginPackageBundle.ts | 1 + .../pluginPackageResourceMaterialization.ts | 1 + .../back/legacyScheduleServiceShadow.test.cjs | 310 ++++++++++++++++++ 9 files changed, 453 insertions(+), 6 deletions(-) create mode 100644 docs/adr/ADR-0445-schedule-service-origin-shadow-run-coverage.md create mode 100644 test/back/legacyScheduleServiceShadow.test.cjs diff --git a/back/runtime/compatibility/legacyExecutionBridge.ts b/back/runtime/compatibility/legacyExecutionBridge.ts index a0af65ee..7adc03c4 100644 --- a/back/runtime/compatibility/legacyExecutionBridge.ts +++ b/back/runtime/compatibility/legacyExecutionBridge.ts @@ -22,6 +22,9 @@ const SHADOW_ORIGINS_ENV = 'QL3_SHADOW_ORIGINS'; const SUPPORTED_SHADOW_ORIGINS = new Set([ 'manual', 'scheduled_node', + 'script', + 'subscription', + 'system', ]); const NOOP_OBSERVATION: LegacyExecutionObservation = Object.freeze({ @@ -81,7 +84,7 @@ function readConfiguredOrigins(): ReadonlySet { incrementFailure('configuration:unsupported_origin'); try { Logger.warn( - '[ql3-shadow] ignored unsupported origin; this slice supports manual,scheduled_node', + '[ql3-shadow] ignored unsupported origin; this slice supports manual,scheduled_node,script,subscription,system', ); } catch { // Invalid compatibility configuration must not affect legacy paths. diff --git a/back/services/schedule.ts b/back/services/schedule.ts index df938674..2b2890a4 100644 --- a/back/services/schedule.ts +++ b/back/services/schedule.ts @@ -11,6 +11,10 @@ import { import dayjs from 'dayjs'; import taskLimit from '../shared/pLimit'; import { spawn } from 'cross-spawn'; +import { createHash } from 'crypto'; +import { observeLegacyExecution } from '../runtime/compatibility/legacyExecutionBridge'; +import { observeLegacyChildProcess } from '../runtime/compatibility/observeLegacyChildProcess'; +import { createLegacyTaskRevision } from '../runtime/compatibility/legacyTaskRevision'; export interface ScheduleTaskType { id?: number; @@ -35,6 +39,14 @@ export interface TaskCallbacks { onError?: (message: string) => Promise; } +function opaqueLegacyTaskId( + runOrigin: ScheduleTaskType['runOrigin'], + id: string, +): string { + const digest = createHash('sha256').update(id).digest('hex').slice(0, 25); + return `legacy-schedule:${runOrigin}:${digest}`; +} + @Service() export default class ScheduleService { private scheduleStacks = new Map(); @@ -76,7 +88,26 @@ export default class ScheduleService { const startTime = dayjs(); await callbacks.onBefore?.(startTime); + const observation = observeLegacyExecution(runOrigin, () => { + const taskRevision = createLegacyTaskRevision({ + command: others.command ?? command, + ...(others.schedule === undefined + ? {} + : { schedule: others.schedule }), + }); + return { + origin: runOrigin, + projectId: 'default', + taskId: opaqueLegacyTaskId(runOrigin, others.id), + taskRevision, + ...(others.name === undefined ? {} : { taskName: others.name }), + triggerType: runOrigin, + triggeredBy: 'legacy:schedule-service', + acceptedAtMs: Date.now(), + }; + }); const cp = spawn(command, { shell: '/bin/bash' }); + if (observation) observeLegacyChildProcess(cp, observation); callbacks.onStart?.(cp, startTime); completionTime === 'start' && resolve(cp.pid); diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index c692e4fa..03ca8957 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,23 @@ 最新增量证据（2026-08-18）： +- D-353/ADR-0445（已接受；`scheduled_system` 的 response-loss-safe 幂等准入待独立 Gate）：3.0 Shadow Run 不再只覆盖 + `manual/scheduled_node`。现有 `ScheduleService.runTask` 在既有 task limit 选中且 `onBefore` 成功后，可按默认关闭的 + `QL3_SHADOW_ORIGINS` 精确旁路观察 `subscription/system/script`；观察器只附着到 Legacy 已创建的同一个 ChildProcess，不调用 + Executor、不第二次 spawn、不改变 callback、PID 或完成结果。Run 继续固定 `executionOwner=legacy`，task revision 只摘要 command/schedule， + task ID 使用 origin-scoped SHA-256 摘要而不复制 Script 路径、System command 或 caller ID 原文。origin 未启用时 factory 不执行，不计算摘要、 + 不加载 Repository、不附加 listener、不写数据库；observer 初始化或写入失败仍为 no-op。真实子进程与 SQLite 集成门已证明三类 origin、Script + start-completion、失败开放和一个完整 succeeded Run/Attempt/八 Event 聚合。阶段门已重跑：专项 27/27、`build:back`、完整 backend + 1,408 pass + 2 条条件 skip/0 fail、18-package clean build/test、14/14 静态审计与 14/14 artifact 档位均通过。artifact 字节为基础 + Edge/Standalone `2589998/2590076`、adopted `2809293/2809416`、application `3632877/3632997`、application-api + `3800430/3800574`、AI `3069251/3069341`、application+AI `4493151/4493283`、MCP `7315930/7316038`；包边界审计同时证明 18 + 个包的根 `src` 只保留公共 `index.ts`，其余实现均在领域子目录，没有以继续拆包掩盖平铺问题。本切片不新增 package、生产依赖、schema、 + migration、SQL、timer、watcher、端口、Kubernetes object 或 Cluster 连接；为修复既存 `build:back` 类型闭包， + runtime-core 两处只把全局 `TextDecoder` 改为显式 `node:util` import，不改变 UTF-8 运行语义。system crond 只有可重复/乱序的 Shell status + callback，尚无稳定 accept identity，因此本门拒绝从结束事实伪造 `scheduled_system` Run，也不开放任何 Primary origin。由于数据库模式、 + PostgreSQL adapter、容器与 Kubernetes 拓扑均未变化，本阶段没有重新运行物理 PostgreSQL HA 或 K3s 门，也不把 D-351/D-352 的相邻基线冒充 + 新证据。 + - D-352/ADR-0444（已接受；首份真实 GHCR response-loss 重放待实际 release tag）：修复 D-351 终态 finalizer 中仍残留的 mutation-before-validation 风险。原 workflow 内联 Node publisher 直接解析 plan 并执行 registry copy，直到生成 observation 时才间接验证 plan self-digest；篡改 plan 可能先产生错误副作用。promotion 现唯一进入 `scripts/ql3-release-tag-finalizer.cjs`：只读取 current-user @@ -9038,7 +9055,7 @@ flowchart LR | PR-1 Run Schema | Incubating | Run/RunAttempt/RunEvent schema、nullable cancel request 与 Attempt deadline 字段及恢复索引、CancellationDispatch 状态/version/lease/backoff schema、Repository port、临时 Sequelize adapter、统一事件大小/分页上限、跨 adapter RunRepository contract suite（原子事务、回滚、Run/Attempt/RetryPolicy CAS、唯一错误、分页与取消恢复）；ADR-0041 的 `pg-0003-run-retry-policy`、capability v2、driver-neutral PostgreSQL Run Repository 与真实 `pg.Pool` 上的共享 Repository/rollback/SQLSTATE contract；ADR-0063/0069/0071/0073/0074/0076 的独立 Node 24 local-sqlite typed schema、十二条 reviewed migration、capability v6、共享 operation authority、readiness/RunRepository/API credential repository/receipt journal/dispatch plan/encrypted Secret envelope/Project Policy/security audit/authorized mutation/stable Identity catalog、Drizzle↔真实 catalog table/column/index/CHECK/FK lockstep、base/adopted/application edge/standalone 产物门禁；ADR-0064 的 legacy baseline/plan digest、Online Backup recovery、side-by-side target migration、staged manifest、双库栅栏 activation、source 生命周期写栅栏、target stable identity 和重启语义；ADR-0065 的独立 cutover authority、外部副作用停机 evidence、append-only journal、start/restart/stop barrier 与 unknown→manual_required 收敛；ADR-0066 的 adopted storage→Run reconciliation→receipt maintenance→domain recovery→lifecycle→admission application gate、严格有界 recovery summary 与 admission-first reverse stop；ADR-0067 的 SQLite 事实驱动 Run 候选源、256 条硬上限、截断失败关闭和唯一 Repository authority；ADR-0068 的 receipt-first Reconciler、callback token/sequence fence、exact local-process identity、Attempt/Run/双 Event 原子终态推进和最终 verifier；ADR-0069 的 local-process 单向包边界、pre-spawn journal、受审 POSIX launcher、immutable receipt、exact identity 和 Profile-aware cleanup lifecycle；ADR-0070 的独立 local-execution、spawn 前后双 transaction CAS、callback digest、exact stop 补偿与 fail-closed starting 保留；ADR-0071 的独立 local-dispatch、不可变 revision/context、Secret-first materializer、Profile Artifact admission、4/64 MiB output hard quota 和窄 application facade；ADR-0073/0074 的 Project-bound SecretRef、AES-256-GCM、外置 keyring 生命周期、双 SQLite authority CAS、application preflight、强 Principal/Policy 和 envelope+audit 原子提交；ADR-0086 的本机 Owner provisioning/challenge/claim/delivery acknowledgement/credential recovery CLI；ADR-0377 的 Local/Cluster 同构、Profile-aware、Project-scoped Artifact range read | fresh database/pepper setup、credential rotation/GC 运维编排与 Secret/Project/Role/Approval 管理 CLI/API/UI、备份/rekey、2.x/target process controller、人工 recovery、target 写后 reconciliation 与完整 cutover/rollback 演练；retry 产品策略、Artifact retention/tombstone stack、具体本机 lifecycle 和 target executable；Linux x64/arm64、PID namespace、断电与固定路由设备门禁；PostgreSQL 16/18 双连接并发与 failover integration；Task revision/context 跨方言 contract/并发压力与引用感知 retention、Keyv 数据迁移 | | PR-2 Run 状态机 | Incubating | 纯转换表、终态/时间/错误/执行器元数据规则、Run version 与 event sequence CAS、事务性 RunCommandService、回滚测试 | 重复 Worker callback/fencing、并发数据库压力测试、Primary 执行链接入 | | PR-3 Executor 端口 | Incubating | ADR-0003、ExecutionSpec/Context/Handle/Result、Executor port、LocalProcessExecutor、进程组取消/超时升级、流式背压、Legacy Cron spec builder、真实进程 contract tests、可复现 edge 基准入口 | 固定 edge/多架构设备基线、Legacy builder 与 makeCommand 差异审计、Primary 生产流量接入 | -| PR-4 Shadow Run | Incubating | origin 三态策略；默认关闭的 `QL3_SHADOW_ORIGINS`；manual 与 scheduled_node 现有 ChildProcess 旁路观察；每个 worker 懒加载；Run/Attempt/Event 影子生命周期；稳定 task revision 与有界日志引用；同 worker 有界注册表和跨 worker 持久化候选关联；stop all/stop instance、Shell callback、乱序/迟到/歧义处理；失败开放和契约测试 | `ScheduleService.runTask` 覆盖的 scheduled_system/subscription/system/script 等高风险入口；启动后 Reconciler、差异报表、可采集指标、资源压力、回滚演练和 Primary 门禁 | +| PR-4 Shadow Run | Incubating | origin 三态策略；默认关闭的 `QL3_SHADOW_ORIGINS`；manual、scheduled_node、subscription、system 与 script 现有 ChildProcess 旁路观察；每个 worker 懒加载；Run/Attempt/Event 影子生命周期；稳定且不复制 caller 原文的 task identity/revision 与有界日志引用；同 worker 有界注册表和跨 worker 持久化候选关联；stop all/stop instance、Shell callback、乱序/迟到/歧义处理；失败开放和契约测试 | system crond `scheduled_system` 的 response-loss-safe accept identity/幂等准入；once/boot/grpc 独立裁决；启动后 Reconciler、差异报表、可采集指标、资源压力、回滚演练和 Primary 门禁 | | PR-5 Primary LocalExecutor | Incubating（默认不激活，仅 manifest-gated manual） | runtime-owned Run 创建器；持久化先于 spawn；Run/Attempt 完整成功、失败、取消、超时与 lost 闭环；Executor handle 身份校验；spawn 后激活写失败的 stop+lost 补偿；completion rejection 安全收敛；独立 Primary 幂等查询与唯一索引竞态裁决；durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询；最多 64 条一页的 cross-worker cancellation source；独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent；最多 64 页的单周期 cancel supervisor；显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner；Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller；完整有界分页且 fail-closed 的 startup Reconcile supervisor；RunningInstance nullable `run_id/attempt_id` 关联；Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance，失败整体回滚；有界且防穿越的 legacy log output ref；manual owner seam、真实本机装配、单 spawn/fail-closed；严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计；HTTP worker 已接轻量 lazy bootstrap，accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活，失败撤销，监听失败和 shutdown 有界停止；Primary timeout 在 spawn 前持久化绝对 deadline，有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch；代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限，cluster-control/worker 拒绝误装本机 SQLite Primary；统一 CompletionService 原子提交 Attempt/Run/双 Event，spawn 前保存 callback token hash、终态推进 sequence，实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window；manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理；Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace；`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描，使终态残留继续可发现；确定无效的已知 Attempt receipt 先持久化隔离状态，再进入确定性私有分片 quarantine；终态 missing 与 quarantine 按 edge/standalone retention 有界清理；非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI；扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数；ENOSPC 与 launcher receipt 存储失败有代码门禁 | 部署配置写入/审批入口与用户可见状态；PostgreSQL CancellationDispatch adapter；cluster-control 生产启动拓扑；固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 | | PR-7 Worker Session、Run Lease 与启动协议基础 | Incubating（默认关闭，独立入口显式 opt-in） | ADR-0012/0013/0014/0021/0057–0061/0108–0121/0231–0239/0377；有界 capability/Placement/Dispatcher；SQLite 协议孵化与 PostgreSQL v9 Session/Run Lease/credential/attestation authority；immutable revision Placement、数据库时钟 keyset candidate、认证 Worker Pull、digest-only offer recovery；versioned capability-free ExecutionSpec response、stable claim 跨重启退避、单 owner 原子 inbox 准入与 TLS 1.3 mTLS/`ql3w` HTTPS client；同一 package journal 上 revision-fenced starting/spawn/started/running/completion 状态、callback digest、tagged no-spawn 与 ambiguous recovery；PostgreSQL starting/running/start-failure/completion 数据库权威事务、精确重放与 cancellation/timeout 优先终态；batch Secret delivery 在 Attempt advisory lock 下复验 Session/Lease/revision 完整围栏并复用单 Agent，Secret-before-Artifact materializer 将同一 log ID 交给 Executor/journal/running ACK；offer-scoped `wlog-*` 私有文件 spool、Edge/Node 容量策略、append/quota/path 防护、barrier 后 output ownership、受审 POSIX Executor、truncation fact、固定内存流式 source、认证 Artifact stream、共享 immutable store port、S3-compatible SSE/checksum/条件 promotion adapter、upload-before-completion 协调，以及 Local/Cluster 同构、Profile-aware、ETag-fenced range read；用户取消 run.stop mutation 以数据库时间写 intent/Event 并在事务内复验 Project/RoleBinding fence；非执行取消 convergence lifecycle、运行期 expiry 与安全 lost retry 已接入 cluster-control 单一全局 cadence；完整 generation/version/token/Attempt fencing；独立最小权限 Worker ingress、CA/CRL 与连接 generation 热重载；offer journal、spawn barrier、receipt-first recovery；独立 `@qinglong/worker-runtime` 的本地 P-256 CSR、key/chain/trust 验证、generation + active pointer 安装和持久退避；默认关闭的 production process 已装配具体 execution graph、完整 Session heartbeat/drain/offline、direct-file bootstrap、单 Agent/单 cadence、startup reconciliation、证书 maintenance、transport fail-close/recovery 与 Edge/Node 有界预算；真实 PostgreSQL 18 + Linux Node 合约已覆盖 Run completion、credential 和 CA 双轮换且保持同一 Session；真实 K3s 合约已覆盖 TLS/credential Secret 分权、双对象 CAS、Recreate 顺序、identity generation 与单节点 PVC recovery；所有能力默认不可达且受 edge/cluster import audit 约束 | 具体 cert-manager/Vault/SPIFFE/离线 CA adapter 与模板、ingress reload controller、生产 RBAC、证书到期告警和 `ql3w` credential recovery 产品面；具体 KMS/Vault Secret provider、对象存储 credential/temporary lifecycle 与 retention/tombstone；Worker 管理 API；真实 Kubernetes 多节点 CSI/node-loss/production 360 秒 drain 与固定 edge 文件系统 suspend/时钟/断电、x64/arm64 资源门禁 | | PR-8 Project/Policy/Approval Core | Incubating（默认拒绝、无生产业务执行入口） | ADR-0028；统一六类 ActorRef 与 exact-shape 校验；`0017` ownerless default Project 和 append-only versioned RoleBinding；owner/admin/operator/viewer 固定矩阵；Project 内 mutation 幂等、expected-version CAS、双 SQLite 连接竞争门禁；archived read-only、revocation、存储损坏 fail-closed；Agent 写/Secret/Tool `require_approval`；ADR-0047 把六类 subject、role/permission matrix 与 fence 抽到 runtime-core，`pg-0004-project-policy`/capability v3 建立 ownerless PostgreSQL baseline、严格 role/state CHECK、append-only runtime 权限、SERIALIZABLE Project lock、mutation replay、双连接单 winner 和 cluster admission authorizer；ADR-0049/`pg-0005` capability v4 建立 stable IdentitySubject、append-only digest-only API credential、真实 cluster bearer authenticator、write-only durable security audit 与最小权限 runtime role，且已验证 HTTP→credential→Policy→audit→handler 纵向链路；ADR-0051 建立 `/api/v3` 认证前 peer/global 双预算、transport-peer-only、无 timer 且有界内存的 overload shield；ADR-0027 Artifact authorizer adapter；ADR-0029 `AuthenticatedPrincipal` contract、`0018` digest-only versioned challenge、CSPRNG/TTL、同事务消费 challenge + 写首 owner、精确重放与双连接竞争/崩溃回滚门禁；ADR-0030 `0019` stable identity/binding、legacy HS384 + current-session membership、logout/platform/revoke/disable、single-factor 与损坏 fail-closed 门禁；ADR-0031 `0020` digest-bound ApprovalRequest、User-only decision、Project/Role version fence、精确 expiry/重放/并发裁决及同事务 immutable dispatch；ADR-0032 `0021` execution backfill、三表原子 consume、稳定 due keyset、claim/renew/start/result fencing、pre-start takeover/post-start recovery-required、attempt budget、handler inspect/digest barrier 和 bounded dispatcher；ADR-0033/`0022` control/resolution backfill、start/renew/completion 原子联动、稳定 recovery keyset、双 resolver claim/takeover、finding/result 精确重放、自动/人工终结、迟到 completion 单 winner 和 evidence-only bounded reconciler；ADR-0034/`0023` 首个 `run.create` canonical plan、Run/Attempt/Event/receipt 同事务、幂等 collision fail-closed、renew/终态 fence、真实 SQLite handler 与 automatic evidence provider；ADR-0035/`0024` 独立 `approval.recover` 矩阵、稳定 User + 五分钟强认证、Project/RoleBinding fence、human resolution + authorization fact 原子提交、撤权竞态与回滚门禁；ADR-0036 recovery-first 单 timer lifecycle、edge/standalone 独立 cadence/页预算、跨周期 cursor、非重叠与有界 stop；ADR-0074 以新的 Node 24 SQLite v5 ownerless Project/RoleBinding/audit authority 和独立 local-secret-admin 提供强 Principal、`secret.manage`、撤权 fence、envelope+allowed audit 原子提交及不回显语义；ADR-0086 以可信 POSIX console 和 staged delivery 完成本机首 Owner 产品 ceremony | fresh database/pepper setup 与安全迁移向导；`shareStore`/Express 到 authentication core 的 production migration；credential rotation/revocation API、mTLS/Worker enrollment、恢复码；Project/Role/Approval/Secret 管理 CLI/API/UI、audit retention/query/export/alert、preview Artifact/digest/immutable plan builder、真实 MFA/hardware adapter、人工 recovery API/UI/独立 rate limit 与审计事件、handler/provider registry、lifecycle startup/shutdown/指标/admission gate；PostgreSQL action/receipt/provider/recovery-authorization 与 OPA adapter、缓存 version 失效；Tool/Package/Secret/Shell 各自的 handler/evidence contract；Secret/Run/Tool/Workflow waiting_approval 全入口装配；完整回滚演练 | @@ -9049,7 +9066,7 @@ ADR-0087 Owner package 更新同样适用于上段 PR-8 累计描述：现行产 PR-8 的 cluster ADR-0049 未完成项由 ADR-0050/0051 部分收敛：Identity register/enable/disable、credential issue/rotate/revoke、mutation ledger、强 actor、同事务 audit、有界 audit query，以及常驻 `/api/v3` 的认证前 overload shield 已孵化完成，但仍默认没有远程管理入口。当前 Gate 仍要求受审 CLI/API/UI、部署 ceremony、管理入口独立 rate limit、双人复核或 break-glass、pepper rotation、audit retention/export/alert；不得把 application service 或 process-local HTTP shield 的存在解释为 cluster-control 已获得管理 authority 或全局 quota。 -未进入当前孵化切片的代码在通过对应 Gate 前必须保持不可达：不得仅因 schema、service 或 Primary 编排器已存在，就让旧 Controller、Scheduler、gRPC callback 或 Shell 脚本直接写入新状态表或调用新 Executor。已接入的 Shadow 观察只能通过默认关闭的 Feature Flag 和 origin owner 决策到达，不得调用 Executor、再次 spawn 或改变 Legacy 返回结果。manual `runSingle` 只增加 owner selection seam；默认没有 router。HTTP bootstrap 每次启动只读取一次 manifest，缺失、禁用、拒绝或非 primary 时保持 Legacy 且不加载重组件；显式 accepted manual primary 会在恢复门禁通过后安装唯一 owner，选中后禁止回退双跑。当前仍不修改 `ScheduleService.runTask`，待其各来源可被拆分识别并建立专门回归门禁后再接入。 +未进入当前孵化切片的代码在通过对应 Gate 前必须保持不可达：不得仅因 schema、service 或 Primary 编排器已存在，就让旧 Controller、Scheduler、gRPC callback 或 Shell 脚本直接写入新状态表或调用新 Executor。已接入的 Shadow 观察只能通过默认关闭的 Feature Flag 和 origin owner 决策到达，不得调用 Executor、再次 spawn 或改变 Legacy 返回结果。manual `runSingle` 只增加 owner selection seam；默认没有 router。HTTP bootstrap 每次启动只读取一次 manifest，缺失、禁用、拒绝或非 primary 时保持 Legacy 且不加载重组件；显式 accepted manual primary 会在恢复门禁通过后安装唯一 owner，选中后禁止回退双跑。ADR-0445 已让 `ScheduleService.runTask` 的 subscription/system/script 在显式 origin flag 下只观察同一个 Legacy ChildProcess；system crond 的 `scheduled_system`、once/boot/grpc 与这些来源的 Primary owner 切换仍须独立门禁。 PR-6 Local Artifact 容量与生命周期当前为 **Incubating（默认不可达）**：已具备 ADR-0026、profile quota/reserve、普通与 durable hard-cap、正/负 truncation fact、`0015` tombstone、`0016` CAS checkpoint、单页 lifecycle、idle 零 checkpoint 写、低敏 JSON-safe summary，以及 ADR-0027/0028 的三态 range query 与默认拒绝 Project authorizer；ADR-0029/0030 已提供默认不可达的认证主体、稳定 legacy User/session mapping 和 bootstrap core。尚需可信 local-console issuer、production session/identity wiring、bootstrap 与 Artifact wire API、存在性屏蔽/审计、production startup/shutdown、指标告警/admission 联动、真实磁盘/多架构门禁及 PostgreSQL/object-store/Worker-spool contract；这些 Gate 未完成前不得装配。 diff --git a/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md b/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md index 6a3b2561..ec70387b 100644 --- a/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md +++ b/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md @@ -5,6 +5,7 @@ - 决策者：QingLong Maintainers - 关联 RFC：[QL-RFC-0001](../QINGLONG_3_0_ARCHITECTURE_RFC.md) - 前置决策：[ADR-0001](./ADR-0001-run-state-and-transaction-boundaries.md) +- Amended by：[ADR-0445](./ADR-0445-schedule-service-origin-shadow-run-coverage.md) ## 1. 决策摘要 @@ -144,13 +145,15 @@ owner 在接受触发时写入执行上下文，并贯穿日志、指标和回 当前孵化实现只开放观察型 Shadow，不通过该环境变量提供 primary： - QL3_SHADOW_ORIGINS=manual,scheduled_node + QL3_SHADOW_ORIGINS=manual,scheduled_node,subscription,system,script - 未设置或设置为空时全部为 off。 -- 当前只接受 `manual` 与 `scheduled_node`；未知 origin 被忽略并记录有界配置告警，不扩大到通用调度入口。 +- ADR-0445 后当前只接受 `manual`、`scheduled_node`、`subscription`、`system` 与 `script`；未知 origin 被忽略并记录有界配置告警， + `scheduled_system`、`once`、`boot` 与 `grpc` 仍不开放。 - 配置在进程内首次使用时读取；edge 不启动 watcher，变更后需要通过既有进程重启或未来的显式 reload 生效。 - 兼容观察器在实际 HTTP/gRPC worker 中按需加载；关闭时不构造 Shadow 事实或任务摘要、不增加 ChildProcess 监听器、不初始化 Repository、不创建后台任务，也不引入额外数据库写入。 -- `manual` 与 `scheduled_node` 都只监听 Legacy 已创建的同一个 ChildProcess。Shadow 代码不得调用 Executor 或第二次 spawn。 +- 所有已开放 origin 都只监听 Legacy 已创建的同一个 ChildProcess。Shadow 代码不得调用 Executor 或第二次 spawn； + `subscription/system/script` 仅在 `ScheduleService` 已选中执行且 `onBefore` 成功后 accepted。 - 任意初始化、接受或后续写入失败都退化为 no-op，只记录不含命令、环境变量和 Secret 的稳定错误类型与有界计数。 - `boot` 虽复用 `runSingle`，仍携带独立 origin，当前不在允许列表中，不能被误记为 manual。 diff --git a/docs/adr/ADR-0445-schedule-service-origin-shadow-run-coverage.md b/docs/adr/ADR-0445-schedule-service-origin-shadow-run-coverage.md new file mode 100644 index 00000000..45856c3e --- /dev/null +++ b/docs/adr/ADR-0445-schedule-service-origin-shadow-run-coverage.md @@ -0,0 +1,80 @@ +# ADR-0445：ScheduleService 执行来源的 Shadow Run 覆盖 + +- 状态：Accepted +- 日期：2026-08-18 +- 关联 RFC：QL-RFC-0001 D-02、D-353、PR-4 +- 关联 ADR：ADR-0001、ADR-0002、ADR-0003 +- Amends：ADR-0002 的当前 Alpha Shadow origin allowlist，不改变 Legacy owner 或 Primary 门禁 + +## 上下文 + +QingLong 3.0 已能旁路观察 `manual` 与 `scheduled_node` 的 Legacy ChildProcess，但 +`ScheduleService.runTask` 仍是 Subscription 更新、System maintenance 和一次性 Script 的共同执行入口。三类调用都由现有 +`cross-spawn` 子进程真实执行，却不创建 Run、RunAttempt 或 RunEvent；因此 3.0 的“原有任务继续运行且每次执行可观察”产品闭环只覆盖了 Cron +主路径，没有覆盖同一进程内的其他用户可达任务。 + +直接把 `ScheduleService` 切为 Primary 会同时影响七个调用点，并绕过仍未完成的 rollout approval、reconciliation、2.x API parity 和回滚门。 +另一方面，只在 callback 或日志层推断任务又会丢失 accepted/spawn 边界，并可能把一次执行关联到错误来源。 + +## 决策 + +1. `QL3_SHADOW_ORIGINS` 的封闭 allowlist 增加 `subscription`、`system`、`script`；默认仍为全部 `off`，未知 origin 仍被忽略并记录有界配置 + 计数。`scheduled_system`、`once`、`boot`、`grpc` 不因本决定自动开放。 +2. `ScheduleService.runTask` 只在既有并发限制已选中任务且 `onBefore` 成功后尝试创建 Shadow accepted fact；随后把观察器附着到同一个 + Legacy ChildProcess。Shadow 路径不得调用 Executor、改变 task limit、延迟 Legacy admission 或产生第二次 spawn。 +3. Shadow Run 固定 `executionOwner=legacy`、Project `default`、原 execution origin 与 + `triggeredBy=legacy:schedule-service`。task revision 只摘要业务 command 与可选 schedule；task ID 使用 + `legacy-schedule::<25-hex>`，其中摘要输入为 Legacy caller ID。任意脚本路径、System command 或 caller ID 原文不得进入 task ID。 +4. origin 未启用时，accepted factory 不执行：不计算 task/revision digest、不附加 ChildProcess listener、不加载 Repository、不写数据库，也不新增 + timer、watcher、queue 或重试。启用后只增加同一子进程的 spawn/error/exit 观察与既有有界 Shadow 写入。 +5. Shadow observer 初始化、accepted 或后续持久化失败继续退化为 no-op；Legacy callback、stdout/stderr、返回 PID、完成结果和用户可见事实源均不受 + 影响。`script` 的 `completionTime=start` 仍在 spawn 后返回原 PID，终态观察异步跟随同一进程。 +6. 本决定不为 `scheduled_system` 伪造 accepted fact。system crond 由 Shell 在另一执行边界启动，目前只有可能重复、乱序或丢失的 status callback， + 缺少 response-loss-safe 的 accept identity；该来源必须在独立 Gate 中先建立稳定触发 ID 与幂等准入。 +7. 本决定不开放任何 Primary origin，不修改 rollout manifest，也不把 Shadow Run 变成 2.x API 成功条件或执行 authority。 + +## 故障与恢复 + +- Shadow origin 未配置或拼写错误：Legacy 正常执行，不构造 fact；未知值只增加低敏配置计数。 +- `onBefore` 失败：执行尚未 accepted/spawn，因此不创建伪 Run；沿用 Legacy 错误行为。 +- observer 初始化或数据库写失败：观察退化为 no-op，同一个 Legacy 子进程继续执行并返回原结果。 +- 子进程启动错误、非零退出或 signal：复用共同 ChildProcess observer 和 Shadow writer，分别收敛为稳定 start-failed/failed 事实；不改变 Legacy + callback。 +- 进程在 `completionTime=start` 返回后结束：已绑定的 observer 继续消费 exit；调用方无需持有 Run authority。 +- 进程重启：环境 allowlist 重新读取；本决定没有 watcher，也不尝试把没有 accepted fact 的历史执行补造成可信 Run。 + +## 部署与资源影响 + +- 不新增 workspace package、生产依赖、schema、migration、SQL、Kubernetes object、端口或常驻进程。 +- 默认 Edge/Standalone/路由设备路径只多一次缓存后的 origin Set 查询；accepted factory 不执行，产物依赖闭包与空闲资源不变。 +- 显式启用的实例为每次匹配执行创建一个 Run、一个 Attempt 和有界事件；这是迁移观测成本，不得默认在低写入寿命设备上启用全部 origin。 +- Cluster Control/Worker 和 Cluster PostgreSQL 路径不导入 Legacy `ScheduleService`，本决定不改变它们的连接、Pool 或部署拓扑。 + +## 被拒绝的替代方案 + +### 直接把 ScheduleService 切到 Primary LocalExecutor + +拒绝。现阶段缺少这三类来源的正式 rollout approval、用户可见状态、2.x API parity 与回滚演练；直接替换会把观察切片变成执行 owner 切换。 + +### 默认观察所有 ExecutionOrigin + +拒绝。默认开启会增加低配设备写入，并把尚未具备稳定 accept identity 的 system crond、boot 与 gRPC 路径错误纳入。 + +### 把 caller ID 或完整 command 写进 task ID + +拒绝。Script ID 可能是路径，System ID 甚至可能来自命令；稳定摘要既保留同一 caller 的关联能力，也避免在 Run 索引和诊断中复制原文。 + +### 只根据结束 callback 补造 Run + +拒绝。结束事实不能可靠证明 accepted/spawn 时间、唯一执行或 task revision；在 status retry 下还会产生重复 Run。 + +## 验证 + +- 27/27 项专项测试通过；独立真实子进程测试覆盖三类 origin、默认环境 allowlist、未启用零 fact、observer 失败开放，以及 Script + start-completion 后的终态观察； +- SQLite 集成测试证明一个 System child 形成 legacy-owned succeeded Run、Attempt 和八个有序 Event，且 caller ID 原文不进入聚合； +- `build:back` 与完整 backend 回归通过（1,408 pass、2 条条件 skip、0 fail），18 个 QL3 package clean build/test 退出 0； +- 14/14 静态审计与 14/14 Edge/Standalone artifact 档位均 compatible；package-boundary 门证明根 `src` 只保留公共 `index.ts`，其余实现位于 + 领域子目录； +- 本决定不改变数据库 schema/adapter 或 Kubernetes/容器拓扑，因此不重跑物理 PostgreSQL HA/K3s 门，也不声称产生新的部署面证据； +- `scheduled_system` 仍作为明确未完成项，不以本测试替代 Shell/response-loss/idempotency 证据。 diff --git a/docs/adr/README.md b/docs/adr/README.md index c3781c9a..f5016c1f 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -448,6 +448,7 @@ | [ADR-0442](./ADR-0442-catalog-ready-terminal-release-tag-publication.md) | Catalog-ready 的终态 Release Tag 发布与闭合收据 | Superseded by ADR-0443（bounded promotion/closure 机制保留） | | [ADR-0443](./ADR-0443-deployment-ready-terminal-release-finalization.md) | Deployment-ready 的终态 Release Finalization | Accepted（首份真实 GHCR deployment-ready finalization 待实际 release tag） | | [ADR-0444](./ADR-0444-fail-closed-release-tag-finalizer-and-replay-rehearsal.md) | Fail-closed Release Tag Finalizer 与重放演练 | Accepted（首份真实 GHCR response-loss 重放待实际 release tag） | +| [ADR-0445](./ADR-0445-schedule-service-origin-shadow-run-coverage.md) | ScheduleService 执行来源的 Shadow Run 覆盖 | Accepted（`scheduled_system` 幂等准入待独立 Gate） | ## 规则 diff --git a/packages/ql3-runtime-core/src/plugin-package/pluginPackageBundle.ts b/packages/ql3-runtime-core/src/plugin-package/pluginPackageBundle.ts index 6f848286..fa1637b2 100644 --- a/packages/ql3-runtime-core/src/plugin-package/pluginPackageBundle.ts +++ b/packages/ql3-runtime-core/src/plugin-package/pluginPackageBundle.ts @@ -4,6 +4,7 @@ import { verify, type KeyObject, } from 'node:crypto'; +import { TextDecoder } from 'node:util'; import { MAX_PLUGIN_PACKAGE_CONTENT_ENTRIES, diff --git a/packages/ql3-runtime-core/src/plugin-package/pluginPackageResourceMaterialization.ts b/packages/ql3-runtime-core/src/plugin-package/pluginPackageResourceMaterialization.ts index 84a16436..4ca44f9f 100644 --- a/packages/ql3-runtime-core/src/plugin-package/pluginPackageResourceMaterialization.ts +++ b/packages/ql3-runtime-core/src/plugin-package/pluginPackageResourceMaterialization.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { TextDecoder } from 'node:util'; import { MAX_PLUGIN_PACKAGE_MANIFEST_BYTES, diff --git a/test/back/legacyScheduleServiceShadow.test.cjs b/test/back/legacyScheduleServiceShadow.test.cjs new file mode 100644 index 00000000..4de80372 --- /dev/null +++ b/test/back/legacyScheduleServiceShadow.test.cjs @@ -0,0 +1,310 @@ +require('ts-node/register/transpile-only'); + +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const path = require('node:path'); +const { test } = require('node:test'); +const { Sequelize } = require('sequelize'); +const ScheduleService = require('../../back/services/schedule').default; +const { + defineSchemaMigrationModel, +} = require('../../back/data/schemaMigration'); +const { runSchemaMigration } = require('../../back/migrations/0002-run-schema'); +const { + runCancellationRequestMigration, +} = require('../../back/migrations/0004-run-cancellation-request'); +const { + runAttemptDeadlineMigration, +} = require('../../back/migrations/0006-run-attempt-deadline'); +const { runMigrations } = require('../../back/migrations/runner'); +const { + LegacySequelizeRunRepository, +} = require('../../back/runtime/adapters/legacy-sequelize/runRepository'); +const { + LegacyShadowRunObserver, +} = require('../../back/runtime/application/legacyShadowRunObserver'); +const { + LegacyShadowRunWriter, +} = require('../../back/runtime/application/legacyShadowRunWriter'); +const { + installLegacyExecutionObserver, +} = require('../../back/runtime/compatibility/legacyExecutionBridge'); +const { + shadowOnlyRollout, +} = require('../../back/runtime/domain/runtimeRollout'); + +function logger() { + return { + info() {}, + warn() {}, + error() {}, + }; +} + +function successfulCommand() { + return `${JSON.stringify(process.execPath)} -e "process.exit(0)"`; +} + +test('admits the three reviewed origins through the default environment boundary', () => { + const source = ` + const bridge = require('./back/runtime/compatibility/legacyExecutionBridge'); + const fact = (origin) => ({ + origin, + projectId: 'default', + taskId: 'opaque', + taskRevision: 'sha256:${'1'.repeat(64)}', + triggerType: origin, + acceptedAtMs: 1, + }); + const result = ['subscription', 'system', 'script', 'boot'].map((origin) => + Boolean(bridge.observeLegacyExecution(origin, () => fact(origin))), + ); + process.stdout.write(JSON.stringify(result)); + process.exit(result.join(',') === 'true,true,true,false' ? 0 : 1); + `; + const child = spawnSync( + process.execPath, + ['-r', 'ts-node/register/transpile-only', '-e', source], + { + cwd: path.resolve(__dirname, '../..'), + env: { + ...process.env, + QL3_SHADOW_ORIGINS: 'subscription,system,script', + }, + encoding: 'utf8', + timeout: 10_000, + }, + ); + + assert.equal(child.status, 0, child.stderr); + assert.equal(child.stdout, '[true,true,true,false]'); +}); + +test('observes subscription, system and script children without replacing legacy execution', async () => { + const facts = []; + const restore = installLegacyExecutionObserver( + { + begin(fact) { + const lifecycle = []; + facts.push({ fact, lifecycle }); + return { + spawned: (value) => lifecycle.push(['spawned', value]), + running: (value) => lifecycle.push(['running', value]), + startFailed: (value) => lifecycle.push(['start_failed', value]), + exited: (value) => lifecycle.push(['exited', value]), + cancelled: (value) => lifecycle.push(['cancelled', value]), + }; + }, + }, + ['subscription', 'system', 'script'], + ); + + try { + const service = new ScheduleService(logger()); + for (const runOrigin of ['subscription', 'system', 'script']) { + let resolveEnd; + const ended = new Promise((resolve) => { + resolveEnd = resolve; + }); + const result = await service.runTask( + successfulCommand(), + { onEnd: async () => resolveEnd() }, + { + id: `private/path/${runOrigin}`, + name: `${runOrigin} task`, + schedule: '0 * * * *', + runOrigin, + }, + runOrigin === 'script' ? 'start' : 'end', + ); + if (runOrigin === 'script') { + assert.equal(result > 0, true); + await ended; + } else { + assert.equal(result.code, 0); + } + } + } finally { + restore(); + } + + assert.equal(facts.length, 3); + assert.deepEqual( + facts.map(({ fact }) => fact.origin), + ['subscription', 'system', 'script'], + ); + for (const { fact, lifecycle } of facts) { + assert.equal(fact.projectId, 'default'); + assert.match( + fact.taskId, + new RegExp(`^legacy-schedule:${fact.origin}:[0-9a-f]{25}$`, 'u'), + ); + assert.equal(fact.taskId.includes('private/path'), false); + assert.match(fact.taskRevision, /^sha256:[0-9a-f]{64}$/u); + assert.equal(fact.triggerType, fact.origin); + assert.equal(fact.triggeredBy, 'legacy:schedule-service'); + assert.equal(Number.isSafeInteger(fact.acceptedAtMs), true); + assert.deepEqual( + lifecycle.map(([operation]) => operation), + ['spawned', 'running', 'exited'], + ); + assert.equal(lifecycle[0][1].pid > 0, true); + assert.equal( + lifecycle[0][1].executorHandle.startsWith('legacy-local:'), + true, + ); + assert.equal(lifecycle[2][1].exitCode, 0); + } +}); + +test('does not construct shadow facts for an origin that was not enabled', async () => { + let accepted = 0; + const restore = installLegacyExecutionObserver( + { + begin() { + accepted += 1; + throw new Error('disabled origin must not reach the observer'); + }, + }, + ['manual'], + ); + + try { + const service = new ScheduleService(logger()); + const result = await service.runTask( + successfulCommand(), + {}, + { + id: 'system-disabled', + runOrigin: 'system', + }, + ); + assert.equal(result.code, 0); + } finally { + restore(); + } + + assert.equal(accepted, 0); +}); + +test('keeps legacy execution successful when the enabled shadow observer fails', async () => { + const restore = installLegacyExecutionObserver( + { + begin() { + throw new Error('shadow storage unavailable'); + }, + }, + ['system'], + ); + + try { + const service = new ScheduleService(logger()); + const result = await service.runTask( + successfulCommand(), + {}, + { + id: 'shadow-failure', + runOrigin: 'system', + }, + ); + assert.equal(result.code, 0); + } finally { + restore(); + } +}); + +test('persists one legacy-owned terminal Run aggregate for a scheduled service child', async () => { + const database = new Sequelize({ + dialect: 'sqlite', + storage: ':memory:', + logging: false, + }); + const runId = '019f70f0-0000-7000-8000-000000000801'; + const attemptId = '019f70f0-0000-7000-8000-000000000802'; + const observations = []; + const failures = []; + let idSequence = 800; + + try { + await runMigrations({ + database, + migrationModel: defineSchemaMigrationModel(database), + migrations: [ + runSchemaMigration, + runCancellationRequestMigration, + runAttemptDeadlineMigration, + ], + logger: { info() {} }, + }); + const repository = new LegacySequelizeRunRepository(database); + const writer = new LegacyShadowRunWriter(repository, () => { + idSequence += 1; + return `019f70f0-0000-7000-8000-${String(idSequence).padStart(12, '0')}`; + }); + const delegate = new LegacyShadowRunObserver( + shadowOnlyRollout(['system']), + writer, + { failure: (failure) => failures.push(failure) }, + ); + const restore = installLegacyExecutionObserver( + { + begin(fact) { + const observation = delegate.begin(fact); + observations.push(observation); + return observation; + }, + }, + ['system'], + ); + + try { + const service = new ScheduleService(logger()); + const result = await service.runTask( + successfulCommand(), + {}, + { + id: 'credential=must-not-persist', + name: 'system maintenance', + runOrigin: 'system', + }, + ); + assert.equal(result.code, 0); + assert.equal(observations.length, 1); + await observations[0].settled(); + } finally { + restore(); + } + + const run = await repository.findRunById(runId); + const attempt = await repository.findAttemptById(attemptId); + const events = await repository.listEvents(runId); + assert.deepEqual(failures, [], JSON.stringify(failures)); + assert.equal(run.executionOrigin, 'system'); + assert.equal(run.executionOwner, 'legacy'); + assert.equal(run.status, 'succeeded'); + assert.equal(run.taskId.includes('credential='), false); + assert.equal(attempt.status, 'succeeded'); + assert.equal(attempt.executorType, 'legacy_local'); + assert.equal(attempt.pid > 0, true); + assert.equal(attempt.exitCode, 0); + assert.deepEqual( + events.map((event) => event.type), + [ + 'run.created', + 'run.queued', + 'run.dispatching', + 'attempt.starting', + 'attempt.running', + 'run.running', + 'attempt.succeeded', + 'run.succeeded', + ], + ); + assert.equal( + JSON.stringify({ run, attempt, events }).includes('credential='), + false, + ); + } finally { + await database.close(); + } +});