From c9e41812cb73ccf2da66cdeef435a7a94fdf7296 Mon Sep 17 00:00:00 2001 From: whyour Date: Fri, 21 Aug 2026 00:52:58 +0800 Subject: [PATCH] feat(ql3): rehearse legacy sqlite upgrade --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 25 +- ...y-sqlite-upgrade-and-rollback-rehearsal.md | 138 ++++++ docs/adr/README.md | 1 + docs/operations/ql3-local-deployment.md | 10 +- docs/operations/ql3-local-sqlite-adoption.md | 143 ++++++ .../local-sqlite-adoption/activation.ts | 10 + .../local-sqlite-adoption/contracts.ts | 1 + .../src/deployment/cutover/legacyStop.ts | 3 + .../deployment/cutover/targetDataEvidence.ts | 18 +- .../src/deployment/cutover/targetEvidence.ts | 3 + .../service-manager/serviceCutoverConsumer.ts | 4 +- .../src/lifecycle/adoptionCli.ts | 4 +- .../src/lifecycle/adoptionCommand.ts | 42 ++ .../src/lifecycle/sqlite-adoption/command.ts | 431 ++++++++++++++++++ .../src/lifecycle/sqlite-adoption/contract.ts | 244 ++++++++++ .../test/cutoverTargetRun.test.cjs | 10 +- .../test/localDeployment.test.cjs | 4 + .../test/serviceCutoverConsumer.test.cjs | 3 +- .../test/sqliteAdoptionCli.test.cjs | 428 +++++++++++++++++ scripts/ql3-cluster-dependency-audit.cjs | 5 + .../ql3-service-manager-bridge-live-actor.cjs | 7 +- test/back/ql3ClusterDependencyAudit.test.cjs | 18 + test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 23 files changed, 1533 insertions(+), 23 deletions(-) create mode 100644 docs/adr/ADR-0476-real-legacy-sqlite-upgrade-and-rollback-rehearsal.md create mode 100644 docs/operations/ql3-local-sqlite-adoption.md create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/adoptionCommand.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/command.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/contract.ts create mode 100644 packages/ql3-local-owner-cli/test/sqliteAdoptionCli.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 896618ae..18a1fcbf 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -6,11 +6,32 @@ - 目标版本：QingLong 3.x - 作者：QingLong Maintainers - 创建日期：2026-07-17 -- 最后更新：2026-08-20 +- 最后更新：2026-08-21 - 讨论范围：架构与演进路线，不包含最终 UI 视觉方案 -最新增量证据（2026-08-20）： +最新增量证据（2026-08-21）： +- D-383/ADR-0476（已接受）：把单个 2.x SQLite 主库接管从分散 API/合成 fixture 推进为产品级真实双态演练。既有一次性 + `ql3-adoption` 新增 exact、私有 command-file 的 `inspect → stage → verify → activation`，从生产形态 Sequelize schema + （Cron、Dependency、App、Auth、Env、Subscription、View、Stats、RunningInstance 与未知 Plugin-owned table）生成独立 recovery + 和 target，只迁移 target，并由 adopted runtime 持有 source write fence。真实演练证明 recovery 不含 3.0 表、target 同时保留 + legacy/plugin 数据与新增 3.0 表；clean target stop 得到 `rollback_candidate` 且 fence 释放后 legacy 写恢复；正式 Run repository + 写 target 后得到 `reconciliation_required`，source 字节和 schema 不变；扩权 command 在 inspect 前失败。演练发现 SQLite Online + Backup 与 source 逻辑等价却不保证物理 SHA 相同，旧 classifier 因比较 recovery SHA 会误拒合法回滚；activation 因此分别绑定 + `sourceSha256`、`recoverySha256`、`targetSha256`，数据证据字段同步纠正为 `sourceMatchesActivation`。GitNexus 对修改点最高为 + MEDIUM（activation payload 7 direct/27 total），无 HIGH/CRITICAL。没有新增 package、dependency、binary、daemon、listener、timer + 或部署对象；Local Owner 新代码内聚在 `lifecycle/sqlite-adoption/`，workspace 保持 18 packages、 + `singleSourcePackages=[]`、`shallowSourcePackages=[]`、`116 source / 115 nested / 1 root binary entry`。真实 rehearsal `3/3`、 + D-383 focused `102/102`、Local Admin `91/91`、Local Owner `190 total / 185 pass / 5 conditional skip / 0 fail`；backend 全量 + `1,535 total / 1,533 pass / 2 conditional skip / 0 fail`，`pnpm build:back` 与 18-package clean build/逐包测试通过；package + boundary、Cluster dependency、Edge import、Service Bridge import、Cluster/Worker deployment、Console 与 distribution 八项审计 + 全 compatible/passed。14 档 artifact audit 全 compatible：基础 Edge/Standalone + `2,598,669 / 2,598,747` bytes、316 files、57 modules；Adopted `2,818,404 / 2,818,527` bytes、336 files、58 modules； + Application+AI `4,502,262 / 4,502,394` bytes、511 files、141 modules；MCP `7,324,601 / 7,324,709` bytes、802 files、 + 227 modules。四阶段均是 one-shot，schema 最多 4096 项、manifest 最大 256 KiB，基础 Edge 不携带 adopted authority；operator + 仍须为 recovery + target 与 SQLite 临时文件预留磁盘，制品 import RSS 不冒充真实路由设备迁移峰值。本阶段不触及 PostgreSQL + schema/ACL/repository/role/Pool/连接/failover，故不重跑且不重新占有 HA 证明。D-384 应从单 SQLite 主库扩展到完整 2.x data + directory 资产盘点/恢复合同与固定物理 Edge 升级演练；target 写后数据 reconciliation 和 OpenRC live actor 仍未完成。 - D-382/ADR-0475（已接受）：扩展 3.0 首发前的 2.x HTTP 兼容基线，使用真实 loopback HTTP、生产 Express middleware、System/ Script/Open Router 与 Celebrate validator，锁定 System config/四类 mutation/reload/notify、Script list/detail/create/rename/run、 Open app CRUD/reset-secret/token issuance，以及面板/Open token、scope、expiration、路径大小写、400/401/500 envelope。测试以 diff --git a/docs/adr/ADR-0476-real-legacy-sqlite-upgrade-and-rollback-rehearsal.md b/docs/adr/ADR-0476-real-legacy-sqlite-upgrade-and-rollback-rehearsal.md new file mode 100644 index 00000000..f4e8ce19 --- /dev/null +++ b/docs/adr/ADR-0476-real-legacy-sqlite-upgrade-and-rollback-rehearsal.md @@ -0,0 +1,138 @@ +# ADR-0476：真实 Legacy SQLite 升级与回滚演练 + +- 状态：Accepted +- 日期：2026-08-21 +- 关联 RFC：QL-RFC-0001 D-383 +- 关联 ADR：ADR-0064、ADR-0310、ADR-0314、ADR-0472、ADR-0475 + +## 背景 + +此前的 Local SQLite adoption、activation、target stop 与 rollback 已分别拥有单元测试和合成 fixture，但还没有一条从生产形态 +QingLong 2.x Sequelize 数据库出发、只走产品 CLI 和正式 repository、同时证明 clean rollback 与 target 写后拒绝回滚的闭环。 +这意味着“代码存在”仍不能证明现有部署用户能安全孵化 3.0。 + +演练还暴露了一个被小型假 fixture 隐藏的架构错误：SQLite Online Backup 生成的 recovery 数据库与 source 在逻辑上可以完全等价， +但页布局、freelist 或 checkpoint 状态可能使两个主文件的物理 SHA-256 不同。旧 classifier 把 live source 的哈希与 +`recoverySha256` 比较，因此会把真实 clean rollback 错判为 `manual_review`。回滚判定需要区分三个事实：activation 时的原始 +source 字节、Online Backup recovery 字节，以及迁移后 target 字节。 + +该能力必须同时适用于资源很小的路由设备和单机节点；Cluster 节点不能因此把 Local SQLite authority、Owner CLI 或迁移写权限 +引入控制面。新增入口也不能继续把 `packages/` 拆成单文件包。 + +编辑前 GitNexus upstream impact 均为有界：adoption CLI `main`、activation parser/preparer/acquirer、target evidence 读取与消费 +均为 LOW；`LocalSqliteActivationPayload` 为 MEDIUM（7 个直接、27 个累计影响），没有 HIGH 或 CRITICAL symbol。实现按该边界 +更新全部 exact parser 和 fixture。 + +## 决策 + +### 1. 在既有 `ql3-adoption` 中提供产品级四阶段命令 + +不增加 package、binary 或依赖。现有一次性 `ql3-adoption run --command-file` 根据 exact operation 延迟加载 SQLite adoption +实现，并保留原 Legacy Crontab adoption 命令。新增操作为： + +1. `local-sqlite.adoption.inspect`：只读盘点 source、schema catalog 与 task adoption plan； +2. `local-sqlite.adoption.stage`：绑定人工复核后的 `expectedPlanDigest`，通过 SQLite Online Backup 生成独立 recovery 和 target， + 只对 target 执行 3.0 migration，并发布 manifest； +3. `local-sqlite.adoption.verify`：重新证明 target、recovery、manifest、migration 与 readiness； +4. `local-sqlite.activation.prepare`：绑定 `expectedManifestDigest`，同时围栏 source 与 target，发布后续 adopted runtime/cutover + 消费的 activation。 + +命令只接受 `schemaVersion=1`、`edge|standalone`、精确 key 集和最长 4096 bytes 的 normalized absolute non-root path；未知字段在 +检查数据库前失败。command file 和所有输出必须是 current-UID、canonical、non-symlink、单链接私有文件；deployment/output +目录必须是 current-UID canonical `0700`，输出只能位于 deployment root 内。source 可以位于既有 2.x data root，但必须由同一 +UID 拥有、不是 symlink/硬链接且不可被 group/world 写。执行前后的 inode、size、mtime、ctime、mode 与 UID 必须稳定。 + +### 2. Activation 分别绑定 source、recovery 和 target 的物理身份 + +`LocalSqliteActivationPayload` 新增 `sourceSha256`。prepare 在 source write fence 内读取该哈希；runtime acquire 在相同围栏内 +重新计算并拒绝 source 字节漂移。`recoverySha256` 继续证明 Online Backup 产物，`targetSha256` 继续证明刚迁移完成的 target， +三个摘要不得互相替代。 + +target stop/reconciliation 现在使用： + +- `targetMatchesActivation = current target SHA-256 == targetSha256`； +- `sourceMatchesActivation = current source SHA-256 == sourceSha256`； +- source 与 target 的 `-wal`、`-shm`、`-journal` 必须全部不存在。 + +只有 target/source 均保持 activation 字节且 sidecar clear 才是 `rollback_candidate`。target 有任何写入或 sidecar 即 +`reconciliation_required`；source 漂移、activation/稳定文件身份无法证明则是 `manual_review`。旧的误导字段名 +`sourceMatchesRecovery` 在 3.0 首发前改为 `sourceMatchesActivation`，证据 digest 和 exact verifier 同步更新；尚未发布的孵化 +记录不作为跨版本兼容格式。 + +### 3. 使用生产形态双态演练，而不是复制合成摘要 + +回归 fixture 建立 2.x 生产形态的 `Crontabs`、`Dependences`、`Apps`、`Auths`、`Envs`、`Subscriptions`、 +`CrontabViews`、`CrontabStats`、`RunningInstances`、`sqlite_sequence`，并保留未知的 `PluginOwnedState`。测试只通过产品 +CLI 完成 inspect → stage → verify → prepare，再启动正式 adopted storage： + +- recovery 保留 legacy/plugin 数据且没有任何 `QingLong3*` 表； +- target 保留 legacy/plugin 数据并只在 target 上增加 3.0 表； +- adopted storage 持有 source write fence，legacy writer 在此期间不能提交； +- target 未写即停止时得到 `rollback_candidate`，fence 释放后 legacy 写恢复； +- 通过正式 Run repository 向 target 写入后得到 `reconciliation_required`，source 不含 3.0 表且字节不变； +- 带额外 authority 字段的命令在 inspect 前失败。 + +这是一条真实 schema/产品路径 rehearsal，但仍是本机临时目录中的自动门，不冒充用户实际路由器、NAS 或生产数据快照。 + +### 4. 低配设备与 Cluster 边界 + +四个操作都是显式的一次性进程，没有 daemon、listener、watcher、timer、poller、queue 或后台 retry。基础 Edge 制品不包含 +Local Admin adoption；只有 adopted profile 增加该能力。SQLite Online Backup 以数据库页为单位，不把整个数据库加载到 JS +heap；schema inventory 上限为 4096 个对象，manifest 上限 256 KiB。operator 仍必须预留 source 之外至少 recovery + target 两份 +数据库及 SQLite 临时/sidecar 余量，空间不足必须在切换前失败，不能删除 recovery 腾空间。 + +Cluster dependency audit 只允许现有 Local Owner 中精确的 `lifecycle/sqlite-adoption/command.ts` 从 +`@qinglong/local-admin` package root 导入 adoption API；其他 Cluster/Worker/Edge source 不能借此获得 Local storage authority。 +实现增加在既有 Local Owner 的 `lifecycle/sqlite-adoption/` 内聚目录，没有新增 workspace package；package audit 保持 +`singleSourcePackages=[]`、`shallowSourcePackages=[]`。 + +## 被否决方案 + +1. **新建 `ql3-sqlite-adoption` package/binary**：没有独立交付或生命周期理由，只会加剧 package 碎片化，拒绝。 +2. **把 recovery SHA 当作原 source SHA**：Online Backup 不保证物理字节相同，会拒绝合法 clean rollback，拒绝。 +3. **直接覆盖 2.x source 后再迁移**：失去独立恢复副本和双态证据，失败时不可安全分类，拒绝。 +4. **target 写后自动覆盖回 source**：Run/Task/Workflow 与 legacy 表之间没有通用无损逆迁移，拒绝。 +5. **在 application 启动时隐式 inspect/stage**：把高风险数据变更藏进 daemon 生命周期，也无法让 operator 审核 digest，拒绝。 +6. **为 Cluster 共用迁移服务**：扩大数据库 authority 和部署闭包，且低配 Local 不需要远端控制面，拒绝。 + +## 升级与回退 + +升级前保留原 2.x source，所有 stage 输出使用新路径且 no-replace 发布。operator 必须保存 inspect 的 `planDigest`、verify 的 +`manifestDigest` 和 prepare 的 `activationDigest`，后续 cutover 命令只能消费这些 exact digest。不得手工编辑 manifest、activation， +不得把 target 重命名覆盖 source。 + +若 3.0 target 从未接受写入并得到 `rollback_candidate`，只能继续既有双阶段 Legacy rollback ceremony；该 classification 本身不 +授权启动 2.x。若 target 已写，必须保留 source、target、recovery 和证据，进入独立 reconciliation，不得自动回退。 + +activation schema 增加 `sourceSha256`，且 evidence 字段改名；旧孵化 activation/target-stop journal 必须从 inspect 开始重新生成， +不能混用旧记录。QingLong 3.0 尚未发布，因此本 ADR 优先修正语义，不维持错误的预发布持久格式。 + +## 验收证据 + +- 产品级真实 SQLite rehearsal `3/3`；Local Admin 全量 `91/91`；Local Owner 全量 + `190 total / 185 pass / 5 conditional skip / 0 fail`。 +- D-383 相关 focused regression 共 `102/102`；backend 全量 + `1,535 total / 1,533 pass / 2 conditional skip / 0 fail`；`pnpm build:back` 通过；18-package clean build 与逐包测试由 + 单条命令完成且退出 0。 +- workspace 保持 18 packages，无新增 dependency、binary 或部署对象；Local Owner 为 + `116 source / 115 nested / 1 root binary entry`，`singleSourcePackages=[]`、`shallowSourcePackages=[]`。 +- 14 档 Local artifact audit 全部 compatible：基础 Edge/Standalone 为 + `2,598,669 / 2,598,747` bytes、316 files、57 loaded modules；Adopted 为 + `2,818,404 / 2,818,527` bytes、336 files、58 loaded modules；Application+AI 为 + `4,502,262 / 4,502,394` bytes、511 files、141 loaded modules；MCP 为 + `7,324,601 / 7,324,709` bytes、802 files、227 loaded modules。 +- package boundary、Cluster dependency、Edge import、Service Bridge import、Cluster/Worker deployment、Console 与 Console + distribution 八项审计全部 compatible/passed。 +- 本阶段不修改 PostgreSQL schema、ACL、repository、role、Pool、连接或 failover 语义，因此不重跑且不重新占有 PostgreSQL HA + 证明；D-373/D-374 的 HA 结果只作为相邻既有基线。 + +## 未完成 + +- 从真实 2.x 完整 data directory 接管 scripts、configs、logs 与其他文件资产； +- 固定物理 Edge/NAS 上的磁盘峰值、耗时、cgroup RSS 与断电恢复演练； +- target 写后的显式数据域 reconciliation/导出工具； +- 真实用户数据快照与 systemd/OpenRC/Compose 全链路升级回退演练； +- OpenRC live actor（待镜像基础设施恢复）。 + +本 ADR 关闭单个生产形态 2.x SQLite 主库的产品级升级与 clean/write-after 双态回滚分类，不代表完整 2.x data directory 或 +QingLong 3.0 GA 升级门已经完成。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 628ad5bf..75c18525 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -479,6 +479,7 @@ | [ADR-0473](./ADR-0473-service-manager-legacy-rollback-commit.md) | Service Manager Legacy Rollback Commit | Accepted（OpenRC live actor 待补） | | [ADR-0474](./ADR-0474-bounded-legacy-core-readiness-proof.md) | 有界 Legacy Core Readiness Proof | Accepted（OpenRC live actor 待补） | | [ADR-0475](./ADR-0475-legacy-system-script-open-api-compatibility.md) | Legacy System、Script 与 Open API 兼容基线 | Accepted | +| [ADR-0476](./ADR-0476-real-legacy-sqlite-upgrade-and-rollback-rehearsal.md) | 真实 Legacy SQLite 升级与回滚演练 | Accepted | ## 规则 diff --git a/docs/operations/ql3-local-deployment.md b/docs/operations/ql3-local-deployment.md index a758492a..78e0fc6d 100644 --- a/docs/operations/ql3-local-deployment.md +++ b/docs/operations/ql3-local-deployment.md @@ -157,6 +157,11 @@ receipt。 ### 2.2 adopted 2.x 的 Docker legacy silence gate +已有 2.x SQLite 部署必须先完成 +[`inspect → stage → verify → activation`](./ql3-local-sqlite-adoption.md)，取得真实、未手写的 manifest、activation 与 digest。 +下列 silence/cutover 命令只消费该证据，不负责隐式迁移数据库。source、target 和 recovery 必须一直保留到回退或 reconciliation +正式闭合。 + adopted target 不得直接使用旧 v1 配置启动。当前第一条可达的 cutover 产品路径只接管由 Docker 精确容器 ID 标识的 2.x owner；systemd/OpenRC legacy、Kubernetes 和远端 cluster 不允许伪装成 Docker evidence。先确保 deployment root 与其 `service/` 都是当前 UID 的 @@ -399,11 +404,12 @@ reconciliation_required manual_review ``` -- `rollback_candidate`：target 主文件仍等于 activation 初始 SHA-256、source 仍等于 recovery SHA-256， +- `rollback_candidate`：target 主文件仍等于 activation 的 `targetSha256`、source 仍等于 activation 的 + `sourceSha256`， 且两者均没有 `-wal`、`-shm`、`-journal` sidecar。 - `reconciliation_required`：target 主文件已经变化，或发现 target SQLite sidecar；必须保留 target 并进入 后续数据域 reconciliation，不能启动 2.x。 -- `manual_review`：activation/文件稳定身份不能证明，或 target 未写但 source 已偏离 recovery；不得猜测。 +- `manual_review`：activation/文件稳定身份不能证明，或 target 未写但 source 已偏离 activation 时原始 source；不得猜测。 `rollback_candidate` 本身不是 legacy restart 授权。实例 head 成功后进入 `target_stopped`，旧 restart command 会在 Docker authority 前失败；只有下一节的双阶段 ceremony 可以请求启动 2.x。 diff --git a/docs/operations/ql3-local-sqlite-adoption.md b/docs/operations/ql3-local-sqlite-adoption.md new file mode 100644 index 00000000..01f7fafc --- /dev/null +++ b/docs/operations/ql3-local-sqlite-adoption.md @@ -0,0 +1,143 @@ +# QingLong 2.x SQLite 接管到 3.0 + +本流程把已有 QingLong 2.x `database.sqlite` 以不覆盖 source 的方式接管为 3.0 adopted storage。它只处理单个 SQLite 主库； +scripts、configs、logs 和其他 data directory 文件不在本流程内。 + +全部命令由最终运行 QingLong 的同一个 POSIX 用户执行。`ql3-adoption` 是一次性 CLI，不启动服务，也不会自动执行 cutover 或 +Legacy rollback。 + +## 1. 空间与停机前检查 + +- `deploymentRoot` 及输出父目录必须是当前 UID、canonical、非 symlink 的 `0700` 目录； +- command file 必须是当前 UID、canonical、单链接 `0600` 文件； +- source 必须是当前 UID 的 canonical 普通单链接文件，且 group/world 不可写；生产环境建议 `0600`； +- target、recovery、manifest、activation 必须尚不存在，并全部位于 `deploymentRoot` 内； +- source、target、recovery、manifest、activation 五个路径必须互不相同； +- 预留 recovery + target 两份数据库以及 SQLite 临时/sidecar 余量。不要以删除 recovery 的方式解决空间不足。 + +inspect 和 Online Backup 可以在 2.x 仍可读时执行，但最终 stage review 后到 activation/cutover 期间必须停止外部 writer，并保证 +source 不再漂移。activation prepare 会尝试取得 source/target write fence；无法取得时失败关闭。 + +低配设备说明：四个阶段都是 one-shot 进程，没有后台 timer、监听端口或自动重试；schema inventory 最多 4096 项，manifest 最多 +256 KiB，SQLite backup 按页复制而不是把整个数据库载入 JS heap。制品审计的 import RSS 不是迁移峰值承诺，正式升级前仍需在目标 +路由器/NAS 上测量可用空间、耗时、峰值 RSS 和断电恢复。 + +## 2. Inspect：生成待审核计划 + +```json +{ + "schemaVersion": 1, + "operation": "local-sqlite.adoption.inspect", + "options": { + "deploymentRoot": "/opt/qinglong3", + "profile": "edge", + "sourcePath": "/opt/qinglong/data/database.sqlite", + "legacyTimezone": "Asia/Shanghai" + } +} +``` + +```sh +chmod 0600 /secure/operator/ql3-sqlite-inspect.json +ql3-adoption run --command-file /secure/operator/ql3-sqlite-inspect.json +``` + +记录返回的 `planDigest`，审核 schema catalog、table names 与 task inventory。不要在审核后修改 source；若 source 变化，重新 inspect， +不要沿用旧 digest。 + +## 3. Stage：生成 recovery、target 与 manifest + +```json +{ + "schemaVersion": 1, + "operation": "local-sqlite.adoption.stage", + "options": { + "deploymentRoot": "/opt/qinglong3", + "profile": "edge", + "sourcePath": "/opt/qinglong/data/database.sqlite", + "targetPath": "/opt/qinglong3/artifacts/qinglong3.sqlite", + "recoveryPath": "/opt/qinglong3/artifacts/database.pre-ql3.sqlite", + "manifestPath": "/opt/qinglong3/artifacts/qinglong3-adoption.json", + "expectedPlanDigest": "REPLACE_WITH_INSPECT_PLAN_DIGEST", + "legacyTimezone": "Asia/Shanghai" + } +} +``` + +```sh +chmod 0600 /secure/operator/ql3-sqlite-stage.json +ql3-adoption run --command-file /secure/operator/ql3-sqlite-stage.json +``` + +stage 通过 SQLite Online Backup 创建独立 recovery 和 target，只在 target 上执行 3.0 migration，并以 `0600` no-replace 文件发布 +结果。source 不会被覆盖。任一输出已存在时不要删除后盲目重跑，应先把现有文件和命令结果作为一次未完成 ceremony 调查。 + +## 4. Verify：独立复验 staged 结果 + +```json +{ + "schemaVersion": 1, + "operation": "local-sqlite.adoption.verify", + "options": { + "deploymentRoot": "/opt/qinglong3", + "profile": "edge", + "targetPath": "/opt/qinglong3/artifacts/qinglong3.sqlite", + "recoveryPath": "/opt/qinglong3/artifacts/database.pre-ql3.sqlite", + "manifestPath": "/opt/qinglong3/artifacts/qinglong3-adoption.json" + } +} +``` + +```sh +chmod 0600 /secure/operator/ql3-sqlite-verify.json +ql3-adoption run --command-file /secure/operator/ql3-sqlite-verify.json +``` + +记录返回的 `manifestDigest`。verify 只接受与 manifest 一致的 recovery/target、完整 migration 和通过的 adopted readiness。 + +## 5. Prepare activation：冻结三份物理事实 + +```json +{ + "schemaVersion": 1, + "operation": "local-sqlite.activation.prepare", + "options": { + "deploymentRoot": "/opt/qinglong3", + "profile": "edge", + "sourcePath": "/opt/qinglong/data/database.sqlite", + "targetPath": "/opt/qinglong3/artifacts/qinglong3.sqlite", + "recoveryPath": "/opt/qinglong3/artifacts/database.pre-ql3.sqlite", + "manifestPath": "/opt/qinglong3/artifacts/qinglong3-adoption.json", + "activationPath": "/opt/qinglong3/artifacts/qinglong3-activation.json", + "expectedManifestDigest": "REPLACE_WITH_VERIFY_MANIFEST_DIGEST" + } +} +``` + +```sh +chmod 0600 /secure/operator/ql3-sqlite-activation.json +ql3-adoption run --command-file /secure/operator/ql3-sqlite-activation.json +``` + +activation 分别记录 `sourceSha256`、`recoverySha256` 和 `targetSha256`。SQLite Online Backup 的 recovery 与 source 可以逻辑等价 +但物理字节不同，因此不能比较两者哈希来判断 clean rollback。记录返回的 `activationDigest`，并把 exact activation path/digest +交给 adopted application 和后续 cutover 命令。 + +## 6. Cutover 与回退判定 + +完成 activation 后，继续执行 [Edge/Standalone 部署准备](./ql3-local-deployment.md) 中的 legacy silence、target start/stop 和双阶段 +rollback ceremony。不要仅凭 activation 启动两个 writer。 + +target stop 后的数据分类为： + +- `rollback_candidate`：target/source 都保持 activation 时字节且双方 sidecar clear； +- `reconciliation_required`：target 已接受写入或存在 target sidecar；保留三份数据库，禁止自动启动 2.x； +- `manual_review`：source 漂移、activation 或稳定文件身份无法证明；禁止猜测。 + +证据字段 `sourceMatchesActivation` 表示 source 与 activation 时的原 source 哈希相同，不表示 source 与 Online Backup recovery 文件 +物理相同。`rollback_candidate` 也不是启动授权，仍须走双阶段 Legacy rollback。 + +## 7. 当前边界 + +本流程不迁移完整 2.x data directory，不做 target 写后的自动逆迁移，不连接 Cluster 控制面，也没有云端恢复服务。任何 +`reconciliation_required` 必须保留现场并等待后续显式数据域工具；不要把 target 覆盖回 source。 diff --git a/packages/ql3-local-admin/src/legacy-adoption/local-sqlite-adoption/activation.ts b/packages/ql3-local-admin/src/legacy-adoption/local-sqlite-adoption/activation.ts index 4db64c2b..bd6b928d 100644 --- a/packages/ql3-local-admin/src/legacy-adoption/local-sqlite-adoption/activation.ts +++ b/packages/ql3-local-admin/src/legacy-adoption/local-sqlite-adoption/activation.ts @@ -22,6 +22,7 @@ import { assertRealParent, assertRegularFile, fileIdentity, + sha256File, sha256Text, writeManifestAtomically, } from './filesystem'; @@ -48,6 +49,7 @@ function parseActivation(value: unknown): LocalSqliteActivation { 'recoverySha256', 'schemaVersion', 'sourcePathDigest', + 'sourceSha256', 'state', 'targetDevice', 'targetInode', @@ -65,6 +67,7 @@ function parseActivation(value: unknown): LocalSqliteActivation { !DIGEST_PATTERN.test(activation.adoptionManifestDigest ?? '') || !DIGEST_PATTERN.test(activation.planDigest ?? '') || !DIGEST_PATTERN.test(activation.sourcePathDigest ?? '') || + !DIGEST_PATTERN.test(activation.sourceSha256 ?? '') || !DIGEST_PATTERN.test(activation.recoverySha256 ?? '') || !DIGEST_PATTERN.test(activation.targetSha256 ?? '') || !DIGEST_PATTERN.test(activation.targetPathDigest ?? '') || @@ -198,6 +201,7 @@ export async function prepareLocalSqliteActivation( options.recoveryPath, adoption, ); + const sourceSha256 = await sha256File(options.sourcePath); const activationVerified = await verifyLocalSqliteAdoptionInternal( options, true, @@ -218,6 +222,7 @@ export async function prepareLocalSqliteActivation( adoptionManifestDigest: adoption.manifestDigest, planDigest: adoption.planDigest, sourcePathDigest: adoption.source.pathDigest, + sourceSha256, recoverySha256: adoption.recovery.sha256, targetSha256: adoption.target.sha256, targetPathDigest: activationVerified.targetIdentity.pathDigest, @@ -286,6 +291,11 @@ export async function acquireLocalSqliteActivation( options.recoveryPath, adoption, ); + if ((await sha256File(options.sourcePath)) !== activation.sourceSha256) { + throw new LocalSqliteAdoptionError( + 'legacy source bytes changed after activation preparation', + ); + } } catch (error) { releaseSourceWriteFence(fence); if (error instanceof LocalSqliteAdoptionError) throw error; diff --git a/packages/ql3-local-admin/src/legacy-adoption/local-sqlite-adoption/contracts.ts b/packages/ql3-local-admin/src/legacy-adoption/local-sqlite-adoption/contracts.ts index b1be1abc..7323e766 100644 --- a/packages/ql3-local-admin/src/legacy-adoption/local-sqlite-adoption/contracts.ts +++ b/packages/ql3-local-admin/src/legacy-adoption/local-sqlite-adoption/contracts.ts @@ -179,6 +179,7 @@ export interface LocalSqliteActivationPayload { readonly adoptionManifestDigest: string; readonly planDigest: string; readonly sourcePathDigest: string; + readonly sourceSha256: string; readonly recoverySha256: string; readonly targetSha256: string; readonly targetPathDigest: string; diff --git a/packages/ql3-local-owner-cli/src/deployment/cutover/legacyStop.ts b/packages/ql3-local-owner-cli/src/deployment/cutover/legacyStop.ts index aac60fa6..f9c227f4 100644 --- a/packages/ql3-local-owner-cli/src/deployment/cutover/legacyStop.ts +++ b/packages/ql3-local-owner-cli/src/deployment/cutover/legacyStop.ts @@ -107,6 +107,7 @@ function verifyActivation( 'recoverySha256', 'schemaVersion', 'sourcePathDigest', + 'sourceSha256', 'state', 'targetDevice', 'targetInode', @@ -129,6 +130,8 @@ function verifyActivation( if ( typeof activationDigest !== 'string' || !DIGEST_PATTERN.test(activationDigest) || + typeof activation.sourceSha256 !== 'string' || + !DIGEST_PATTERN.test(activation.sourceSha256) || digest(payload) !== activationDigest ) { configurationError('activation digest does not match'); diff --git a/packages/ql3-local-owner-cli/src/deployment/cutover/targetDataEvidence.ts b/packages/ql3-local-owner-cli/src/deployment/cutover/targetDataEvidence.ts index bf71d9f6..4b007fae 100644 --- a/packages/ql3-local-owner-cli/src/deployment/cutover/targetDataEvidence.ts +++ b/packages/ql3-local-owner-cli/src/deployment/cutover/targetDataEvidence.ts @@ -14,7 +14,7 @@ const HASH_BUFFER_BYTES = 64 * 1024; export interface TargetDataReconciliationEvidence { readonly disposition: LocalDeploymentTargetReconciliationDisposition; readonly targetMatchesActivation: boolean | null; - readonly sourceMatchesRecovery: boolean | null; + readonly sourceMatchesActivation: boolean | null; readonly targetSidecarsClear: boolean | null; readonly sourceSidecarsClear: boolean | null; readonly targetFileIdentityDigest: string; @@ -194,6 +194,8 @@ export function readTargetDataReconciliationEvidenceForPaths( !DIGEST_PATTERN.test(activation.targetSha256) || typeof activation.recoverySha256 !== 'string' || !DIGEST_PATTERN.test(activation.recoverySha256) || + typeof activation.sourceSha256 !== 'string' || + !DIGEST_PATTERN.test(activation.sourceSha256) || typeof activation.targetDevice !== 'string' || typeof activation.targetInode !== 'string' || cutoverDigest(payload) !== activationDigest @@ -218,17 +220,17 @@ export function readTargetDataReconciliationEvidenceForPaths( throw new Error('target database stable identity drifted'); } const targetMatchesActivation = target.sha256 === activation.targetSha256; - const sourceMatchesRecovery = source.sha256 === activation.recoverySha256; + const sourceMatchesActivation = source.sha256 === activation.sourceSha256; const disposition = !targetMatchesActivation || !target.sidecarsClear ? ('reconciliation_required' as const) - : sourceMatchesRecovery && source.sidecarsClear + : sourceMatchesActivation && source.sidecarsClear ? ('rollback_candidate' as const) : ('manual_review' as const); return evidence({ disposition, targetMatchesActivation, - sourceMatchesRecovery, + sourceMatchesActivation, targetSidecarsClear: target.sidecarsClear, sourceSidecarsClear: source.sidecarsClear, targetFileIdentityDigest: target.identityDigest, @@ -238,7 +240,7 @@ export function readTargetDataReconciliationEvidenceForPaths( return evidence({ disposition: 'manual_review', targetMatchesActivation: null, - sourceMatchesRecovery: null, + sourceMatchesActivation: null, targetSidecarsClear: null, sourceSidecarsClear: null, targetFileIdentityDigest: UNKNOWN_DIGEST, @@ -256,7 +258,7 @@ export function verifyTargetDataReconciliationEvidence( 'disposition', 'evidenceDigest', 'sourceFileIdentityDigest', - 'sourceMatchesRecovery', + 'sourceMatchesActivation', 'sourceSidecarsClear', 'targetFileIdentityDigest', 'targetMatchesActivation', @@ -270,8 +272,8 @@ export function verifyTargetDataReconciliationEvidence( candidate.disposition !== 'manual_review') || (candidate.targetMatchesActivation !== null && typeof candidate.targetMatchesActivation !== 'boolean') || - (candidate.sourceMatchesRecovery !== null && - typeof candidate.sourceMatchesRecovery !== 'boolean') || + (candidate.sourceMatchesActivation !== null && + typeof candidate.sourceMatchesActivation !== 'boolean') || (candidate.targetSidecarsClear !== null && typeof candidate.targetSidecarsClear !== 'boolean') || (candidate.sourceSidecarsClear !== null && diff --git a/packages/ql3-local-owner-cli/src/deployment/cutover/targetEvidence.ts b/packages/ql3-local-owner-cli/src/deployment/cutover/targetEvidence.ts index 144f07b3..5085663e 100644 --- a/packages/ql3-local-owner-cli/src/deployment/cutover/targetEvidence.ts +++ b/packages/ql3-local-owner-cli/src/deployment/cutover/targetEvidence.ts @@ -147,6 +147,7 @@ export function verifyTargetRunActivation( 'recoverySha256', 'schemaVersion', 'sourcePathDigest', + 'sourceSha256', 'state', 'targetDevice', 'targetInode', @@ -166,6 +167,8 @@ export function verifyTargetRunActivation( activationDigest !== command.request.expectedActivationDigest || typeof activationDigest !== 'string' || !DIGEST_PATTERN.test(activationDigest) || + typeof activation.sourceSha256 !== 'string' || + !DIGEST_PATTERN.test(activation.sourceSha256) || cutoverDigest(payload) !== activationDigest ) { configurationError('activation does not match the target run request'); diff --git a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverConsumer.ts b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverConsumer.ts index 19feccca..5120d3c8 100644 --- a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverConsumer.ts +++ b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverConsumer.ts @@ -404,6 +404,8 @@ function verifyAdoptedEvidence( activation.profile !== intent.profile || activation.sourcePathDigest !== textDigest(binding.sourcePath) || activation.targetPathDigest !== textDigest(binding.targetPath) || + typeof activation.sourceSha256 !== 'string' || + !DIGEST_PATTERN.test(activation.sourceSha256) || typeof activation.recoverySha256 !== 'string' || !DIGEST_PATTERN.test(activation.recoverySha256) || typeof activation.adoptionManifestDigest !== 'string' || @@ -451,7 +453,7 @@ function verifyAdoptedEvidence( if ( activation.targetDevice !== target.device || activation.targetInode !== target.inode || - sourceSha256 !== activation.recoverySha256 || + sourceSha256 !== activation.sourceSha256 || recoverySha256 !== activation.recoverySha256 ) { configurationError('adopted data evidence drifted'); diff --git a/packages/ql3-local-owner-cli/src/lifecycle/adoptionCli.ts b/packages/ql3-local-owner-cli/src/lifecycle/adoptionCli.ts index 054cc1e8..05a40b9a 100644 --- a/packages/ql3-local-owner-cli/src/lifecycle/adoptionCli.ts +++ b/packages/ql3-local-owner-cli/src/lifecycle/adoptionCli.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node // Keep the one-shot adoption binary beside its lifecycle command. -import { runLegacyCrontabAdoptionCommandFile } from './adoption'; +import { runLocalAdoptionProductCommandFile } from './adoptionCommand'; const USAGE = 'Usage: ql3-adoption run --command-file /absolute/private-command.json'; @@ -40,7 +40,7 @@ async function main(argv: readonly string[]): Promise { return; } try { - const result = await runLegacyCrontabAdoptionCommandFile(argv[2]!); + const result = await runLocalAdoptionProductCommandFile(argv[2]!); process.stdout.write(`${JSON.stringify(result)}\n`); } catch (error) { const candidate = error as { diff --git a/packages/ql3-local-owner-cli/src/lifecycle/adoptionCommand.ts b/packages/ql3-local-owner-cli/src/lifecycle/adoptionCommand.ts new file mode 100644 index 00000000..1df01ced --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/adoptionCommand.ts @@ -0,0 +1,42 @@ +import { readPrivateLocalCommandFile } from '@qinglong/local-command-file'; + +import { runLegacyCrontabAdoptionCommandFile } from './adoption'; +import { + isLocalSqliteAdoptionProductOperation, + type LocalSqliteAdoptionProductOperation, +} from './sqlite-adoption/contract'; +import type { LocalSqliteAdoptionProductCommandResult } from './sqlite-adoption/command'; + +export type LocalAdoptionProductCommandResult = + | Awaited> + | LocalSqliteAdoptionProductCommandResult; + +function operation(value: unknown): unknown { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as { readonly operation?: unknown }).operation + : undefined; +} + +export async function runLocalAdoptionProductCommandFile( + commandFilePath: string, +): Promise> { + let candidate: unknown; + try { + candidate = readPrivateLocalCommandFile(commandFilePath); + } catch { + // Preserve the established legacy error mapping for unreadable files. + return runLegacyCrontabAdoptionCommandFile(commandFilePath); + } + const selected = operation(candidate); + if (!isLocalSqliteAdoptionProductOperation(selected)) { + return runLegacyCrontabAdoptionCommandFile(commandFilePath); + } + const { runLocalSqliteAdoptionProductCommand } = await import( + './sqlite-adoption/command.js' + ); + return runLocalSqliteAdoptionProductCommand( + candidate as { + readonly operation: LocalSqliteAdoptionProductOperation; + }, + ); +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/command.ts b/packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/command.ts new file mode 100644 index 00000000..03e407b8 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/command.ts @@ -0,0 +1,431 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { + inspectLegacySqlitePath, + prepareLocalSqliteActivation, + stageLocalSqliteAdoption, + verifyLocalSqliteAdoption, + type LegacySqliteAdoptionPlan, + type LocalSqliteActivation, + type LocalSqliteAdoptionManifest, +} from '@qinglong/local-admin'; + +import { + LocalSqliteAdoptionCliConfigurationError, + normalizeLocalSqliteAdoptionProductCommand, + type LocalSqliteAdoptionProductCommand, +} from './contract'; + +interface StableFileIdentity { + readonly path: string; + readonly device: bigint; + readonly inode: bigint; + readonly size: bigint; + readonly modifiedAtNs: bigint; + readonly changedAtNs: bigint; + readonly mode: number; + readonly uid: number; +} + +interface StableDirectoryIdentity { + readonly path: string; + readonly device: bigint; + readonly inode: bigint; + readonly mode: number; + readonly uid: number; +} + +export type LocalSqliteAdoptionProductCommandResult = Readonly<{ + schemaVersion: 1; + operation: LocalSqliteAdoptionProductCommand['operation']; + status: 'inspected' | 'staged' | 'verified' | 'prepared'; + evidence: Readonly>; +}>; + +function currentUid(): number { + if ( + typeof process.getuid !== 'function' || + typeof process.geteuid !== 'function' || + process.getuid() !== process.geteuid() + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'real and effective POSIX users must match', + ); + } + return process.getuid(); +} + +function inside(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return ( + relative !== '' && + relative !== '..' && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) + ); +} + +function directoryIdentity( + directoryPath: string, + uid: number, + label: string, +): StableDirectoryIdentity { + let stat: fs.BigIntStats; + try { + stat = fs.lstatSync(directoryPath, { bigint: true }); + } catch (error) { + throw new LocalSqliteAdoptionCliConfigurationError( + `${label} is unavailable`, + error, + ); + } + const mode = Number(stat.mode) & 0o777; + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + Number(stat.uid) !== uid || + mode !== 0o700 || + fs.realpathSync(directoryPath) !== directoryPath + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + `${label} must be an owner-controlled 0700 canonical directory`, + ); + } + return Object.freeze({ + path: directoryPath, + device: stat.dev, + inode: stat.ino, + mode, + uid, + }); +} + +function fileIdentity( + filePath: string, + uid: number, + label: string, + requirePrivateMode: boolean, +): StableFileIdentity { + let stat: fs.BigIntStats; + try { + stat = fs.lstatSync(filePath, { bigint: true }); + } catch (error) { + throw new LocalSqliteAdoptionCliConfigurationError( + `${label} is unavailable`, + error, + ); + } + const mode = Number(stat.mode) & 0o777; + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1n || + Number(stat.uid) !== uid || + (requirePrivateMode ? mode !== 0o600 : (mode & 0o022) !== 0) || + fs.realpathSync(filePath) !== filePath || + stat.size < 1n + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + `${label} identity or mode is invalid`, + ); + } + return Object.freeze({ + path: filePath, + device: stat.dev, + inode: stat.ino, + size: stat.size, + modifiedAtNs: stat.mtimeNs, + changedAtNs: stat.ctimeNs, + mode, + uid, + }); +} + +function sameDirectory(expected: StableDirectoryIdentity): void { + const actual = directoryIdentity( + expected.path, + expected.uid, + 'authority directory', + ); + if ( + actual.device !== expected.device || + actual.inode !== expected.inode || + actual.mode !== expected.mode + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'authority directory changed during command execution', + ); + } +} + +function sameFile(expected: StableFileIdentity): void { + const actual = fileIdentity( + expected.path, + expected.uid, + 'authority file', + expected.mode === 0o600, + ); + if ( + actual.device !== expected.device || + actual.inode !== expected.inode || + actual.size !== expected.size || + actual.modifiedAtNs !== expected.modifiedAtNs || + actual.changedAtNs !== expected.changedAtNs || + actual.mode !== expected.mode + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'authority file changed during command execution', + ); + } +} + +function assertMissing(filePath: string, label: string): void { + try { + fs.lstatSync(filePath); + } catch (error) { + if ( + error && + typeof error === 'object' && + 'code' in error && + error.code === 'ENOENT' + ) { + return; + } + throw new LocalSqliteAdoptionCliConfigurationError( + `${label} cannot be inspected`, + error, + ); + } + throw new LocalSqliteAdoptionCliConfigurationError( + `${label} must not already exist`, + ); +} + +function authorityProof(command: Readonly): { + readonly uid: number; + verify(): void; + verifyCreated(paths: readonly string[]): void; +} { + const uid = currentUid(); + const options = command.options; + const root = directoryIdentity(options.deploymentRoot, uid, 'deploymentRoot'); + const sourcePaths = + 'sourcePath' in options ? [options.sourcePath] : ([] as string[]); + const immutablePaths = [ + ...sourcePaths, + ...(command.operation === 'local-sqlite.adoption.verify' || + command.operation === 'local-sqlite.activation.prepare' + ? [ + command.options.targetPath, + command.options.recoveryPath, + command.options.manifestPath, + ] + : []), + ]; + const files = immutablePaths.map((candidate) => + fileIdentity( + candidate, + uid, + candidate === ('sourcePath' in options ? options.sourcePath : undefined) + ? 'legacy source' + : 'adoption evidence', + candidate !== ('sourcePath' in options ? options.sourcePath : undefined), + ), + ); + const outputPaths = + command.operation === 'local-sqlite.adoption.stage' + ? [ + command.options.targetPath, + command.options.recoveryPath, + command.options.manifestPath, + ] + : command.operation === 'local-sqlite.activation.prepare' + ? [command.options.activationPath] + : []; + const outputDirectories = [...new Set(outputPaths.map(path.dirname))].map( + (directory) => { + if ( + !inside(options.deploymentRoot, directory) && + directory !== options.deploymentRoot + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'adoption outputs must remain inside deploymentRoot', + ); + } + return directoryIdentity(directory, uid, 'output directory'); + }, + ); + for (const outputPath of outputPaths) { + if (!inside(options.deploymentRoot, outputPath)) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'adoption outputs must remain inside deploymentRoot', + ); + } + assertMissing(outputPath, 'adoption output'); + } + const uniqueFiles = new Set( + files.map((entry) => `${entry.device}:${entry.inode}`), + ); + if (uniqueFiles.size !== files.length) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'adoption authority files must not share an inode', + ); + } + return Object.freeze({ + uid, + verify() { + if (currentUid() !== uid) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'POSIX user changed during command execution', + ); + } + sameDirectory(root); + for (const directory of outputDirectories) sameDirectory(directory); + for (const file of files) sameFile(file); + }, + verifyCreated(paths: readonly string[]) { + for (const filePath of paths) { + fileIdentity(filePath, uid, 'created adoption output', true); + } + }, + }); +} + +function planEvidence(plan: Readonly) { + return Object.freeze({ + profile: plan.profile, + planDigest: plan.planDigest, + source: Object.freeze({ + fileName: plan.source.fileName, + pathDigest: plan.source.pathDigest, + bytes: plan.source.bytes, + }), + catalog: Object.freeze({ + digest: plan.catalog.digest, + objectCount: plan.catalog.objectCount, + tableCount: plan.catalog.tableNames.length, + tableNames: plan.catalog.tableNames, + }), + tasks: plan.tasks, + }); +} + +function adoptionEvidence(manifest: Readonly) { + return Object.freeze({ + profile: manifest.profile, + planDigest: manifest.planDigest, + manifestDigest: manifest.manifestDigest, + createdAtMs: manifest.createdAtMs, + source: Object.freeze({ + fileName: manifest.source.fileName, + pathDigest: manifest.source.pathDigest, + bytes: manifest.source.bytes, + }), + catalog: Object.freeze({ + digest: manifest.catalog.digest, + objectCount: manifest.catalog.objectCount, + tableCount: manifest.catalog.tableNames.length, + }), + tasks: manifest.tasks, + recovery: manifest.recovery, + target: manifest.target, + readiness: manifest.readiness, + }); +} + +function activationEvidence(activation: Readonly) { + return Object.freeze({ ...activation }); +} + +export async function runLocalSqliteAdoptionProductCommand( + value: unknown, +): Promise { + const command = normalizeLocalSqliteAdoptionProductCommand(value); + const proof = authorityProof(command); + if (command.operation === 'local-sqlite.adoption.inspect') { + const options = command.options; + const plan = inspectLegacySqlitePath({ + sourcePath: options.sourcePath, + profile: options.profile, + ...(options.legacyTimezone === undefined + ? {} + : { legacyTimezone: options.legacyTimezone }), + }); + proof.verify(); + return Object.freeze({ + schemaVersion: 1, + operation: command.operation, + status: 'inspected', + evidence: planEvidence(plan), + }); + } + if (command.operation === 'local-sqlite.adoption.stage') { + const options = command.options; + const manifest = await stageLocalSqliteAdoption({ + sourcePath: options.sourcePath, + targetPath: options.targetPath, + recoveryPath: options.recoveryPath, + manifestPath: options.manifestPath, + profile: options.profile, + expectedPlanDigest: options.expectedPlanDigest, + ...(options.legacyTimezone === undefined + ? {} + : { legacyTimezone: options.legacyTimezone }), + }); + proof.verify(); + proof.verifyCreated([ + options.targetPath, + options.recoveryPath, + options.manifestPath, + ]); + return Object.freeze({ + schemaVersion: 1, + operation: command.operation, + status: 'staged', + evidence: adoptionEvidence(manifest), + }); + } + if (command.operation === 'local-sqlite.adoption.verify') { + const options = command.options; + const manifest = await verifyLocalSqliteAdoption({ + targetPath: options.targetPath, + recoveryPath: options.recoveryPath, + manifestPath: options.manifestPath, + }); + if (manifest.profile !== options.profile) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'verified adoption profile drifted', + ); + } + proof.verify(); + return Object.freeze({ + schemaVersion: 1, + operation: command.operation, + status: 'verified', + evidence: adoptionEvidence(manifest), + }); + } + const options = command.options; + const activation = await prepareLocalSqliteActivation({ + sourcePath: options.sourcePath, + targetPath: options.targetPath, + recoveryPath: options.recoveryPath, + manifestPath: options.manifestPath, + activationPath: options.activationPath, + expectedManifestDigest: options.expectedManifestDigest, + }); + if (activation.profile !== options.profile) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'prepared activation profile drifted', + ); + } + proof.verify(); + proof.verifyCreated([options.activationPath]); + return Object.freeze({ + schemaVersion: 1, + operation: command.operation, + status: 'prepared', + evidence: activationEvidence(activation), + }); +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/contract.ts b/packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/contract.ts new file mode 100644 index 00000000..b6d30b08 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/contract.ts @@ -0,0 +1,244 @@ +import path from 'node:path'; + +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const MAX_PATH_BYTES = 4_096; + +export type LocalSqliteAdoptionProductOperation = + | 'local-sqlite.adoption.inspect' + | 'local-sqlite.adoption.stage' + | 'local-sqlite.adoption.verify' + | 'local-sqlite.activation.prepare'; + +interface LocalSqliteAdoptionCommandOptionsBase { + readonly deploymentRoot: string; + readonly profile: 'edge' | 'standalone'; +} + +export interface InspectLocalSqliteAdoptionCommand { + readonly schemaVersion: 1; + readonly operation: 'local-sqlite.adoption.inspect'; + readonly options: LocalSqliteAdoptionCommandOptionsBase & { + readonly sourcePath: string; + readonly legacyTimezone?: string; + }; +} + +export interface StageLocalSqliteAdoptionCommand { + readonly schemaVersion: 1; + readonly operation: 'local-sqlite.adoption.stage'; + readonly options: LocalSqliteAdoptionCommandOptionsBase & { + readonly sourcePath: string; + readonly targetPath: string; + readonly recoveryPath: string; + readonly manifestPath: string; + readonly expectedPlanDigest: string; + readonly legacyTimezone?: string; + }; +} + +export interface VerifyLocalSqliteAdoptionCommand { + readonly schemaVersion: 1; + readonly operation: 'local-sqlite.adoption.verify'; + readonly options: LocalSqliteAdoptionCommandOptionsBase & { + readonly targetPath: string; + readonly recoveryPath: string; + readonly manifestPath: string; + }; +} + +export interface PrepareLocalSqliteActivationCommand { + readonly schemaVersion: 1; + readonly operation: 'local-sqlite.activation.prepare'; + readonly options: LocalSqliteAdoptionCommandOptionsBase & { + readonly sourcePath: string; + readonly targetPath: string; + readonly recoveryPath: string; + readonly manifestPath: string; + readonly activationPath: string; + readonly expectedManifestDigest: string; + }; +} + +export type LocalSqliteAdoptionProductCommand = + | InspectLocalSqliteAdoptionCommand + | StageLocalSqliteAdoptionCommand + | VerifyLocalSqliteAdoptionCommand + | PrepareLocalSqliteActivationCommand; + +export class LocalSqliteAdoptionCliConfigurationError extends TypeError { + readonly code = 'LOCAL_SQLITE_ADOPTION_CLI_CONFIGURATION_INVALID'; + + constructor(message: string, readonly cause?: unknown) { + super(`Local SQLite adoption CLI configuration is invalid: ${message}`); + this.name = 'LocalSqliteAdoptionCliConfigurationError'; + } +} + +function exactKeys(value: object, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const canonical = [...expected].sort(); + return ( + actual.length === canonical.length && + actual.every((key, index) => key === canonical[index]) + ); +} + +function boundedPath(value: unknown, label: string): string { + if ( + typeof value !== 'string' || + !path.isAbsolute(value) || + path.parse(value).root === value || + path.normalize(value) !== value || + value.includes('\0') || + Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + `${label} must be a normalized bounded absolute non-root path`, + ); + } + return value; +} + +function optionalTimezone(options: Record): string[] { + return options.legacyTimezone === undefined ? [] : ['legacyTimezone']; +} + +function assertTimezone(value: unknown): void { + if ( + value !== undefined && + (typeof value !== 'string' || + value.length < 1 || + value.length > 128 || + /[\0\r\n]/.test(value)) + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'legacyTimezone is invalid', + ); + } +} + +function assertDistinct(paths: readonly string[]): void { + const resolved = paths.map((value) => path.resolve(value)); + if (new Set(resolved).size !== resolved.length) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'SQLite adoption paths must be distinct', + ); + } +} + +export function isLocalSqliteAdoptionProductOperation( + value: unknown, +): value is LocalSqliteAdoptionProductOperation { + return ( + value === 'local-sqlite.adoption.inspect' || + value === 'local-sqlite.adoption.stage' || + value === 'local-sqlite.adoption.verify' || + value === 'local-sqlite.activation.prepare' + ); +} + +export function normalizeLocalSqliteAdoptionProductCommand( + value: unknown, +): Readonly { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + !exactKeys(value, ['schemaVersion', 'operation', 'options']) + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'command shape is invalid', + ); + } + const candidate = value as Record; + if ( + candidate.schemaVersion !== 1 || + !isLocalSqliteAdoptionProductOperation(candidate.operation) || + !candidate.options || + typeof candidate.options !== 'object' || + Array.isArray(candidate.options) + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'command value is invalid', + ); + } + const options = candidate.options as Record; + const expected = + candidate.operation === 'local-sqlite.adoption.inspect' + ? [ + 'deploymentRoot', + ...optionalTimezone(options), + 'profile', + 'sourcePath', + ] + : candidate.operation === 'local-sqlite.adoption.stage' + ? [ + 'deploymentRoot', + 'expectedPlanDigest', + ...optionalTimezone(options), + 'manifestPath', + 'profile', + 'recoveryPath', + 'sourcePath', + 'targetPath', + ] + : candidate.operation === 'local-sqlite.adoption.verify' + ? [ + 'deploymentRoot', + 'manifestPath', + 'profile', + 'recoveryPath', + 'targetPath', + ] + : [ + 'activationPath', + 'deploymentRoot', + 'expectedManifestDigest', + 'manifestPath', + 'profile', + 'recoveryPath', + 'sourcePath', + 'targetPath', + ]; + if ( + !exactKeys(options, expected) || + (options.profile !== 'edge' && options.profile !== 'standalone') + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'command options are invalid', + ); + } + assertTimezone(options.legacyTimezone); + boundedPath(options.deploymentRoot, 'deploymentRoot'); + for (const key of expected.filter((name) => name.endsWith('Path'))) { + boundedPath(options[key], key); + } + if ( + candidate.operation === 'local-sqlite.adoption.stage' && + (typeof options.expectedPlanDigest !== 'string' || + !DIGEST_PATTERN.test(options.expectedPlanDigest)) + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'expectedPlanDigest is invalid', + ); + } + if ( + candidate.operation === 'local-sqlite.activation.prepare' && + (typeof options.expectedManifestDigest !== 'string' || + !DIGEST_PATTERN.test(options.expectedManifestDigest)) + ) { + throw new LocalSqliteAdoptionCliConfigurationError( + 'expectedManifestDigest is invalid', + ); + } + assertDistinct( + [ + options.sourcePath, + options.targetPath, + options.recoveryPath, + options.manifestPath, + options.activationPath, + ].filter((entry): entry is string => typeof entry === 'string'), + ); + return Object.freeze(value as LocalSqliteAdoptionProductCommand); +} diff --git a/packages/ql3-local-owner-cli/test/cutoverTargetRun.test.cjs b/packages/ql3-local-owner-cli/test/cutoverTargetRun.test.cjs index 11db98a8..3493e3d1 100644 --- a/packages/ql3-local-owner-cli/test/cutoverTargetRun.test.cjs +++ b/packages/ql3-local-owner-cli/test/cutoverTargetRun.test.cjs @@ -154,6 +154,10 @@ function fixture(t) { .createHash('sha256') .update(legacySourcePath, 'utf8') .digest('hex'), + sourceSha256: crypto + .createHash('sha256') + .update(fs.readFileSync(legacySourcePath)) + .digest('hex'), recoverySha256: crypto .createHash('sha256') .update(fs.readFileSync(recoveryPath)) @@ -917,7 +921,7 @@ test('stops an active target and proves an unchanged rollback candidate', async assert.equal(request.state, 'target_stop_requested'); assert.equal(outcome.state, 'target_stopped'); assert.equal(outcome.evidence.reconciliation.targetMatchesActivation, true); - assert.equal(outcome.evidence.reconciliation.sourceMatchesRecovery, true); + assert.equal(outcome.evidence.reconciliation.sourceMatchesActivation, true); const replay = stopLocalDeploymentDockerTarget(stopCommand(state), { validateSocket() { @@ -1304,7 +1308,7 @@ test('treats target SQLite sidecars as reconciliation-required', async (t) => { assert.equal(outcome.evidence.reconciliation.targetSidecarsClear, false); }); -test('requires manual review when legacy source no longer matches recovery', async (t) => { +test('requires manual review when legacy source no longer matches activation', async (t) => { const state = fixture(t); await runLocalDeploymentDockerTarget(command(state), harness(state)); fs.appendFileSync(state.legacySourcePath, 'offline-legacy-drift\n'); @@ -1320,7 +1324,7 @@ test('requires manual review when legacy source no longer matches recovery', asy ), ); assert.equal(outcome.evidence.reconciliation.targetMatchesActivation, true); - assert.equal(outcome.evidence.reconciliation.sourceMatchesRecovery, false); + assert.equal(outcome.evidence.reconciliation.sourceMatchesActivation, false); }); test('recovers a crash after the stop barrier by converging stop-and-inspect', async (t) => { diff --git a/packages/ql3-local-owner-cli/test/localDeployment.test.cjs b/packages/ql3-local-owner-cli/test/localDeployment.test.cjs index c4f09827..3e9d1d1e 100644 --- a/packages/ql3-local-owner-cli/test/localDeployment.test.cjs +++ b/packages/ql3-local-owner-cli/test/localDeployment.test.cjs @@ -168,6 +168,10 @@ function legacyStopCommand(state, cutoverId = 'cutover-edge-1') { .createHash('sha256') .update(path.resolve(legacySourcePath), 'utf8') .digest('hex'), + sourceSha256: crypto + .createHash('sha256') + .update(fs.readFileSync(legacySourcePath)) + .digest('hex'), recoverySha256: '4'.repeat(64), targetSha256: '5'.repeat(64), targetPathDigest: '6'.repeat(64), diff --git a/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs b/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs index 01828bd7..4c002f86 100644 --- a/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs +++ b/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs @@ -90,6 +90,7 @@ function fixture(t) { adoptionManifestDigest: manifestDigest, planDigest: '2'.repeat(64), sourcePathDigest: sha256(sourcePath), + sourceSha256: sha256(fs.readFileSync(sourcePath)), recoverySha256: sha256(fs.readFileSync(recoveryPath)), targetSha256: sha256(fs.readFileSync(targetPath)), targetPathDigest: sha256(targetPath), @@ -694,7 +695,7 @@ test('prepares and exactly replays lossless service-manager legacy rollback evid ); assert.equal(record.stoppedRecordDigest, stoppedResult.recordDigest); assert.equal(record.reconciliation.targetMatchesActivation, true); - assert.equal(record.reconciliation.sourceMatchesRecovery, true); + assert.equal(record.reconciliation.sourceMatchesActivation, true); assert.equal(record.reconciliation.targetSidecarsClear, true); assert.equal(record.reconciliation.sourceSidecarsClear, true); const replay = prepareLocalServiceManagerLegacyRollback(command); diff --git a/packages/ql3-local-owner-cli/test/sqliteAdoptionCli.test.cjs b/packages/ql3-local-owner-cli/test/sqliteAdoptionCli.test.cjs new file mode 100644 index 00000000..fc4fef81 --- /dev/null +++ b/packages/ql3-local-owner-cli/test/sqliteAdoptionCli.test.cjs @@ -0,0 +1,428 @@ +const assert = require('node:assert/strict'); +const { createHash } = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const { test } = require('node:test'); + +const { + bootstrapLocalAdoptedProfileStorage, +} = require('@qinglong/local-admin/adopted-profile'); +const { + readTargetDataReconciliationEvidenceForPaths, +} = require('../dist/deployment/cutover/targetDataEvidence'); + +const BINARY = path.join(__dirname, '../dist/lifecycle/adoptionCli.js'); + +function sha256(filePath) { + return createHash('sha256').update(fs.readFileSync(filePath)).digest('hex'); +} + +function createLegacyDatabase(sourcePath) { + const source = new DatabaseSync(sourcePath); + source.exec(` + CREATE TABLE "Crontabs" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), + command VARCHAR(255), schedule VARCHAR(255), timestamp VARCHAR(255), + saved TINYINT(1), status DECIMAL, isSystem DECIMAL, pid DECIMAL, + isDisabled DECIMAL, isPinned DECIMAL, log_path VARCHAR(255), labels JSON, + last_running_time DECIMAL, last_execution_time DECIMAL, sub_id DECIMAL, + extra_schedules JSON, task_before VARCHAR(255), task_after VARCHAR(255), + log_name VARCHAR(255), allow_multiple_instances DECIMAL, + work_dir VARCHAR(255), createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "Dependences" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), type DECIMAL, + timestamp VARCHAR(255), status DECIMAL, log JSON, remark VARCHAR(255), + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "Apps" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), scopes JSON, + client_id VARCHAR(255), client_secret VARCHAR(255), tokens JSON, + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "Auths" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, ip VARCHAR(255), type VARCHAR(255), + info JSON, createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "Envs" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, value VARCHAR(255), + timestamp VARCHAR(255), status DECIMAL, position DECIMAL, + name VARCHAR(255), remarks VARCHAR(255), isPinned DECIMAL, labels JSON, + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "Subscriptions" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), url VARCHAR(255), + schedule VARCHAR(255), interval_schedule JSON, type VARCHAR(255), + whitelist VARCHAR(255), blacklist VARCHAR(255), status DECIMAL, + dependences VARCHAR(255), extensions VARCHAR(255), sub_before VARCHAR(255), + sub_after VARCHAR(255), branch VARCHAR(255), pull_type VARCHAR(255), + pull_option JSON, pid DECIMAL, is_disabled DECIMAL, log_path VARCHAR(255), + schedule_type VARCHAR(255), alias VARCHAR(255), proxy VARCHAR(255), + autoAddCron DECIMAL, autoDelCron DECIMAL, + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "CrontabViews" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), position DECIMAL, + isDisabled DECIMAL, filters JSON, sorts JSON, filterRelation VARCHAR(255), + type DECIMAL, createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "CrontabStats" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, ref_id DECIMAL NOT NULL, + date VARCHAR(255) NOT NULL, run_count DECIMAL, success_count DECIMAL, + fail_count DECIMAL, total_time DECIMAL, max_time DECIMAL, + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "RunningInstances" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, cron_id DECIMAL NOT NULL, + run_id VARCHAR(36), attempt_id VARCHAR(36), pid DECIMAL, + log_path VARCHAR(255), started_at DECIMAL NOT NULL, finished_at DECIMAL, + status DECIMAL NOT NULL, exit_code DECIMAL, + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "PluginOwnedState" ( + id INTEGER PRIMARY KEY, payload TEXT NOT NULL + ); + INSERT INTO "Crontabs" ( + id, name, command, schedule, status, isDisabled, isPinned, + createdAt, updatedAt + ) VALUES ( + 1, 'Production-shaped legacy task', 'task /scripts/legacy.sh', + '0 0 * * *', 1, 0, 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ); + INSERT INTO "Envs" ( + id, name, value, status, position, createdAt, updatedAt + ) VALUES ( + 1, 'LEGACY_VALUE', 'preserved', 0, 100, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ); + INSERT INTO "Auths" ( + id, type, info, createdAt, updatedAt + ) VALUES ( + 1, 'systemConfig', '{"timezone":"UTC"}', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ); + INSERT INTO "Apps" ( + id, name, scopes, client_id, client_secret, createdAt, updatedAt + ) VALUES ( + 1, 'legacy-app', '["crons"]', 'legacy-client', 'legacy-secret', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ); + INSERT INTO "Subscriptions" ( + id, name, alias, url, schedule, createdAt, updatedAt + ) VALUES ( + 1, 'legacy-subscription', 'legacy-subscription', + 'https://example.invalid/repo.git', '0 1 * * *', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ); + INSERT INTO "PluginOwnedState" (id, payload) + VALUES (1, '{"preserved":true}'); + `); + source.close(); + fs.chmodSync(sourcePath, 0o600); +} + +function fixture(t) { + const deploymentRoot = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-sqlite-upgrade-')), + ); + fs.chmodSync(deploymentRoot, 0o700); + t.after(() => fs.rmSync(deploymentRoot, { recursive: true, force: true })); + const commandsDirectory = path.join(deploymentRoot, 'commands'); + const artifactsDirectory = path.join(deploymentRoot, 'artifacts'); + fs.mkdirSync(commandsDirectory, { mode: 0o700 }); + fs.mkdirSync(artifactsDirectory, { mode: 0o700 }); + const value = { + deploymentRoot, + commandsDirectory, + sourcePath: path.join(deploymentRoot, 'database.sqlite'), + targetPath: path.join(artifactsDirectory, 'qinglong3.sqlite'), + recoveryPath: path.join(artifactsDirectory, 'database.pre-ql3.sqlite'), + manifestPath: path.join(artifactsDirectory, 'qinglong3-adoption.json'), + activationPath: path.join(artifactsDirectory, 'qinglong3-activation.json'), + }; + createLegacyDatabase(value.sourcePath); + return value; +} + +function runCommand(value, name, operation, options) { + const commandPath = path.join(value.commandsDirectory, `${name}.json`); + fs.writeFileSync( + commandPath, + `${JSON.stringify({ schemaVersion: 1, operation, options })}\n`, + { mode: 0o600 }, + ); + const child = spawnSync( + process.execPath, + [BINARY, 'run', '--command-file', commandPath], + { encoding: 'utf8' }, + ); + assert.equal(child.status, 0, child.stderr); + assert.equal(child.stderr, ''); + return JSON.parse(child.stdout); +} + +function baseOptions(value) { + return { + deploymentRoot: value.deploymentRoot, + profile: 'edge', + }; +} + +function prepareAdoption(t) { + const value = fixture(t); + const sourceBefore = sha256(value.sourcePath); + const inspected = runCommand( + value, + 'inspect', + 'local-sqlite.adoption.inspect', + { + ...baseOptions(value), + sourcePath: value.sourcePath, + legacyTimezone: 'UTC', + }, + ); + const staged = runCommand(value, 'stage', 'local-sqlite.adoption.stage', { + ...baseOptions(value), + sourcePath: value.sourcePath, + targetPath: value.targetPath, + recoveryPath: value.recoveryPath, + manifestPath: value.manifestPath, + expectedPlanDigest: inspected.evidence.planDigest, + legacyTimezone: 'UTC', + }); + const verified = runCommand(value, 'verify', 'local-sqlite.adoption.verify', { + ...baseOptions(value), + targetPath: value.targetPath, + recoveryPath: value.recoveryPath, + manifestPath: value.manifestPath, + }); + const prepared = runCommand( + value, + 'prepare', + 'local-sqlite.activation.prepare', + { + ...baseOptions(value), + sourcePath: value.sourcePath, + targetPath: value.targetPath, + recoveryPath: value.recoveryPath, + manifestPath: value.manifestPath, + activationPath: value.activationPath, + expectedManifestDigest: verified.evidence.manifestDigest, + }, + ); + assert.equal(inspected.status, 'inspected'); + assert.deepEqual(inspected.evidence.catalog.tableNames, [ + 'Apps', + 'Auths', + 'CrontabStats', + 'CrontabViews', + 'Crontabs', + 'Dependences', + 'Envs', + 'PluginOwnedState', + 'RunningInstances', + 'Subscriptions', + 'sqlite_sequence', + ]); + assert.equal(staged.status, 'staged'); + assert.equal(verified.status, 'verified'); + assert.equal(prepared.status, 'prepared'); + assert.equal(sha256(value.sourcePath), sourceBefore); + for (const outputPath of [ + value.targetPath, + value.recoveryPath, + value.manifestPath, + value.activationPath, + ]) { + assert.equal(fs.statSync(outputPath).mode & 0o777, 0o600); + } + return { + ...value, + sourceBefore, + activationDigest: prepared.evidence.activationDigest, + }; +} + +async function startAdoptedStorage(value) { + return bootstrapLocalAdoptedProfileStorage({ + enabled: true, + profile: 'edge', + sourcePath: value.sourcePath, + targetPath: value.targetPath, + recoveryPath: value.recoveryPath, + manifestPath: value.manifestPath, + activationPath: value.activationPath, + expectedActivationDigest: value.activationDigest, + busyTimeoutMs: 100, + audit() {}, + adoptionAudit() {}, + }); +} + +function reconciliation(value) { + return readTargetDataReconciliationEvidenceForPaths( + { + profile: 'edge', + activationPath: value.activationPath, + legacySourcePath: value.sourcePath, + targetDatabasePath: value.targetPath, + expectedActivationDigest: value.activationDigest, + }, + process.getuid(), + ); +} + +test('upgrades a production-shaped 2.x SQLite database and admits clean rollback', async (t) => { + const value = prepareAdoption(t); + const recovery = new DatabaseSync(value.recoveryPath, { readOnly: true }); + assert.equal( + recovery + .prepare( + `SELECT COUNT(*) AS count FROM sqlite_master + WHERE name LIKE 'QingLong3%'`, + ) + .get().count, + 0, + ); + assert.equal( + recovery.prepare('SELECT payload FROM "PluginOwnedState"').get().payload, + '{"preserved":true}', + ); + recovery.close(); + + const target = new DatabaseSync(value.targetPath, { readOnly: true }); + assert.equal( + target.prepare('SELECT COUNT(*) AS count FROM "Crontabs"').get().count, + 1, + ); + assert.equal( + target.prepare('SELECT value FROM "Envs" WHERE id = 1').get().value, + 'preserved', + ); + assert.ok( + target + .prepare( + `SELECT COUNT(*) AS count FROM sqlite_master + WHERE type = 'table' AND name LIKE 'QingLong3%'`, + ) + .get().count > 0, + ); + target.close(); + + const storage = await startAdoptedStorage(value); + assert.equal(storage.status, 'adopted_storage_ready'); + const legacyWriter = new DatabaseSync(value.sourcePath, { timeout: 100 }); + assert.throws( + () => + legacyWriter + .prepare( + `INSERT INTO "Envs" ( + name, value, createdAt, updatedAt + ) VALUES ('BLOCKED', 'blocked', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + ) + .run(), + (error) => error && error.errstr === 'database is locked', + ); + await storage.stop(); + const cleanEvidence = reconciliation(value); + assert.equal( + cleanEvidence.disposition, + 'rollback_candidate', + JSON.stringify(cleanEvidence), + ); + legacyWriter + .prepare( + `INSERT INTO "Envs" ( + name, value, createdAt, updatedAt + ) VALUES ('AFTER_ROLLBACK', 'released', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + ) + .run(); + legacyWriter.close(); + const rolledBack = new DatabaseSync(value.sourcePath, { readOnly: true }); + assert.equal( + rolledBack + .prepare(`SELECT value FROM "Envs" WHERE name = 'AFTER_ROLLBACK'`) + .get().value, + 'released', + ); + rolledBack.close(); +}); + +test('requires reconciliation after the adopted target accepts a Run', async (t) => { + const value = prepareAdoption(t); + const storage = await startAdoptedStorage(value); + assert.equal(storage.status, 'adopted_storage_ready'); + await storage.runs.transaction((transaction) => + transaction.insertRun({ + id: '019f9a00-0000-4000-a000-000000000383', + projectId: 'default', + taskId: 'legacy-cron:1', + taskRevision: 'revision-1', + taskName: 'D383 reconciliation proof', + legacyCronId: 1, + triggerType: 'manual', + executionOrigin: 'manual', + executionOwner: 'runtime', + triggeredBy: 'user:1', + status: 'created', + version: 0, + eventSequence: 0, + priority: 0, + createdAtMs: 1_760_000_000_383, + }), + ); + await storage.stop(); + + const evidence = reconciliation(value); + assert.equal(evidence.disposition, 'reconciliation_required'); + assert.equal(evidence.targetMatchesActivation, false); + assert.equal( + evidence.sourceMatchesActivation, + true, + JSON.stringify(evidence), + ); + assert.equal(sha256(value.sourcePath), value.sourceBefore); + const source = new DatabaseSync(value.sourcePath, { readOnly: true }); + assert.equal( + source + .prepare( + `SELECT COUNT(*) AS count FROM sqlite_master + WHERE name = 'QingLong3Runs'`, + ) + .get().count, + 0, + ); + source.close(); +}); + +test('rejects widened SQLite adoption command intent before inspection', (t) => { + const value = fixture(t); + const commandPath = path.join(value.commandsDirectory, 'widened.json'); + fs.writeFileSync( + commandPath, + `${JSON.stringify({ + schemaVersion: 1, + operation: 'local-sqlite.adoption.inspect', + options: { + ...baseOptions(value), + sourcePath: value.sourcePath, + extraAuthority: true, + }, + })}\n`, + { mode: 0o600 }, + ); + const child = spawnSync( + process.execPath, + [BINARY, 'run', '--command-file', commandPath], + { encoding: 'utf8' }, + ); + assert.equal(child.status, 1); + assert.equal(child.stdout, ''); + assert.equal( + JSON.parse(child.stderr).code, + 'LOCAL_SQLITE_ADOPTION_CLI_CONFIGURATION_INVALID', + ); +}); diff --git a/scripts/ql3-cluster-dependency-audit.cjs b/scripts/ql3-cluster-dependency-audit.cjs index a47d03c8..1fe840f5 100644 --- a/scripts/ql3-cluster-dependency-audit.cjs +++ b/scripts/ql3-cluster-dependency-audit.cjs @@ -1757,6 +1757,11 @@ function auditSourceImports(root, packagePath, findings) { '@qinglong/local-sqlite/bootstrap', ].includes(specifier) ) && + !( + path.relative(packageDirectory, filePath) === + 'src/lifecycle/sqlite-adoption/command.ts' && + specifier === '@qinglong/local-admin' + ) && !( path.relative(packageDirectory, filePath) === 'src/plugin-package/pluginPackageCommand.ts' && diff --git a/scripts/ql3-service-manager-bridge-live-actor.cjs b/scripts/ql3-service-manager-bridge-live-actor.cjs index 90066522..eb7bd569 100644 --- a/scripts/ql3-service-manager-bridge-live-actor.cjs +++ b/scripts/ql3-service-manager-bridge-live-actor.cjs @@ -138,8 +138,7 @@ function descriptor(kind, uid, gid, root, adopted = false) { function legacyDescriptor(kind, uid, gid, port) { const nodeExecutable = fs.realpathSync(process.execPath); - const commandArgs = - `/workspace/scripts/ql3-service-manager-legacy-live-service.cjs ${port}`; + const commandArgs = `/workspace/scripts/ql3-service-manager-legacy-live-service.cjs ${port}`; if (kind === 'systemd') { return [ '[Unit]', @@ -298,6 +297,10 @@ function adoptedFixture(root, kind, uid, gid) { .createHash('sha256') .update(sourcePath) .digest('hex'), + sourceSha256: crypto + .createHash('sha256') + .update(fs.readFileSync(sourcePath)) + .digest('hex'), recoverySha256: crypto .createHash('sha256') .update(fs.readFileSync(recoveryPath)) diff --git a/test/back/ql3ClusterDependencyAudit.test.cjs b/test/back/ql3ClusterDependencyAudit.test.cjs index f27eaa14..4becb1d7 100644 --- a/test/back/ql3ClusterDependencyAudit.test.cjs +++ b/test/back/ql3ClusterDependencyAudit.test.cjs @@ -1786,8 +1786,13 @@ test('confines adoption and Package command authorities to owner CLI subpaths', ); const sourceDirectory = path.join(root, 'packages/ql3-local-owner-cli/src'); const lifecycleDirectory = path.join(sourceDirectory, 'lifecycle'); + const sqliteAdoptionDirectory = path.join( + lifecycleDirectory, + 'sqlite-adoption', + ); const pluginPackageDirectory = path.join(sourceDirectory, 'plugin-package'); fs.mkdirSync(lifecycleDirectory, { recursive: true }); + fs.mkdirSync(sqliteAdoptionDirectory, { recursive: true }); fs.mkdirSync(pluginPackageDirectory, { recursive: true }); fs.writeFileSync( path.join(lifecycleDirectory, 'adoption.ts'), @@ -1799,6 +1804,13 @@ test('confines adoption and Package command authorities to owner CLI subpaths', "import { forbidden } from '@qinglong/local-admin/runtime';", ].join('\n'), ); + fs.writeFileSync( + path.join(sqliteAdoptionDirectory, 'command.ts'), + [ + "import { inspectLegacySqlitePath } from '@qinglong/local-admin';", + "import { forbidden } from '@qinglong/local-admin/runtime';", + ].join('\n'), + ); const applicationCommandDirectory = path.join( sourceDirectory, 'application-command', @@ -1866,6 +1878,12 @@ test('confines adoption and Package command authorities to owner CLI subpaths', file: 'packages/ql3-local-owner-cli/src/lifecycle/adoption.ts', specifier: '@qinglong/local-admin/runtime', }, + { + code: 'FORBIDDEN_LOCAL_ADOPTION_CLI_AUTHORITY_IMPORT', + file: + 'packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/command.ts', + specifier: '@qinglong/local-admin/runtime', + }, { code: 'FORBIDDEN_LOCAL_ADOPTION_CLI_AUTHORITY_IMPORT', file: 'packages/ql3-local-owner-cli/src/unrelated.ts', diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index 06f37e3b..c7448f48 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -207,10 +207,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: localOwnerCli.rootSourceFileRoles, }, { - sourceFiles: 113, + sourceFiles: 116, rootSourceFiles: 1, rootSourceLines: 50, - nestedSourceFiles: 112, + nestedSourceFiles: 115, rootSourceFileRoles: { 'cli.ts': 'binary_entry' }, }, );