From 19bb09faa37bf85a5218a57813c864b178f17bdd Mon Sep 17 00:00:00 2001 From: whyour Date: Fri, 21 Aug 2026 10:09:34 +0800 Subject: [PATCH] feat(ql3): stage legacy data directory --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 19 + ...8-private-legacy-data-directory-staging.md | 133 +++++ docs/adr/README.md | 1 + .../ql3-local-data-directory-adoption.md | 97 +++- .../src/lifecycle/adoptionCommand.ts | 10 +- .../data-directory-adoption/command.ts | 31 ++ .../data-directory-adoption/contract.ts | 178 ++++++- .../data-directory-adoption/filesystem.ts | 430 ++++++++++++++++ .../data-directory-adoption/manifest.ts | 437 ++++++++++++++++ .../data-directory-adoption/staging.ts | 319 ++++++++++++ .../dataDirectoryAdoptionStageCli.test.cjs | 481 ++++++++++++++++++ scripts/ql3-cluster-dependency-audit.cjs | 5 + test/back/ql3ClusterDependencyAudit.test.cjs | 20 +- test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 14 files changed, 2139 insertions(+), 26 deletions(-) create mode 100644 docs/adr/ADR-0478-private-legacy-data-directory-staging.md create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/command.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/filesystem.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/manifest.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/staging.ts create mode 100644 packages/ql3-local-owner-cli/test/dataDirectoryAdoptionStageCli.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 27ac8c7a..cee9baf0 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,25 @@ 最新增量证据（2026-08-21）： +- D-385/ADR-0478（已接受）：把 D-384 只读 data directory plan 推进为产品级私有 stage/verify。既有一次性 `ql3-adoption` + 新增 exact `local-data-directory.adoption.stage|verify`；`stagingRoot` 必须是私有 deployment root 内、legacy data root 外的 + no-replace 路径。stage 只把 `scripts/upload` 放入 `copy-reviewed`，把 `config/db/ssh.d` 放入 `transform-input`，排除 + `db/database.sqlite` 及 sidecar；日志/备份保持外部，repo/raw/dep_cache/deps 在目标重建。复制与 verify 均拒绝 link、特殊文件、 + owner/mode 漂移和额外条目，目录/文件统一 `0700/0600`，使用 64 KiB 缓冲，并在复制循环再次执行 Edge/Standalone 条目、总字节、 + 单文件和深度预算。目录计划必须为 reviewable、恰有一个主库且无 active sidecar；SQLite source 精确绑定 + `/db/database.sqlite`，命令复用 D-383 activation acquisition，在复制/校验期间持有 source write fence，并把 activation + 与 SQLite adoption manifest digest 写入内容无关目录清单。`.incomplete` 在创建根后先持久化，payload/manifest 完成后才移除； + crash residue、目标/源漂移和扩权 command 全部失败关闭。能力继续内聚在 `lifecycle/data-directory-adoption/`，按 + `contract/command/staging/filesystem/manifest/inventory` 职责组织；最终 orchestration 为 319 行，安全文件系统原语与 manifest + 验证分别为 430/437 行，没有再拆 workspace package。整体不新增 dependency、binary 或常驻对象；workspace 仍为 18 packages， + Local Owner 为 `122 source / 121 nested / 1 root binary entry`。D-385 focused inspect/stage/verify `10/10`，Local Owner + `200 total / 195 pass / 5 conditional skip / 0 fail`；backend `1,535 total / 1,533 pass / 2 conditional skip / 0 fail`， + `pnpm build:back` 与 18-package clean build/逐包测试通过。八项架构审计和按顺序执行的 14 档 artifact audit 全 compatible；基础 + Edge/Standalone `2,598,669 / 2,598,747` bytes、316 files、57 modules，Adopted `2,818,404 / 2,818,527` bytes、336 files、 + 58 modules，Application+AI `4,502,262 / 4,502,394` bytes、511 files、141 modules，MCP + `7,324,601 / 7,324,709` bytes、802 files、227 modules，证明一次性 adoption authority 未进入常驻制品。本切片不改变 + PostgreSQL 语义，因此不重新占有 HA 证明。config/Keyv/SSH 目标转换、固定物理 Edge 的 RSS/I/O/ENOSPC/断电门及 + systemd/OpenRC/Compose lineage 留给 D-386 以后完成。 - D-384/ADR-0477（已接受）：完整 2.x data directory 接管先落地为一次性、只读、有界 inventory，而不是直接复用 legacy shell `tar` 或盲拷整个目录。既有 `ql3-adoption` 新增 exact 私有命令 `local-data-directory.adoption.inspect`，固定分类 `config/db/ssh.d→transform`、`scripts/upload→copy_reviewed`、`log/syslog/bak→retain_external(root-only)`、 diff --git a/docs/adr/ADR-0478-private-legacy-data-directory-staging.md b/docs/adr/ADR-0478-private-legacy-data-directory-staging.md new file mode 100644 index 00000000..1240cefd --- /dev/null +++ b/docs/adr/ADR-0478-private-legacy-data-directory-staging.md @@ -0,0 +1,133 @@ +# ADR-0478：私有 Legacy Data Directory 暂存与稳定校验 + +- 状态：Accepted +- 日期：2026-08-21 +- 关联：QL-RFC-0001、ADR-0476、ADR-0477 + +## 上下文 + +ADR-0477 已把完整 QingLong 2.x `data` 目录收敛为确定性、有界、内容无关的接管计划，但只读计划不是可恢复副本。下一步必须把 +审核过的 `scripts/upload` 和待转换的 `config/db/ssh.d` 放入私有暂存区，同时继续排除历史日志、备份、仓库 checkout、原始缓存和 +跨架构依赖缓存。 + +目录暂存不能脱离 ADR-0476 的主 SQLite 接管独立成立。否则 operator 可能把一版目录计划与另一版主库 target/activation 混合，或在 +2.x 主库仍可写时生成看似完整但跨资产不一致的副本。低配路由设备还要求复制过程使用固定内存、硬容量预算并在崩溃后留下可识别的 +不完整状态，而不是把整个目录先读入内存或交给无边界 `tar`。 + +## 决策 + +### 1. 在既有领域目录扩展 stage/verify + +`@qinglong/local-owner-cli` 的 `lifecycle/data-directory-adoption/` 增加两个 exact operation： + +- `local-data-directory.adoption.stage`； +- `local-data-directory.adoption.verify`。 + +两者继续使用既有一次性 `ql3-adoption` 私有 command-file 入口。实现由同一领域目录中的 `contract`、`inventory`、`staging` 和产品 +`command` 组合，不新增 workspace package、第三方依赖、binary、daemon、listener、watcher、timer、数据库连接或部署对象。 + +### 2. 使用目录计划与 SQLite activation 双围栏 + +stage 必须提交精确 `expectedPlanDigest`，verify 必须提交精确 `expectedManifestDigest`。两者还必须提交完整且 exact 的 SQLite binding： + +- `sourcePath`、`targetPath`、`recoveryPath`、`manifestPath`、`activationPath`； +- `expectedActivationDigest`。 + +SQLite source 必须严格等于 `/db/database.sqlite`，其余 SQLite adoption 证据必须位于 `dataRoot` 外。命令复用 +`@qinglong/local-admin/runtime` 的 `acquireLocalSqliteActivation`，重新验证 adoption manifest、target identity、source snapshot 与 +activation digest，并在目录复制/静态校验期间持有 source `BEGIN IMMEDIATE` 写栅栏。Profile、SQLite activation digest 和 SQLite +adoption manifest digest 都进入目录清单。 + +### 3. 固定、私有、no-replace 暂存布局 + +`stagingRoot` 必须是 `0700` canonical `deploymentRoot` 内的不存在路径，且不能位于 `dataRoot` 内。stage 使用 no-replace 创建它，并只产生： + +```text +stagingRoot/ + manifest.json 0600 + payload/ + copy-reviewed/ 0700 + scripts/... + upload/... + transform-input/ 0700 + config/... + db/... # 不含 database.sqlite 及其 sidecar + ssh.d/... +``` + +目录统一为 `0700`，文件统一为 `0600`。`log/syslog/bak` 保持外部，`repo/raw/dep_cache/deps` 在目标重新生成。主 SQLite 不进入目录 +payload，因为 ADR-0476 的 recovery/target/activation 已是它的独立恢复权威。 + +### 4. 固定内存复制、重复预算与稳定身份 + +复制仅接受当前 UID 拥有、group/world 不可写、非 symlink 的目录和普通单链接文件。源文件通过 `O_NOFOLLOW` descriptor 和 64 KiB +缓冲流式复制，打开前后必须保持 device、inode、mode、link count、UID、size、mtime 和 ctime。源目录遍历前后也必须稳定。 + +复制循环独立重复执行 ADR-0477 的 Profile 条目数、总字节、单文件和深度预算；不能只依赖较早的 inspect。复制完成并释放 SQLite +栅栏后，再重新生成完整目录计划，必须与审核计划逐字段一致。目标 verify 同样按固定顺序、固定内存重新哈希，拒绝额外条目、缺失项、 +symlink、硬链接、特殊文件、错误 owner 和非私有 mode。 + +### 5. 显式崩溃残留与内容无关清单 + +stage 创建根后立即以 no-replace 写入并持久化固定 `.incomplete` 标记；复制或清单发布前后的任何失败都不覆盖、不自动重用该目录。 +只有 payload 已持久化、`manifest.json` 以 no-replace 写入并同步后才删除标记。verify 要求根目录精确只有 `payload` 与 `manifest.json`， +因此任何残留标记或额外文件都失败关闭。operator 必须保留现场或显式移走失败目录，再使用新路径重试。 + +清单不包含原始绝对路径、任意用户文件名或文件内容，只保存：Profile、时间、目录 plan digest、SQLite 两个 digest、源/暂存路径摘要， +以及两个固定 payload group 的类别、计数、字节和语义 digest。语义 digest 绑定相对路径、entry kind、文件大小与内容摘要；源权限和时间 +由 plan digest 绑定，目标则强制归一化私有权限。 + +## 被拒绝的替代方案 + +### 直接复用 2.x tar 导出/导入 + +拒绝。它无法表达固定处置矩阵、双 digest 围栏、稳定 descriptor、Profile 预算和崩溃残留状态，也会把缓存和秘密材料混成一个恢复单元。 + +### 暂存成功后自动删除失败残留 + +拒绝。崩溃或 I/O 错误后不能证明每个创建对象仍属于本次调用；保留 `.incomplete` 比递归清理更容易审计，也避免错误删除 operator 资产。 + +### 不绑定 SQLite activation + +拒绝。目录 payload 与主库 target 会成为两个可任意拼接的时间点，无法证明后续 config/Keyv 转换使用的是同一接管快照。 + +### 再拆一个 workspace package + +拒绝。stage/verify 与 ADR-0477 inventory 是同一短生命周期 Local Owner capability，没有独立部署和依赖生命周期；继续内聚可避免 +“一个文件一个包”和平铺根源码两种碎片化。 + +## 影响 + +### 正面 + +- 完整 2.x 目录首次得到 no-replace、可重复 verify 的私有迁移输入； +- 主库接管与目录接管通过真实 activation 写栅栏和摘要链绑定； +- Edge/Standalone 复制器保持 64 KiB 固定缓冲并重复硬预算； +- 崩溃残留不会被静默当作成功或被重试覆盖； +- 能直接复制的资产与需要转换、外部保留、目标重建的资产保持物理隔离。 + +### 代价与限制 + +- stage 需要再次完整读取相关资产，并额外占用 payload 等量磁盘； +- SQLite 写栅栏只保护主库；其他文件依靠逐文件稳定身份和 stage 前后完整计划复核，不是跨文件系统事务； +- `.incomplete` 残留需要 operator 显式处置； +- 本阶段只产出转换输入，不实现 config、Keyv、SSH 的目标模型转换； +- 尚未把目录清单接入 systemd/OpenRC/Compose cutover lineage，也未完成固定物理 Edge 的断电、ENOSPC 和闪存写放大证明。 + +## 验证 + +- D-385 聚焦 data directory inspect/stage/verify `10/10`，使用真实 ADR-0476 SQLite 链； +- 覆盖 reviewed payload、主库/缓存/日志排除、私有 mode、source/target drift、activation drift、no-replace crash residue、 + verify exact replay 与扩权命令； +- Local Owner `200 total / 195 pass / 5 conditional skip / 0 fail`；backend + `1,535 total / 1,533 pass / 2 conditional skip / 0 fail`，`pnpm build:back` 通过； +- 18-package clean build/逐包测试、八项架构审计与按顺序执行的 14 档 artifact audit 全部通过； +- GitNexus impact 最高 LOW，无跨模块 execution flow；change audit 作为提交前最后门禁。 + +本阶段不修改 PostgreSQL schema、ACL、repository、role、Pool、连接或 failover 语义，因此不重新占有 PostgreSQL HA 证明。 + +## 后续 + +- D-386：把 `config`、Keyv 与 `ssh.d` 转换输入变成版本化目标模型和恢复合同； +- 在固定物理 Edge/NAS 上执行 stage/verify 的 RSS、I/O、磁盘峰值、ENOSPC 与受控断电演练； +- 将目录 manifest digest 接入 systemd/OpenRC/Compose cutover、rollback 和发布证据 lineage。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 8eedbc9a..0b2151b6 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -481,6 +481,7 @@ | [ADR-0475](./ADR-0475-legacy-system-script-open-api-compatibility.md) | Legacy System、Script 与 Open API 兼容基线 | Accepted | | [ADR-0476](./ADR-0476-real-legacy-sqlite-upgrade-and-rollback-rehearsal.md) | 真实 Legacy SQLite 升级与回滚演练 | Accepted | | [ADR-0477](./ADR-0477-bounded-legacy-data-directory-inventory.md) | 有界 Legacy Data Directory 盘点 | Accepted | +| [ADR-0478](./ADR-0478-private-legacy-data-directory-staging.md) | 私有 Legacy Data Directory 暂存与稳定校验 | Accepted | ## 规则 diff --git a/docs/operations/ql3-local-data-directory-adoption.md b/docs/operations/ql3-local-data-directory-adoption.md index fd4d930d..25b8f0a7 100644 --- a/docs/operations/ql3-local-data-directory-adoption.md +++ b/docs/operations/ql3-local-data-directory-adoption.md @@ -1,7 +1,7 @@ -# QingLong 2.x Data Directory 盘点 +# QingLong 2.x Data Directory 接管 -本流程为完整 QingLong 2.x `data` 目录生成一个只读、确定性、按 Profile 有界的 3.0 接管计划。它不会复制、压缩、删除或修改 -任何文件，也不替代 [SQLite 接管流程](./ql3-local-sqlite-adoption.md)。 +本流程先为完整 QingLong 2.x `data` 目录生成一个只读、确定性、按 Profile 有界的 3.0 接管计划，再把审核过的资产 no-replace +暂存并稳定校验。它不会删除或修改源文件，也不替代 [SQLite 接管流程](./ql3-local-sqlite-adoption.md)。 ## 1. 前置条件 @@ -10,6 +10,9 @@ - `dataRoot` 必须由当前 UID 拥有，且 group/world 不可写； - command file 继续遵守 `ql3-adoption` 的当前 UID、canonical、单链接、`0600` 私有文件要求； - 生产盘点建议先停止 2.x writer。若文件或目录在盘点中变化，命令会失败关闭，不会给出部分成功计划。 +- stage 前必须已经完成 SQLite inspect、stage、verify 与 activation，并保留五个绝对路径和 `activationDigest`； +- `deploymentRoot` 与 `stagingRoot` 的父目录必须是当前 UID 拥有的 canonical `0700` 目录；`stagingRoot` 必须尚不存在且位于 + `deploymentRoot` 内、`dataRoot` 外。 ## 2. 执行 inspect @@ -82,22 +85,96 @@ ql3-adoption run --command-file /secure/operator/ql3-data-directory-inspect.json 超过任一限制都会以 `LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID` 失败。不要为了通过门禁临时改名、删除或排除资产；先保留 现场并决定它应拆分为外部恢复资产、在目标重新生成，还是进入后续人工迁移协议。 -## 6. 常见失败 +## 6. 执行私有 stage + +审核 `assessment=reviewable`、`primaryDatabaseFiles=1`、所有 `activeSqliteSidecars=0` 后，提交完整 plan 与 SQLite activation 双围栏： + +```json +{ + "schemaVersion": 1, + "operation": "local-data-directory.adoption.stage", + "options": { + "deploymentRoot": "/opt/qinglong3/adoption", + "dataRoot": "/opt/qinglong/data", + "stagingRoot": "/opt/qinglong3/adoption/staging/reviewed-data", + "profile": "edge", + "expectedPlanDigest": "<64-hex-directory-plan-digest>", + "sqlite": { + "sourcePath": "/opt/qinglong/data/db/database.sqlite", + "targetPath": "/opt/qinglong3/adoption/sqlite/qinglong3.sqlite", + "recoveryPath": "/opt/qinglong3/adoption/sqlite/database.pre-ql3.sqlite", + "manifestPath": "/opt/qinglong3/adoption/sqlite/adoption.json", + "activationPath": "/opt/qinglong3/adoption/sqlite/activation.json", + "expectedActivationDigest": "<64-hex-sqlite-activation-digest>" + } + } +} +``` + +成功结果为 `status=staged`。暂存区固定包含 `payload/copy-reviewed/{scripts,upload}`、 +`payload/transform-input/{config,db,ssh.d}` 与私有 `manifest.json`；不存在的类别不会被制造。`db/database.sqlite` 及其 sidecar 不复制， +主库恢复继续以 SQLite adoption 的 recovery/target/activation 为权威。日志、备份和缓存也不会进入 payload。 + +所有目录归一化为 `0700`，所有文件归一化为 `0600`。复制使用 64 KiB 缓冲，并再次执行当前 Profile 的条目、字节、单文件和深度预算。 + +## 7. 执行稳定 verify + +保存 stage 返回的 `manifestDigest`，使用同一组路径和 SQLite activation 执行： + +```json +{ + "schemaVersion": 1, + "operation": "local-data-directory.adoption.verify", + "options": { + "deploymentRoot": "/opt/qinglong3/adoption", + "dataRoot": "/opt/qinglong/data", + "stagingRoot": "/opt/qinglong3/adoption/staging/reviewed-data", + "profile": "edge", + "expectedManifestDigest": "<64-hex-directory-manifest-digest>", + "sqlite": { + "sourcePath": "/opt/qinglong/data/db/database.sqlite", + "targetPath": "/opt/qinglong3/adoption/sqlite/qinglong3.sqlite", + "recoveryPath": "/opt/qinglong3/adoption/sqlite/database.pre-ql3.sqlite", + "manifestPath": "/opt/qinglong3/adoption/sqlite/adoption.json", + "activationPath": "/opt/qinglong3/adoption/sqlite/activation.json", + "expectedActivationDigest": "<64-hex-sqlite-activation-digest>" + } + } +} +``` + +verify 会重新验证目录计划、SQLite activation/source/target、清单 exact shape、私有权限和完整 payload 语义摘要。成功结果为 +`status=verified`，且 evidence 应与 stage 的低敏 evidence 一致。 + +## 8. 崩溃残留 + +stage 创建暂存根后立即写入 `.incomplete`。只有 payload 和 `manifest.json` 都持久化后才删除它。命令失败或进程崩溃后： + +- 不要直接把残留目录当作恢复资产； +- 不要在原路径重试，stage 会 no-replace 拒绝； +- 先保存现场用于诊断，再由 operator 显式移走残留目录，并使用一个新的空路径重试； +- verify 遇到 `.incomplete`、额外文件或缺失文件一律失败关闭。 + +## 9. 常见失败 - 根目录或条目 group/world 可写：修正 ownership/permission 后重新盘点； - symlink、硬链接或特殊文件：保留现场，确认来源和目标后人工处置；盘点不会跟随或读取； - 目录在盘点中变化：停止 2.x writer、同步器、下载器和仓库更新后重试； - 单文件或总内容超过 Profile 预算：不要改用 `tar` 绕过；为该资产设计独立流式迁移/外部保留流程； - 未知顶层条目：根据插件或用户资产来源登记明确处置，再进入后续 staging 设计。 +- activation 不匹配：重新执行 SQLite verify/activation，不能只替换 digest； +- `stagingRoot` 已存在：检查是否为崩溃残留，禁止覆盖或合并； +- stage/verify 后源或目标 drift：停止所有 writer，回到 inspect，生成并重新审核新的 plan。 -## 7. 当前边界 +## 10. 当前边界 -本命令只生成只读计划。它尚不： +本流程已经提供 inspect、私有 stage 和稳定 verify。它仍不： -- 复制、压缩、删除或转换任何资产； -- 创建 no-replace stage、verify manifest 或 recovery； -- 把目录计划绑定到 SQLite `planDigest`、`manifestDigest` 或 `activationDigest`； +- 删除或修改任何源资产； +- 把 `config`、Keyv 或 `ssh.d` 自动转换为 3.0 目标模型； +- 把历史日志/备份复制到默认目标，或复用跨架构 repo/dependency cache； - 授权 service-manager/Compose cutover 或 Legacy rollback； - 证明固定物理路由器/NAS 上的耗时、RSS、I/O、磁盘峰值和断电恢复。 -在后续 D-385 stage/verify 合同完成前，保留 inspect 输出和原始 2.x data directory，不要把目录计划当作自动迁移完成证明。 +只有 `status=verified` 仍不是 cutover 授权。继续保留原始 2.x data directory、SQLite recovery 和两份 manifest，等待后续转换与部署 +lineage 完成。 diff --git a/packages/ql3-local-owner-cli/src/lifecycle/adoptionCommand.ts b/packages/ql3-local-owner-cli/src/lifecycle/adoptionCommand.ts index 42ffa9bc..e4c7fc3a 100644 --- a/packages/ql3-local-owner-cli/src/lifecycle/adoptionCommand.ts +++ b/packages/ql3-local-owner-cli/src/lifecycle/adoptionCommand.ts @@ -2,7 +2,7 @@ import { readPrivateLocalCommandFile } from '@qinglong/local-command-file'; import { runLegacyCrontabAdoptionCommandFile } from './adoption'; import { isLocalDataDirectoryAdoptionOperation } from './data-directory-adoption/contract'; -import type { LocalDataDirectoryAdoptionInspectResult } from './data-directory-adoption/inventory'; +import type { LocalDataDirectoryAdoptionProductCommandResult } from './data-directory-adoption/command'; import { isLocalSqliteAdoptionProductOperation, type LocalSqliteAdoptionProductOperation, @@ -12,7 +12,7 @@ import type { LocalSqliteAdoptionProductCommandResult } from './sqlite-adoption/ export type LocalAdoptionProductCommandResult = | Awaited> | LocalSqliteAdoptionProductCommandResult - | LocalDataDirectoryAdoptionInspectResult; + | LocalDataDirectoryAdoptionProductCommandResult; function operation(value: unknown): unknown { return value && typeof value === 'object' && !Array.isArray(value) @@ -32,10 +32,10 @@ export async function runLocalAdoptionProductCommandFile( } const selected = operation(candidate); if (isLocalDataDirectoryAdoptionOperation(selected)) { - const { inspectLocalDataDirectoryAdoption } = await import( - './data-directory-adoption/inventory.js' + const { runLocalDataDirectoryAdoptionProductCommand } = await import( + './data-directory-adoption/command.js' ); - return inspectLocalDataDirectoryAdoption(candidate); + return runLocalDataDirectoryAdoptionProductCommand(candidate); } if (!isLocalSqliteAdoptionProductOperation(selected)) { return runLegacyCrontabAdoptionCommandFile(commandFilePath); diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/command.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/command.ts new file mode 100644 index 00000000..316764d6 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/command.ts @@ -0,0 +1,31 @@ +import { + LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION, + LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION, + normalizeLocalDataDirectoryAdoptionCommand, +} from './contract'; +import { + inspectLocalDataDirectoryAdoption, + type LocalDataDirectoryAdoptionInspectResult, +} from './inventory'; +import { + stageLocalDataDirectoryAdoption, + verifyLocalDataDirectoryAdoption, + type LocalDataDirectoryAdoptionMutationResult, +} from './staging'; + +export type LocalDataDirectoryAdoptionProductCommandResult = + | LocalDataDirectoryAdoptionInspectResult + | LocalDataDirectoryAdoptionMutationResult; + +export async function runLocalDataDirectoryAdoptionProductCommand( + value: unknown, +): Promise> { + const command = normalizeLocalDataDirectoryAdoptionCommand(value); + if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) { + return inspectLocalDataDirectoryAdoption(command); + } + if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION) { + return stageLocalDataDirectoryAdoption(command); + } + return verifyLocalDataDirectoryAdoption(command); +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/contract.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/contract.ts index 77d1ae23..f7447273 100644 --- a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/contract.ts +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/contract.ts @@ -1,9 +1,19 @@ import path from 'node:path'; const MAX_PATH_BYTES = 4_096; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; export const LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION = 'local-data-directory.adoption.inspect' as const; +export const LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION = + 'local-data-directory.adoption.stage' as const; +export const LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION = + 'local-data-directory.adoption.verify' as const; + +export type LocalDataDirectoryAdoptionOperation = + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION; export interface InspectLocalDataDirectoryAdoptionCommand { readonly schemaVersion: 1; @@ -14,6 +24,44 @@ export interface InspectLocalDataDirectoryAdoptionCommand { }; } +export interface LocalDataDirectoryAdoptionSqliteBinding { + readonly sourcePath: string; + readonly targetPath: string; + readonly recoveryPath: string; + readonly manifestPath: string; + readonly activationPath: string; + readonly expectedActivationDigest: string; +} + +interface LocalDataDirectoryAdoptionMutationOptions { + readonly deploymentRoot: string; + readonly dataRoot: string; + readonly stagingRoot: string; + readonly profile: 'edge' | 'standalone'; + readonly sqlite: LocalDataDirectoryAdoptionSqliteBinding; +} + +export interface StageLocalDataDirectoryAdoptionCommand { + readonly schemaVersion: 1; + readonly operation: typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION; + readonly options: LocalDataDirectoryAdoptionMutationOptions & { + readonly expectedPlanDigest: string; + }; +} + +export interface VerifyLocalDataDirectoryAdoptionCommand { + readonly schemaVersion: 1; + readonly operation: typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION; + readonly options: LocalDataDirectoryAdoptionMutationOptions & { + readonly expectedManifestDigest: string; + }; +} + +export type LocalDataDirectoryAdoptionCommand = + | InspectLocalDataDirectoryAdoptionCommand + | StageLocalDataDirectoryAdoptionCommand + | VerifyLocalDataDirectoryAdoptionCommand; + export class LocalDataDirectoryAdoptionConfigurationError extends TypeError { readonly code = 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID'; @@ -45,13 +93,77 @@ function normalizedAbsolutePath(value: unknown): value is string { export function isLocalDataDirectoryAdoptionOperation( value: unknown, -): value is typeof LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION { - return value === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION; +): value is LocalDataDirectoryAdoptionOperation { + return ( + value === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION || + value === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION || + value === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION + ); } -export function normalizeInspectLocalDataDirectoryAdoptionCommand( +function normalizeSqliteBinding( value: unknown, -): Readonly { +): Readonly { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + !exactKeys(value, [ + 'activationPath', + 'expectedActivationDigest', + 'manifestPath', + 'recoveryPath', + 'sourcePath', + 'targetPath', + ]) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'SQLite activation binding shape is invalid', + ); + } + const binding = value as Record; + for (const key of [ + 'activationPath', + 'manifestPath', + 'recoveryPath', + 'sourcePath', + 'targetPath', + ]) { + if (!normalizedAbsolutePath(binding[key])) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'SQLite activation binding path is invalid', + ); + } + } + if ( + typeof binding.expectedActivationDigest !== 'string' || + !DIGEST_PATTERN.test(binding.expectedActivationDigest) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'SQLite activation digest is invalid', + ); + } + if ( + new Set( + [ + binding.activationPath, + binding.manifestPath, + binding.recoveryPath, + binding.sourcePath, + binding.targetPath, + ].map((candidate) => path.resolve(candidate as string)), + ).size !== 5 + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'SQLite activation paths must be distinct', + ); + } + return Object.freeze(value as LocalDataDirectoryAdoptionSqliteBinding); +} + +export function normalizeLocalDataDirectoryAdoptionCommand( + value: unknown, +): Readonly { if ( !value || typeof value !== 'object' || @@ -68,15 +180,35 @@ export function normalizeInspectLocalDataDirectoryAdoptionCommand( !isLocalDataDirectoryAdoptionOperation(candidate.operation) || !candidate.options || typeof candidate.options !== 'object' || - Array.isArray(candidate.options) || - !exactKeys(candidate.options, ['dataRoot', 'profile']) + Array.isArray(candidate.options) ) { throw new LocalDataDirectoryAdoptionConfigurationError( 'command value is invalid', ); } const options = candidate.options as Record; + const expectedKeys = + candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION + ? ['dataRoot', 'profile'] + : candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION + ? [ + 'dataRoot', + 'deploymentRoot', + 'expectedPlanDigest', + 'profile', + 'sqlite', + 'stagingRoot', + ] + : [ + 'dataRoot', + 'deploymentRoot', + 'expectedManifestDigest', + 'profile', + 'sqlite', + 'stagingRoot', + ]; if ( + !exactKeys(options, expectedKeys) || !normalizedAbsolutePath(options.dataRoot) || (options.profile !== 'edge' && options.profile !== 'standalone') ) { @@ -84,5 +216,37 @@ export function normalizeInspectLocalDataDirectoryAdoptionCommand( 'command options are invalid', ); } - return Object.freeze(value as InspectLocalDataDirectoryAdoptionCommand); + if (candidate.operation !== LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) { + if ( + !normalizedAbsolutePath(options.deploymentRoot) || + !normalizedAbsolutePath(options.stagingRoot) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'adoption root path is invalid', + ); + } + normalizeSqliteBinding(options.sqlite); + const digest = + candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION + ? options.expectedPlanDigest + : options.expectedManifestDigest; + if (typeof digest !== 'string' || !DIGEST_PATTERN.test(digest)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'reviewed adoption digest is invalid', + ); + } + } + return Object.freeze(value as LocalDataDirectoryAdoptionCommand); +} + +export function normalizeInspectLocalDataDirectoryAdoptionCommand( + value: unknown, +): Readonly { + const command = normalizeLocalDataDirectoryAdoptionCommand(value); + if (command.operation !== LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'inspection operation is invalid', + ); + } + return command; } diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/filesystem.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/filesystem.ts new file mode 100644 index 00000000..165ae74f --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/filesystem.ts @@ -0,0 +1,430 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + LocalDataDirectoryAdoptionConfigurationError, + type StageLocalDataDirectoryAdoptionCommand, + type VerifyLocalDataDirectoryAdoptionCommand, +} from './contract'; + +const HASH_BUFFER_BYTES = 64 * 1024; +const MAX_RELATIVE_PATH_BYTES = 4_096; + +export interface RootAuthority { + readonly uid: number; + readonly deploymentRoot: string; + readonly dataRoot: string; + readonly stagingRoot: string; +} + +export interface CopyBudget { + readonly maxEntries: number; + readonly maxHashedBytes: number; + readonly maxFileBytes: number; + readonly maxDepth: number; +} + +export interface MutableCopyBudget { + entries: number; + bytes: number; +} + +function currentUid(): number { + if ( + typeof process.getuid !== 'function' || + typeof process.geteuid !== 'function' || + process.getuid() !== process.geteuid() + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'real and effective POSIX users must match', + ); + } + return process.getuid(); +} + +function inside(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return ( + relative !== '' && + relative !== '..' && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) + ); +} + +export function assertPrivateDirectory( + directoryPath: string, + uid: number, + label: string, +): fs.BigIntStats { + let stat: fs.BigIntStats; + try { + stat = fs.lstatSync(directoryPath, { bigint: true }); + } catch (error) { + throw new LocalDataDirectoryAdoptionConfigurationError( + `${label} is unavailable`, + error, + ); + } + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + stat.uid !== BigInt(uid) || + (stat.mode & 0o777n) !== 0o700n || + fs.realpathSync(directoryPath) !== directoryPath + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + `${label} must be an owner-controlled 0700 canonical directory`, + ); + } + return stat; +} + +function assertMissing(candidate: string, label: string): void { + try { + fs.lstatSync(candidate); + } catch (error) { + if ( + error && + typeof error === 'object' && + 'code' in error && + error.code === 'ENOENT' + ) { + return; + } + throw new LocalDataDirectoryAdoptionConfigurationError( + `${label} cannot be inspected`, + error, + ); + } + throw new LocalDataDirectoryAdoptionConfigurationError( + `${label} must not already exist`, + ); +} + +export function rootAuthority( + options: + | StageLocalDataDirectoryAdoptionCommand['options'] + | VerifyLocalDataDirectoryAdoptionCommand['options'], + requireMissing: boolean, +): Readonly { + const uid = currentUid(); + assertPrivateDirectory(options.deploymentRoot, uid, 'deploymentRoot'); + if ( + !inside(options.deploymentRoot, options.stagingRoot) || + options.dataRoot === options.stagingRoot || + inside(options.dataRoot, options.stagingRoot) || + inside(options.stagingRoot, options.dataRoot) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'stagingRoot must be isolated inside deploymentRoot', + ); + } + const stagingParent = path.dirname(options.stagingRoot); + if ( + stagingParent !== options.deploymentRoot && + !inside(options.deploymentRoot, stagingParent) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'stagingRoot parent must remain inside deploymentRoot', + ); + } + assertPrivateDirectory(stagingParent, uid, 'stagingRoot parent'); + const expectedSource = path.join(options.dataRoot, 'db', 'database.sqlite'); + if (options.sqlite.sourcePath !== expectedSource) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'SQLite activation source must be the reviewed primary database', + ); + } + for (const candidate of [ + options.sqlite.targetPath, + options.sqlite.recoveryPath, + options.sqlite.manifestPath, + options.sqlite.activationPath, + ]) { + if (candidate === options.dataRoot || inside(options.dataRoot, candidate)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'SQLite adoption evidence must remain outside dataRoot', + ); + } + } + if (requireMissing) assertMissing(options.stagingRoot, 'stagingRoot'); + else assertPrivateDirectory(options.stagingRoot, uid, 'stagingRoot'); + return Object.freeze({ + uid, + deploymentRoot: options.deploymentRoot, + dataRoot: options.dataRoot, + stagingRoot: options.stagingRoot, + }); +} + +export function sameStat(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +export function sortedNames(directoryPath: string): readonly string[] { + return fs + .readdirSync(directoryPath) + .sort((left, right) => + Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')), + ); +} + +function assertRelativePath(value: string): void { + if ( + value.length < 1 || + path.isAbsolute(value) || + value === '..' || + value.startsWith(`..${path.sep}`) || + Buffer.byteLength(value, 'utf8') > MAX_RELATIVE_PATH_BYTES + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'payload relative path is invalid or too long', + ); + } +} + +export function syncDirectory(directoryPath: string): void { + const descriptor = fs.openSync(directoryPath, fs.constants.O_RDONLY); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +export function writeExclusiveJson(filePath: string, value: object): void { + const descriptor = fs.openSync(filePath, 'wx', 0o600); + try { + const bytes = Buffer.from(`${JSON.stringify(value)}\n`, 'utf8'); + let offset = 0; + while (offset < bytes.length) { + offset += fs.writeSync( + descriptor, + bytes, + offset, + bytes.length - offset, + null, + ); + } + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function copyStableFile( + sourcePath: string, + destinationPath: string, + expected: fs.BigIntStats, +): void { + const source = fs.openSync( + sourcePath, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + let destination: number | undefined; + const buffer = Buffer.allocUnsafe(HASH_BUFFER_BYTES); + try { + const before = fs.fstatSync(source, { bigint: true }); + if (!before.isFile() || !sameStat(expected, before)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'source file identity changed before staging', + ); + } + destination = fs.openSync(destinationPath, 'wx', 0o600); + for (;;) { + const count = fs.readSync(source, buffer, 0, buffer.length, null); + if (count === 0) break; + let offset = 0; + while (offset < count) { + offset += fs.writeSync( + destination, + buffer, + offset, + count - offset, + null, + ); + } + } + fs.fsyncSync(destination); + if (!sameStat(before, fs.fstatSync(source, { bigint: true }))) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'source file changed during staging', + ); + } + } finally { + buffer.fill(0); + if (destination !== undefined) fs.closeSync(destination); + fs.closeSync(source); + } +} + +function shouldExcludeDatabaseEntry( + category: string, + relative: string, +): boolean { + return ( + category === 'db' && + /^(?:database\.sqlite|database\.sqlite-(?:wal|shm|journal))$/.test(relative) + ); +} + +export function copyCategory( + sourceRoot: string, + destinationRoot: string, + category: string, + uid: number, + limits: Readonly, + shared: MutableCopyBudget, +): void { + const sourceCategory = path.join(sourceRoot, category); + let categoryStat: fs.BigIntStats; + try { + categoryStat = fs.lstatSync(sourceCategory, { bigint: true }); + } catch (error) { + if ( + error && + typeof error === 'object' && + 'code' in error && + error.code === 'ENOENT' + ) { + return; + } + throw error; + } + if ( + !categoryStat.isDirectory() || + categoryStat.isSymbolicLink() || + categoryStat.uid !== BigInt(uid) || + (categoryStat.mode & 0o022n) !== 0n + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'source category identity is unsafe', + ); + } + const destinationCategory = path.join(destinationRoot, category); + fs.mkdirSync(destinationCategory, { mode: 0o700 }); + + const visit = ( + sourceDirectory: string, + destinationDirectory: string, + expectedDirectory: fs.BigIntStats, + depth: number, + ): void => { + if (depth > limits.maxDepth) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'source payload depth exceeds the Profile budget', + ); + } + for (const name of sortedNames(sourceDirectory)) { + const sourceEntry = path.join(sourceDirectory, name); + const categoryRelative = path.relative(sourceCategory, sourceEntry); + assertRelativePath(categoryRelative); + if (shouldExcludeDatabaseEntry(category, categoryRelative)) continue; + shared.entries += 1; + if (shared.entries > limits.maxEntries) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'source payload entry count exceeds the Profile budget', + ); + } + const destinationEntry = path.join(destinationDirectory, name); + const stat = fs.lstatSync(sourceEntry, { bigint: true }); + if ( + stat.isSymbolicLink() || + stat.uid !== BigInt(uid) || + (stat.mode & 0o022n) !== 0n + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'source payload entry identity is unsafe', + ); + } + if (stat.isDirectory()) { + fs.mkdirSync(destinationEntry, { mode: 0o700 }); + visit(sourceEntry, destinationEntry, stat, depth + 1); + syncDirectory(destinationEntry); + } else if (stat.isFile() && stat.nlink === 1n) { + if ( + stat.size < 0n || + stat.size > BigInt(limits.maxFileBytes) || + stat.size > BigInt(Number.MAX_SAFE_INTEGER) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'source payload file exceeds the Profile budget', + ); + } + const bytes = Number(stat.size); + if ( + !Number.isSafeInteger(shared.bytes + bytes) || + shared.bytes + bytes > limits.maxHashedBytes + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'source payload bytes exceed the Profile budget', + ); + } + shared.bytes += bytes; + copyStableFile(sourceEntry, destinationEntry, stat); + } else { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'source payload entry kind is unsafe', + ); + } + } + if ( + !sameStat( + expectedDirectory, + fs.lstatSync(sourceDirectory, { bigint: true }), + ) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'source directory changed during staging', + ); + } + }; + + visit(sourceCategory, destinationCategory, categoryStat, 1); + syncDirectory(destinationCategory); +} + +export function stableFileDigest( + filePath: string, + expected: fs.BigIntStats, +): string { + const descriptor = fs.openSync( + filePath, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + const buffer = Buffer.allocUnsafe(HASH_BUFFER_BYTES); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if (!before.isFile() || !sameStat(expected, before)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged file identity changed before verification', + ); + } + const hash = crypto.createHash('sha256'); + for (;;) { + const count = fs.readSync(descriptor, buffer, 0, buffer.length, null); + if (count === 0) break; + hash.update(buffer.subarray(0, count)); + } + if (!sameStat(before, fs.fstatSync(descriptor, { bigint: true }))) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged file changed during verification', + ); + } + return hash.digest('hex'); + } finally { + buffer.fill(0); + fs.closeSync(descriptor); + } +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/manifest.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/manifest.ts new file mode 100644 index 00000000..6f7fa437 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/manifest.ts @@ -0,0 +1,437 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { LocalDataDirectoryAdoptionConfigurationError } from './contract'; +import { + assertPrivateDirectory, + sameStat, + sortedNames, + stableFileDigest, + type RootAuthority, +} from './filesystem'; + +const MAX_MANIFEST_BYTES = 64 * 1024; +const MAX_RELATIVE_PATH_BYTES = 4_096; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +export const MANIFEST_NAME = 'manifest.json'; + +export const PAYLOAD_GROUPS = Object.freeze([ + Object.freeze({ + name: 'copy_reviewed' as const, + directoryName: 'copy-reviewed', + categories: Object.freeze(['scripts', 'upload'] as const), + }), + Object.freeze({ + name: 'transform_input' as const, + directoryName: 'transform-input', + categories: Object.freeze(['config', 'db', 'ssh.d'] as const), + }), +]); + +type PayloadGroupName = (typeof PAYLOAD_GROUPS)[number]['name']; + +export interface LocalDataDirectoryPayloadEvidence { + readonly name: PayloadGroupName; + readonly categories: readonly string[]; + readonly entries: number; + readonly directories: number; + readonly files: number; + readonly bytes: number; + readonly digest: string; +} + +export interface LocalDataDirectoryAdoptionManifestPayload { + readonly schemaVersion: 1; + readonly kind: 'qinglong3-legacy-data-directory-adoption'; + readonly state: 'staged'; + readonly profile: 'edge' | 'standalone'; + readonly createdAtMs: number; + readonly planDigest: string; + readonly sqliteActivationDigest: string; + readonly sqliteAdoptionManifestDigest: string; + readonly dataRootPathDigest: string; + readonly stagingRootPathDigest: string; + readonly payload: readonly LocalDataDirectoryPayloadEvidence[]; +} + +export interface LocalDataDirectoryAdoptionManifest + extends LocalDataDirectoryAdoptionManifestPayload { + readonly manifestDigest: string; +} + +interface MutablePayloadSummary { + entries: number; + directories: number; + files: number; + bytes: number; +} + +export function sha256Text(value: string): string { + return crypto.createHash('sha256').update(value, 'utf8').digest('hex'); +} + +function exactKeys(value: object, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const canonical = [...expected].sort(); + return ( + actual.length === canonical.length && + actual.every((key, index) => key === canonical[index]) + ); +} + +function assertRelativePath(value: string): void { + if ( + value.length < 1 || + path.isAbsolute(value) || + value === '..' || + value.startsWith(`..${path.sep}`) || + Buffer.byteLength(value, 'utf8') > MAX_RELATIVE_PATH_BYTES + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'payload relative path is invalid or too long', + ); + } +} + +function payloadEvidence( + groupRoot: string, + group: (typeof PAYLOAD_GROUPS)[number], + uid: number, +): Readonly { + assertPrivateDirectory(groupRoot, uid, 'payload group'); + const allowed = new Set(group.categories); + const summary: MutablePayloadSummary = { + entries: 0, + directories: 0, + files: 0, + bytes: 0, + }; + const hash = crypto.createHash('sha256'); + const visit = (directoryPath: string, expected: fs.BigIntStats): void => { + for (const name of sortedNames(directoryPath)) { + const entryPath = path.join(directoryPath, name); + const relative = path.relative(groupRoot, entryPath); + assertRelativePath(relative); + if (directoryPath === groupRoot && !allowed.has(name)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged payload contains an unexpected category', + ); + } + const stat = fs.lstatSync(entryPath, { bigint: true }); + summary.entries += 1; + if ( + stat.isSymbolicLink() || + stat.uid !== BigInt(uid) || + (stat.mode & 0o777n) !== (stat.isDirectory() ? 0o700n : 0o600n) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged payload identity or mode is invalid', + ); + } + const canonicalRelative = relative.split(path.sep).join('/'); + if (stat.isDirectory()) { + summary.directories += 1; + hash.update( + `${JSON.stringify({ + relative: canonicalRelative, + kind: 'directory', + })}\n`, + 'utf8', + ); + visit(entryPath, stat); + } else if (stat.isFile() && stat.nlink === 1n) { + if (stat.size < 0n || stat.size > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged payload file size is unsupported', + ); + } + const bytes = Number(stat.size); + if (!Number.isSafeInteger(summary.bytes + bytes)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged payload byte total is unsupported', + ); + } + summary.files += 1; + summary.bytes += bytes; + hash.update( + `${JSON.stringify({ + relative: canonicalRelative, + kind: 'file', + bytes, + contentDigest: stableFileDigest(entryPath, stat), + })}\n`, + 'utf8', + ); + } else { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged payload entry kind is invalid', + ); + } + } + if (!sameStat(expected, fs.lstatSync(directoryPath, { bigint: true }))) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged payload directory changed during verification', + ); + } + }; + visit(groupRoot, fs.lstatSync(groupRoot, { bigint: true })); + return Object.freeze({ + name: group.name, + categories: group.categories, + ...summary, + digest: hash.digest('hex'), + }); +} + +export function inspectPayload( + stagingRoot: string, + uid: number, +): readonly LocalDataDirectoryPayloadEvidence[] { + const payloadRoot = path.join(stagingRoot, 'payload'); + const payloadRootBefore = assertPrivateDirectory( + payloadRoot, + uid, + 'payload root', + ); + const expectedGroupNames = PAYLOAD_GROUPS.map((group) => group.directoryName); + if ( + JSON.stringify(sortedNames(payloadRoot)) !== + JSON.stringify( + [...expectedGroupNames].sort((left, right) => + Buffer.compare(Buffer.from(left), Buffer.from(right)), + ), + ) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'payload group set is invalid', + ); + } + const evidence = Object.freeze( + PAYLOAD_GROUPS.map((group) => + payloadEvidence(path.join(payloadRoot, group.directoryName), group, uid), + ), + ); + if ( + !sameStat(payloadRootBefore, fs.lstatSync(payloadRoot, { bigint: true })) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'payload root changed during verification', + ); + } + return evidence; +} + +function parsePayloadEvidence( + value: unknown, +): LocalDataDirectoryPayloadEvidence { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + !exactKeys(value, [ + 'bytes', + 'categories', + 'digest', + 'directories', + 'entries', + 'files', + 'name', + ]) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'manifest payload evidence shape is invalid', + ); + } + const candidate = value as Partial; + const group = PAYLOAD_GROUPS.find((entry) => entry.name === candidate.name); + if ( + !group || + JSON.stringify(candidate.categories) !== JSON.stringify(group.categories) || + !DIGEST_PATTERN.test(candidate.digest ?? '') || + ![ + candidate.entries, + candidate.directories, + candidate.files, + candidate.bytes, + ].every( + (number) => Number.isSafeInteger(number) && (number as number) >= 0, + ) || + candidate.entries !== + (candidate.directories as number) + (candidate.files as number) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'manifest payload evidence is invalid', + ); + } + return value as LocalDataDirectoryPayloadEvidence; +} + +function parseManifest(value: unknown): LocalDataDirectoryAdoptionManifest { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + !exactKeys(value, [ + 'createdAtMs', + 'dataRootPathDigest', + 'kind', + 'manifestDigest', + 'payload', + 'planDigest', + 'profile', + 'schemaVersion', + 'sqliteActivationDigest', + 'sqliteAdoptionManifestDigest', + 'stagingRootPathDigest', + 'state', + ]) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest shape is invalid', + ); + } + const manifest = value as Partial; + if ( + manifest.schemaVersion !== 1 || + manifest.kind !== 'qinglong3-legacy-data-directory-adoption' || + manifest.state !== 'staged' || + (manifest.profile !== 'edge' && manifest.profile !== 'standalone') || + !Number.isSafeInteger(manifest.createdAtMs) || + (manifest.createdAtMs as number) < 0 || + ![ + manifest.manifestDigest, + manifest.planDigest, + manifest.sqliteActivationDigest, + manifest.sqliteAdoptionManifestDigest, + manifest.dataRootPathDigest, + manifest.stagingRootPathDigest, + ].every( + (digest) => typeof digest === 'string' && DIGEST_PATTERN.test(digest), + ) || + !Array.isArray(manifest.payload) || + manifest.payload.length !== PAYLOAD_GROUPS.length + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest value is invalid', + ); + } + const parsedPayload = manifest.payload.map(parsePayloadEvidence); + if ( + JSON.stringify(parsedPayload.map((entry) => entry.name)) !== + JSON.stringify(PAYLOAD_GROUPS.map((entry) => entry.name)) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest payload order is invalid', + ); + } + const { manifestDigest, ...payload } = + manifest as LocalDataDirectoryAdoptionManifest; + if (sha256Text(JSON.stringify(payload)) !== manifestDigest) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest digest does not match', + ); + } + return manifest as LocalDataDirectoryAdoptionManifest; +} + +function readManifest( + stagingRoot: string, + uid: number, +): Readonly { + const manifestPath = path.join(stagingRoot, MANIFEST_NAME); + const stat = fs.lstatSync(manifestPath, { bigint: true }); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1n || + stat.uid !== BigInt(uid) || + (stat.mode & 0o777n) !== 0o600n || + stat.size < 1n || + stat.size > BigInt(MAX_MANIFEST_BYTES) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest identity, mode, or size is invalid', + ); + } + const descriptor = fs.openSync( + manifestPath, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if (!sameStat(stat, before)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest identity changed before reading', + ); + } + const content = fs.readFileSync(descriptor, 'utf8'); + if (!sameStat(before, fs.fstatSync(descriptor, { bigint: true }))) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest changed while reading', + ); + } + return parseManifest(JSON.parse(content)); + } catch (error) { + if (error instanceof LocalDataDirectoryAdoptionConfigurationError) { + throw error; + } + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest JSON is invalid', + error, + ); + } finally { + fs.closeSync(descriptor); + } +} + +function assertCompleteRoot(stagingRoot: string): void { + if ( + JSON.stringify(sortedNames(stagingRoot)) !== + JSON.stringify([MANIFEST_NAME, 'payload'].sort()) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging root is incomplete or contains unexpected entries', + ); + } +} + +export function verifyStaticStage( + authority: Readonly, + expectedManifestDigest: string, +): Readonly { + const rootBefore = assertPrivateDirectory( + authority.stagingRoot, + authority.uid, + 'stagingRoot', + ); + assertCompleteRoot(authority.stagingRoot); + const manifest = readManifest(authority.stagingRoot, authority.uid); + if (manifest.manifestDigest !== expectedManifestDigest) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest no longer matches the reviewed digest', + ); + } + if ( + manifest.dataRootPathDigest !== sha256Text(authority.dataRoot) || + manifest.stagingRootPathDigest !== sha256Text(authority.stagingRoot) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest path binding is invalid', + ); + } + const actualPayload = inspectPayload(authority.stagingRoot, authority.uid); + if (JSON.stringify(actualPayload) !== JSON.stringify(manifest.payload)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged payload no longer matches the manifest', + ); + } + if ( + !sameStat(rootBefore, fs.lstatSync(authority.stagingRoot, { bigint: true })) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging root changed during verification', + ); + } + return manifest; +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/staging.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/staging.ts new file mode 100644 index 00000000..fa6c000d --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/staging.ts @@ -0,0 +1,319 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { + acquireLocalSqliteActivation, + type LocalSqliteActivationFence, +} from '@qinglong/local-admin/runtime'; + +import { + LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION, + LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION, + LocalDataDirectoryAdoptionConfigurationError, + type LocalDataDirectoryAdoptionSqliteBinding, + type StageLocalDataDirectoryAdoptionCommand, + type VerifyLocalDataDirectoryAdoptionCommand, +} from './contract'; +import { + copyCategory, + rootAuthority, + syncDirectory, + writeExclusiveJson, + type MutableCopyBudget, +} from './filesystem'; +import { + MANIFEST_NAME, + PAYLOAD_GROUPS, + inspectPayload, + sha256Text, + verifyStaticStage, + type LocalDataDirectoryAdoptionManifest, + type LocalDataDirectoryAdoptionManifestPayload, + type LocalDataDirectoryPayloadEvidence, +} from './manifest'; +import { + inspectLocalDataDirectoryAdoption, + type LocalDataDirectoryAdoptionEvidence, +} from './inventory'; + +const INCOMPLETE_NAME = '.incomplete'; + +export type { + LocalDataDirectoryAdoptionManifest, + LocalDataDirectoryPayloadEvidence, +} from './manifest'; + +export interface LocalDataDirectoryAdoptionMutationResult { + readonly schemaVersion: 1; + readonly operation: + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION; + readonly status: 'staged' | 'verified'; + readonly evidence: Readonly<{ + profile: 'edge' | 'standalone'; + createdAtMs: number; + planDigest: string; + manifestDigest: string; + sqliteActivationDigest: string; + sqliteAdoptionManifestDigest: string; + payload: readonly LocalDataDirectoryPayloadEvidence[]; + }>; +} + +function inspectPlan( + dataRoot: string, + profile: 'edge' | 'standalone', +): Readonly { + return inspectLocalDataDirectoryAdoption({ + schemaVersion: 1, + operation: 'local-data-directory.adoption.inspect', + options: { dataRoot, profile }, + }).evidence; +} + +function assertReviewablePlan( + plan: Readonly, + expectedPlanDigest: string, +): void { + const database = plan.categories.find((category) => category.name === 'db'); + if ( + plan.planDigest !== expectedPlanDigest || + plan.assessment !== 'reviewable' || + plan.totalUnsafeEntries !== 0 || + plan.unknownTopLevelEntries !== 0 || + !database || + database.primaryDatabaseFiles !== 1 || + plan.categories.some((category) => category.activeSqliteSidecars !== 0) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'data directory no longer matches a reviewable migration plan', + ); + } +} + +async function acquireSqliteFence( + binding: Readonly, + profile: 'edge' | 'standalone', +): Promise> { + const fence = await acquireLocalSqliteActivation({ + sourcePath: binding.sourcePath, + targetPath: binding.targetPath, + recoveryPath: binding.recoveryPath, + manifestPath: binding.manifestPath, + activationPath: binding.activationPath, + expectedActivationDigest: binding.expectedActivationDigest, + }); + if (fence.activation.profile !== profile) { + await fence.release(); + throw new LocalDataDirectoryAdoptionConfigurationError( + 'SQLite activation profile does not match directory adoption', + ); + } + return fence; +} + +function manifestPayload( + command: Readonly, + plan: Readonly, + fence: Readonly, + payload: readonly LocalDataDirectoryPayloadEvidence[], +): Readonly { + const createdAtMs = Date.now(); + if (!Number.isSafeInteger(createdAtMs) || createdAtMs < 0) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'system clock returned an invalid timestamp', + ); + } + return Object.freeze({ + schemaVersion: 1, + kind: 'qinglong3-legacy-data-directory-adoption', + state: 'staged', + profile: command.options.profile, + createdAtMs, + planDigest: plan.planDigest, + sqliteActivationDigest: fence.activation.activationDigest, + sqliteAdoptionManifestDigest: fence.adoption.manifestDigest, + dataRootPathDigest: sha256Text(command.options.dataRoot), + stagingRootPathDigest: sha256Text(command.options.stagingRoot), + payload, + }); +} + +function result( + operation: + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION, + status: 'staged' | 'verified', + manifest: Readonly, +): Readonly { + return Object.freeze({ + schemaVersion: 1, + operation, + status, + evidence: Object.freeze({ + profile: manifest.profile, + createdAtMs: manifest.createdAtMs, + planDigest: manifest.planDigest, + manifestDigest: manifest.manifestDigest, + sqliteActivationDigest: manifest.sqliteActivationDigest, + sqliteAdoptionManifestDigest: manifest.sqliteAdoptionManifestDigest, + payload: manifest.payload, + }), + }); +} + +export async function stageLocalDataDirectoryAdoption( + command: Readonly, +): Promise> { + try { + const authority = rootAuthority(command.options, true); + const before = inspectPlan( + command.options.dataRoot, + command.options.profile, + ); + assertReviewablePlan(before, command.options.expectedPlanDigest); + const fence = await acquireSqliteFence( + command.options.sqlite, + command.options.profile, + ); + let payload: readonly LocalDataDirectoryPayloadEvidence[]; + try { + fs.mkdirSync(command.options.stagingRoot, { mode: 0o700 }); + writeExclusiveJson( + path.join(command.options.stagingRoot, INCOMPLETE_NAME), + { + schemaVersion: 1, + kind: 'qinglong3-legacy-data-directory-adoption-incomplete', + }, + ); + syncDirectory(command.options.stagingRoot); + syncDirectory(path.dirname(command.options.stagingRoot)); + const payloadRoot = path.join(command.options.stagingRoot, 'payload'); + fs.mkdirSync(payloadRoot, { mode: 0o700 }); + const copyBudget: MutableCopyBudget = { entries: 0, bytes: 0 }; + for (const group of PAYLOAD_GROUPS) { + const groupRoot = path.join(payloadRoot, group.directoryName); + fs.mkdirSync(groupRoot, { mode: 0o700 }); + for (const category of group.categories) { + copyCategory( + command.options.dataRoot, + groupRoot, + category, + authority.uid, + before.budget, + copyBudget, + ); + } + syncDirectory(groupRoot); + } + syncDirectory(payloadRoot); + fence.assertTargetIdentity(); + payload = inspectPayload(command.options.stagingRoot, authority.uid); + } finally { + await fence.release(); + } + const after = inspectPlan( + command.options.dataRoot, + command.options.profile, + ); + assertReviewablePlan(after, command.options.expectedPlanDigest); + if (JSON.stringify(after) !== JSON.stringify(before)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'data directory changed during staging', + ); + } + const manifestBase = manifestPayload(command, after, fence, payload); + const manifest = Object.freeze({ + ...manifestBase, + manifestDigest: sha256Text(JSON.stringify(manifestBase)), + }); + writeExclusiveJson( + path.join(command.options.stagingRoot, MANIFEST_NAME), + manifest, + ); + syncDirectory(command.options.stagingRoot); + fs.unlinkSync(path.join(command.options.stagingRoot, INCOMPLETE_NAME)); + syncDirectory(command.options.stagingRoot); + const verified = verifyStaticStage(authority, manifest.manifestDigest); + return result( + LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION, + 'staged', + verified, + ); + } catch (error) { + if (error instanceof LocalDataDirectoryAdoptionConfigurationError) { + throw error; + } + throw new LocalDataDirectoryAdoptionConfigurationError( + 'data directory staging failed', + error, + ); + } +} + +export async function verifyLocalDataDirectoryAdoption( + command: Readonly, +): Promise> { + try { + const authority = rootAuthority(command.options, false); + const manifest = verifyStaticStage( + authority, + command.options.expectedManifestDigest, + ); + if ( + manifest.profile !== command.options.profile || + manifest.sqliteActivationDigest !== + command.options.sqlite.expectedActivationDigest + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staging manifest authority binding is invalid', + ); + } + const before = inspectPlan( + command.options.dataRoot, + command.options.profile, + ); + assertReviewablePlan(before, manifest.planDigest); + const fence = await acquireSqliteFence( + command.options.sqlite, + command.options.profile, + ); + try { + if ( + fence.activation.activationDigest !== manifest.sqliteActivationDigest || + fence.adoption.manifestDigest !== manifest.sqliteAdoptionManifestDigest + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'SQLite activation no longer matches the staging manifest', + ); + } + verifyStaticStage(authority, command.options.expectedManifestDigest); + fence.assertTargetIdentity(); + } finally { + await fence.release(); + } + const after = inspectPlan( + command.options.dataRoot, + command.options.profile, + ); + assertReviewablePlan(after, manifest.planDigest); + if (JSON.stringify(after) !== JSON.stringify(before)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'data directory changed during verification', + ); + } + return result( + LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION, + 'verified', + manifest, + ); + } catch (error) { + if (error instanceof LocalDataDirectoryAdoptionConfigurationError) { + throw error; + } + throw new LocalDataDirectoryAdoptionConfigurationError( + 'data directory verification failed', + error, + ); + } +} diff --git a/packages/ql3-local-owner-cli/test/dataDirectoryAdoptionStageCli.test.cjs b/packages/ql3-local-owner-cli/test/dataDirectoryAdoptionStageCli.test.cjs new file mode 100644 index 00000000..914cfeed --- /dev/null +++ b/packages/ql3-local-owner-cli/test/dataDirectoryAdoptionStageCli.test.cjs @@ -0,0 +1,481 @@ +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const { test } = require('node:test'); + +const BINARY = path.join(__dirname, '../dist/lifecycle/adoptionCli.js'); +const DIRECTORY_INSPECT = 'local-data-directory.adoption.inspect'; +const DIRECTORY_STAGE = 'local-data-directory.adoption.stage'; +const DIRECTORY_VERIFY = 'local-data-directory.adoption.verify'; + +function privateDirectory(directoryPath) { + fs.mkdirSync(directoryPath, { recursive: true, mode: 0o700 }); + fs.chmodSync(directoryPath, 0o700); +} + +function privateFile(filePath, content) { + privateDirectory(path.dirname(filePath)); + fs.writeFileSync(filePath, content, { mode: 0o600 }); + fs.chmodSync(filePath, 0o600); +} + +function createLegacyDatabase(sourcePath) { + const source = new DatabaseSync(sourcePath); + source.exec(` + CREATE TABLE "Crontabs" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), + command VARCHAR(255), schedule VARCHAR(255), timestamp VARCHAR(255), + saved TINYINT(1), status DECIMAL, isSystem DECIMAL, pid DECIMAL, + isDisabled DECIMAL, isPinned DECIMAL, log_path VARCHAR(255), labels JSON, + last_running_time DECIMAL, last_execution_time DECIMAL, sub_id DECIMAL, + extra_schedules JSON, task_before VARCHAR(255), task_after VARCHAR(255), + log_name VARCHAR(255), allow_multiple_instances DECIMAL, + work_dir VARCHAR(255), createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "Dependences" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), type DECIMAL, + timestamp VARCHAR(255), status DECIMAL, log JSON, remark VARCHAR(255), + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "Apps" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), scopes JSON, + client_id VARCHAR(255), client_secret VARCHAR(255), tokens JSON, + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "Auths" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, ip VARCHAR(255), type VARCHAR(255), + info JSON, createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "Envs" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, value VARCHAR(255), + timestamp VARCHAR(255), status DECIMAL, position DECIMAL, + name VARCHAR(255), remarks VARCHAR(255), isPinned DECIMAL, labels JSON, + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "Subscriptions" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), url VARCHAR(255), + schedule VARCHAR(255), interval_schedule JSON, type VARCHAR(255), + whitelist VARCHAR(255), blacklist VARCHAR(255), status DECIMAL, + dependences VARCHAR(255), extensions VARCHAR(255), sub_before VARCHAR(255), + sub_after VARCHAR(255), branch VARCHAR(255), pull_type VARCHAR(255), + pull_option JSON, pid DECIMAL, is_disabled DECIMAL, log_path VARCHAR(255), + schedule_type VARCHAR(255), alias VARCHAR(255), proxy VARCHAR(255), + autoAddCron DECIMAL, autoDelCron DECIMAL, + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "CrontabViews" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), position DECIMAL, + isDisabled DECIMAL, filters JSON, sorts JSON, filterRelation VARCHAR(255), + type DECIMAL, createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "CrontabStats" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, ref_id DECIMAL NOT NULL, + date VARCHAR(255) NOT NULL, run_count DECIMAL, success_count DECIMAL, + fail_count DECIMAL, total_time DECIMAL, max_time DECIMAL, + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "RunningInstances" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, cron_id DECIMAL NOT NULL, + run_id VARCHAR(36), attempt_id VARCHAR(36), pid DECIMAL, + log_path VARCHAR(255), started_at DECIMAL NOT NULL, finished_at DECIMAL, + status DECIMAL NOT NULL, exit_code DECIMAL, + createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL + ); + CREATE TABLE "PluginOwnedState" ( + id INTEGER PRIMARY KEY, payload TEXT NOT NULL + ); + INSERT INTO "Crontabs" ( + id, name, command, schedule, status, isDisabled, isPinned, + createdAt, updatedAt + ) VALUES ( + 1, 'Legacy task', 'task /scripts/legacy.sh', '0 0 * * *', + 1, 0, 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ); + INSERT INTO "Envs" ( + id, name, value, status, position, createdAt, updatedAt + ) VALUES ( + 1, 'LEGACY_VALUE', 'preserved', 0, 100, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ); + INSERT INTO "PluginOwnedState" (id, payload) + VALUES (1, '{"preserved":true}'); + `); + source.close(); + fs.chmodSync(sourcePath, 0o600); +} + +function fixture(t) { + const deploymentRoot = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-directory-stage-')), + ); + fs.chmodSync(deploymentRoot, 0o700); + t.after(() => fs.rmSync(deploymentRoot, { recursive: true, force: true })); + const value = { + deploymentRoot, + commandsDirectory: path.join(deploymentRoot, 'commands'), + artifactsDirectory: path.join(deploymentRoot, 'artifacts'), + stagingParent: path.join(deploymentRoot, 'staging'), + dataRoot: path.join(deploymentRoot, 'legacy-data'), + }; + privateDirectory(value.commandsDirectory); + privateDirectory(value.artifactsDirectory); + privateDirectory(value.stagingParent); + privateDirectory(value.dataRoot); + Object.assign(value, { + sourcePath: path.join(value.dataRoot, 'db', 'database.sqlite'), + targetPath: path.join(value.artifactsDirectory, 'qinglong3.sqlite'), + recoveryPath: path.join( + value.artifactsDirectory, + 'database.pre-ql3.sqlite', + ), + sqliteManifestPath: path.join( + value.artifactsDirectory, + 'qinglong3-sqlite-adoption.json', + ), + activationPath: path.join( + value.artifactsDirectory, + 'qinglong3-sqlite-activation.json', + ), + stagingRoot: path.join(value.stagingParent, 'reviewed-data'), + }); + privateDirectory(path.dirname(value.sourcePath)); + createLegacyDatabase(value.sourcePath); + privateFile(path.join(value.dataRoot, 'db', 'keyv.sqlite'), 'legacy-keyv'); + privateFile(path.join(value.dataRoot, 'config', 'config.sh'), 'export A=1\n'); + privateFile( + path.join(value.dataRoot, 'scripts', 'jobs', 'example.sh'), + 'echo qinglong\n', + ); + privateFile( + path.join(value.dataRoot, 'upload', 'avatar.bin'), + Buffer.from([1, 2, 3]), + ); + privateFile( + path.join(value.dataRoot, 'ssh.d', 'repository-key'), + 'private-key', + ); + privateFile( + path.join(value.dataRoot, 'repo', 'cache', 'ignored'), + 'regenerate-me', + ); + privateFile( + path.join(value.dataRoot, 'log', 'history', 'ignored'), + 'retain-me', + ); + return value; +} + +function runRaw(value, name, operation, options) { + const commandPath = path.join(value.commandsDirectory, `${name}.json`); + fs.writeFileSync( + commandPath, + `${JSON.stringify({ schemaVersion: 1, operation, options })}\n`, + { mode: 0o600 }, + ); + fs.chmodSync(commandPath, 0o600); + return spawnSync( + process.execPath, + [BINARY, 'run', '--command-file', commandPath], + { encoding: 'utf8' }, + ); +} + +function run(value, name, operation, options) { + const child = runRaw(value, name, operation, options); + assert.equal(child.status, 0, child.stderr); + assert.equal(child.stderr, ''); + return { child, result: JSON.parse(child.stdout) }; +} + +function prepare(value) { + const base = { deploymentRoot: value.deploymentRoot, profile: 'edge' }; + const sqlitePlan = run( + value, + 'sqlite-inspect', + 'local-sqlite.adoption.inspect', + { ...base, sourcePath: value.sourcePath, legacyTimezone: 'UTC' }, + ).result; + run(value, 'sqlite-stage', 'local-sqlite.adoption.stage', { + ...base, + sourcePath: value.sourcePath, + targetPath: value.targetPath, + recoveryPath: value.recoveryPath, + manifestPath: value.sqliteManifestPath, + expectedPlanDigest: sqlitePlan.evidence.planDigest, + legacyTimezone: 'UTC', + }); + const sqliteVerified = run( + value, + 'sqlite-verify', + 'local-sqlite.adoption.verify', + { + ...base, + targetPath: value.targetPath, + recoveryPath: value.recoveryPath, + manifestPath: value.sqliteManifestPath, + }, + ).result; + const activation = run( + value, + 'sqlite-activate', + 'local-sqlite.activation.prepare', + { + ...base, + sourcePath: value.sourcePath, + targetPath: value.targetPath, + recoveryPath: value.recoveryPath, + manifestPath: value.sqliteManifestPath, + activationPath: value.activationPath, + expectedManifestDigest: sqliteVerified.evidence.manifestDigest, + }, + ).result; + const directoryPlan = run(value, 'directory-inspect', DIRECTORY_INSPECT, { + dataRoot: value.dataRoot, + profile: 'edge', + }).result; + return { + directoryPlanDigest: directoryPlan.evidence.planDigest, + activationDigest: activation.evidence.activationDigest, + }; +} + +function sqliteBinding(value, activationDigest) { + return { + sourcePath: value.sourcePath, + targetPath: value.targetPath, + recoveryPath: value.recoveryPath, + manifestPath: value.sqliteManifestPath, + activationPath: value.activationPath, + expectedActivationDigest: activationDigest, + }; +} + +function stageOptions(value, prepared) { + return { + deploymentRoot: value.deploymentRoot, + dataRoot: value.dataRoot, + stagingRoot: value.stagingRoot, + profile: 'edge', + expectedPlanDigest: prepared.directoryPlanDigest, + sqlite: sqliteBinding(value, prepared.activationDigest), + }; +} + +function verifyOptions(value, prepared, manifestDigest) { + return { + deploymentRoot: value.deploymentRoot, + dataRoot: value.dataRoot, + stagingRoot: value.stagingRoot, + profile: 'edge', + expectedManifestDigest: manifestDigest, + sqlite: sqliteBinding(value, prepared.activationDigest), + }; +} + +test('stages only reviewed payloads behind the real SQLite activation fence', (t) => { + const value = fixture(t); + const prepared = prepare(value); + const staged = run( + value, + 'directory-stage', + DIRECTORY_STAGE, + stageOptions(value, prepared), + ); + + assert.equal(staged.result.status, 'staged'); + assert.match(staged.result.evidence.manifestDigest, /^[0-9a-f]{64}$/); + assert.deepEqual(fs.readdirSync(value.stagingRoot).sort(), [ + 'manifest.json', + 'payload', + ]); + const expectedFiles = [ + ['payload', 'copy-reviewed', 'scripts', 'jobs', 'example.sh'], + ['payload', 'copy-reviewed', 'upload', 'avatar.bin'], + ['payload', 'transform-input', 'config', 'config.sh'], + ['payload', 'transform-input', 'db', 'keyv.sqlite'], + ['payload', 'transform-input', 'ssh.d', 'repository-key'], + ]; + for (const parts of expectedFiles) { + const filePath = path.join(value.stagingRoot, ...parts); + assert.equal(fs.statSync(filePath).mode & 0o777, 0o600); + } + assert.equal( + fs.existsSync( + path.join( + value.stagingRoot, + 'payload', + 'transform-input', + 'db', + 'database.sqlite', + ), + ), + false, + ); + assert.equal(staged.child.stdout.includes(value.dataRoot), false); + assert.equal(staged.child.stdout.includes('example.sh'), false); + assert.equal(staged.child.stdout.includes('private-key'), false); + + const verified = run( + value, + 'directory-verify', + DIRECTORY_VERIFY, + verifyOptions(value, prepared, staged.result.evidence.manifestDigest), + ).result; + assert.equal(verified.status, 'verified'); + assert.deepEqual(verified.evidence, staged.result.evidence); + const replayed = run( + value, + 'directory-verify-replay', + DIRECTORY_VERIFY, + verifyOptions(value, prepared, staged.result.evidence.manifestDigest), + ).result; + assert.deepEqual(replayed, verified); +}); + +test('verification rejects staged payload and source drift', (t) => { + const value = fixture(t); + const prepared = prepare(value); + const staged = run( + value, + 'stage-before-drift', + DIRECTORY_STAGE, + stageOptions(value, prepared), + ).result; + const stagedScript = path.join( + value.stagingRoot, + 'payload', + 'copy-reviewed', + 'scripts', + 'jobs', + 'example.sh', + ); + fs.writeFileSync(stagedScript, 'tampered\n'); + const targetDrift = runRaw( + value, + 'verify-target-drift', + DIRECTORY_VERIFY, + verifyOptions(value, prepared, staged.evidence.manifestDigest), + ); + assert.equal(targetDrift.status, 1); + assert.equal( + JSON.parse(targetDrift.stderr).code, + 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID', + ); + + fs.writeFileSync(stagedScript, 'echo qinglong\n'); + fs.chmodSync(stagedScript, 0o600); + + privateFile( + path.join(value.dataRoot, 'scripts', 'jobs', 'example.sh'), + 'source-drift\n', + ); + const sourceDrift = runRaw( + value, + 'verify-source-drift', + DIRECTORY_VERIFY, + verifyOptions(value, prepared, staged.evidence.manifestDigest), + ); + assert.equal(sourceDrift.status, 1); + assert.equal( + JSON.parse(sourceDrift.stderr).code, + 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID', + ); +}); + +test('verification never follows a staged payload symlink', (t) => { + const value = fixture(t); + const prepared = prepare(value); + const staged = run( + value, + 'stage-before-link', + DIRECTORY_STAGE, + stageOptions(value, prepared), + ).result; + const stagedScript = path.join( + value.stagingRoot, + 'payload', + 'copy-reviewed', + 'scripts', + 'jobs', + 'example.sh', + ); + fs.unlinkSync(stagedScript); + fs.symlinkSync(value.sourcePath, stagedScript); + + const child = runRaw( + value, + 'verify-link', + DIRECTORY_VERIFY, + verifyOptions(value, prepared, staged.evidence.manifestDigest), + ); + assert.equal(child.status, 1); + assert.equal(child.stdout, ''); + assert.equal( + JSON.parse(child.stderr).code, + 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID', + ); +}); + +test('staging is no-replace and fails before copying on activation drift', (t) => { + const value = fixture(t); + const prepared = prepare(value); + privateDirectory(value.stagingRoot); + privateFile(path.join(value.stagingRoot, '.incomplete'), 'crash-residue'); + const residue = runRaw( + value, + 'stage-residue', + DIRECTORY_STAGE, + stageOptions(value, prepared), + ); + assert.equal(residue.status, 1); + assert.equal( + fs.readFileSync(path.join(value.stagingRoot, '.incomplete'), 'utf8'), + 'crash-residue', + ); + fs.rmSync(value.stagingRoot, { recursive: true }); + + const drifted = stageOptions(value, prepared); + drifted.sqlite.expectedActivationDigest = '0'.repeat(64); + const activationDrift = runRaw( + value, + 'stage-activation-drift', + DIRECTORY_STAGE, + drifted, + ); + assert.equal(activationDrift.status, 1); + assert.equal(fs.existsSync(value.stagingRoot), false); +}); + +test('widened directory staging commands fail closed before source access', (t) => { + const value = fixture(t); + const child = runRaw(value, 'widened-stage', DIRECTORY_STAGE, { + deploymentRoot: value.deploymentRoot, + dataRoot: path.join(value.deploymentRoot, 'missing-source'), + stagingRoot: value.stagingRoot, + profile: 'edge', + expectedPlanDigest: '0'.repeat(64), + sqlite: { + sourcePath: path.join( + value.deploymentRoot, + 'missing-source', + 'db', + 'database.sqlite', + ), + targetPath: path.join(value.artifactsDirectory, 'missing-target'), + recoveryPath: path.join(value.artifactsDirectory, 'missing-recovery'), + manifestPath: path.join(value.artifactsDirectory, 'missing-manifest'), + activationPath: path.join(value.artifactsDirectory, 'missing-activation'), + expectedActivationDigest: '0'.repeat(64), + }, + extraAuthority: true, + }); + assert.equal(child.status, 1); + assert.equal(child.stdout, ''); + assert.equal( + JSON.parse(child.stderr).code, + 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID', + ); +}); diff --git a/scripts/ql3-cluster-dependency-audit.cjs b/scripts/ql3-cluster-dependency-audit.cjs index 1fe840f5..be81634f 100644 --- a/scripts/ql3-cluster-dependency-audit.cjs +++ b/scripts/ql3-cluster-dependency-audit.cjs @@ -1762,6 +1762,11 @@ function auditSourceImports(root, packagePath, findings) { 'src/lifecycle/sqlite-adoption/command.ts' && specifier === '@qinglong/local-admin' ) && + !( + path.relative(packageDirectory, filePath) === + 'src/lifecycle/data-directory-adoption/staging.ts' && + specifier === '@qinglong/local-admin/runtime' + ) && !( path.relative(packageDirectory, filePath) === 'src/plugin-package/pluginPackageCommand.ts' && diff --git a/test/back/ql3ClusterDependencyAudit.test.cjs b/test/back/ql3ClusterDependencyAudit.test.cjs index 4becb1d7..9a9b469e 100644 --- a/test/back/ql3ClusterDependencyAudit.test.cjs +++ b/test/back/ql3ClusterDependencyAudit.test.cjs @@ -1790,9 +1790,14 @@ test('confines adoption and Package command authorities to owner CLI subpaths', lifecycleDirectory, 'sqlite-adoption', ); + const dataDirectoryAdoptionDirectory = path.join( + lifecycleDirectory, + 'data-directory-adoption', + ); const pluginPackageDirectory = path.join(sourceDirectory, 'plugin-package'); fs.mkdirSync(lifecycleDirectory, { recursive: true }); fs.mkdirSync(sqliteAdoptionDirectory, { recursive: true }); + fs.mkdirSync(dataDirectoryAdoptionDirectory, { recursive: true }); fs.mkdirSync(pluginPackageDirectory, { recursive: true }); fs.writeFileSync( path.join(lifecycleDirectory, 'adoption.ts'), @@ -1811,6 +1816,13 @@ test('confines adoption and Package command authorities to owner CLI subpaths', "import { forbidden } from '@qinglong/local-admin/runtime';", ].join('\n'), ); + fs.writeFileSync( + path.join(dataDirectoryAdoptionDirectory, 'staging.ts'), + [ + "import { activation } from '@qinglong/local-admin/runtime';", + "import { forbidden } from '@qinglong/local-admin';", + ].join('\n'), + ); const applicationCommandDirectory = path.join( sourceDirectory, 'application-command', @@ -1880,8 +1892,12 @@ test('confines adoption and Package command authorities to owner CLI subpaths', }, { code: 'FORBIDDEN_LOCAL_ADOPTION_CLI_AUTHORITY_IMPORT', - file: - 'packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/command.ts', + file: 'packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/staging.ts', + specifier: '@qinglong/local-admin', + }, + { + code: 'FORBIDDEN_LOCAL_ADOPTION_CLI_AUTHORITY_IMPORT', + file: 'packages/ql3-local-owner-cli/src/lifecycle/sqlite-adoption/command.ts', specifier: '@qinglong/local-admin/runtime', }, { diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index bbc00ff5..ecca580b 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -207,10 +207,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: localOwnerCli.rootSourceFileRoles, }, { - sourceFiles: 118, + sourceFiles: 122, rootSourceFiles: 1, rootSourceLines: 50, - nestedSourceFiles: 117, + nestedSourceFiles: 121, rootSourceFileRoles: { 'cli.ts': 'binary_entry' }, }, );