From 69c322fa8d003377205cf06f15ec6f83cb210d0d Mon Sep 17 00:00:00 2001 From: whyour Date: Fri, 21 Aug 2026 10:56:24 +0800 Subject: [PATCH] feat(ql3): prepare legacy data transformation --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 17 + ...d-legacy-config-keyv-ssh-transformation.md | 164 +++++++ docs/adr/README.md | 1 + .../ql3-local-data-directory-adoption.md | 101 +++- .../data-directory-adoption/command.ts | 18 +- .../data-directory-adoption/contract.ts | 115 ++++- .../transformation/config.ts | 218 +++++++++ .../transformation/files.ts | 316 ++++++++++++ .../transformation/keyv.ts | 351 ++++++++++++++ .../transformation/manifest.ts | 260 ++++++++++ .../transformation/model.ts | 430 +++++++++++++++++ .../transformation/ssh.ts | 224 +++++++++ .../transformation/transformation.ts | 248 ++++++++++ .../dataDirectoryAdoptionStageCli.test.cjs | 448 ++++++++++++++++++ test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 15 files changed, 2878 insertions(+), 37 deletions(-) create mode 100644 docs/adr/ADR-0479-private-versioned-legacy-config-keyv-ssh-transformation.md create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/config.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/files.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/keyv.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/manifest.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/model.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/ssh.ts create mode 100644 packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/transformation.ts diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index cee9baf0..58f1fa0e 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,23 @@ 最新增量证据（2026-08-21）： +- D-386/ADR-0479（已接受）：把 D-385 的 `config/db/ssh.d` 私有 snapshot 转成 no-replace、版本化、Project-bound 的 prepared model， + 新增 exact `local-data-directory.adoption.transform|transform.verify`。`config.sh` 永不执行，只把简单非空 export 写入独立 + `qinglong3-local-secret-value` 文件，非 export setting 退役，复杂/重复行转人工复核；Keyv 以 read-only defensive SQLite 和 reviewed + v4 schema 有界读取，`authInfo` 固定淘汰并要求 credential reissue，`apps/lang` 只与主数据库 reconciliation，未知 row/schema 转人工； + SSH 只接受私钥/config 严格配对，私钥转 Secret，`ProxyCommand` 文本和 host-key bypass 永不继承，binding 固定 disabled 并要求 operator + 验证 host key。私有 `model/` 含待认证导入材料，根 manifest/stdout 只含摘要；转换前后重复验证 D-385 source，目标 exact verify 绑定 + stage、Project/path 摘要与 transformation digest。`.incomplete` 保留失败现场，成功只表示 `prepared`，不授权目标数据库写入或激活。 + Edge/Standalone Secret 上限为 128/512，Keyv 上限为 256/2,048 rows 和 4/16 MiB；无网络、常驻资源、新 package、依赖、binary 或 + 部署对象。代码内聚在 `lifecycle/data-directory-adoption/transformation/` 的七个职责文件，workspace 仍为 18 packages、 + `singleSourcePackages=[]`、`shallowSourcePackages=[]`，Local Owner 为 `129 source / 128 nested / 1 root binary entry`。D-386 data + directory 聚焦套件 `10/10`；Local Owner `205 total / 200 pass / 5 conditional skip / 0 fail`，backend + `1,535 total / 1,533 pass / 2 conditional skip / 0 fail`，`pnpm build:back` 与 18-package clean build/逐包测试通过。八项架构 + 审计和按顺序执行的 14 档 artifact audit 全 compatible；基础 Edge/Standalone `2,598,669 / 2,598,747` bytes、316 files、 + 57 modules，Adopted `2,818,404 / 2,818,527` bytes、336 files、58 modules，Application+AI + `4,502,262 / 4,502,394` bytes、511 files、141 modules，MCP `7,324,601 / 7,324,709` bytes、802 files、227 modules，均与 + D-385 相同，证明转换 authority 未进入低配常驻闭包。本切片不改变 PostgreSQL 语义，因此不重新占有 HA 证明。D-387 应实现受认证、 + 可审计、可重放的 prepared-model apply/commit 与 Secret 回收，而不是在转换器中混入写 authority。 - D-385/ADR-0478（已接受）：把 D-384 只读 data directory plan 推进为产品级私有 stage/verify。既有一次性 `ql3-adoption` 新增 exact `local-data-directory.adoption.stage|verify`；`stagingRoot` 必须是私有 deployment root 内、legacy data root 外的 no-replace 路径。stage 只把 `scripts/upload` 放入 `copy-reviewed`，把 `config/db/ssh.d` 放入 `transform-input`，排除 diff --git a/docs/adr/ADR-0479-private-versioned-legacy-config-keyv-ssh-transformation.md b/docs/adr/ADR-0479-private-versioned-legacy-config-keyv-ssh-transformation.md new file mode 100644 index 00000000..138c8c45 --- /dev/null +++ b/docs/adr/ADR-0479-private-versioned-legacy-config-keyv-ssh-transformation.md @@ -0,0 +1,164 @@ +# ADR-0479：私有、版本化的 Legacy Config、Keyv 与 SSH 转换 + +- 状态：Accepted +- 日期：2026-08-21 +- 关联：QL-RFC-0001、ADR-0207、ADR-0395、ADR-0476、ADR-0477、ADR-0478 + +## 上下文 + +ADR-0478 已把 QingLong 2.x `config`、`db` 与 `ssh.d` 固定为私有、稳定、可重复校验的 `transform-input`，但暂存副本还不能直接成为 +3.0 配置。三类输入具有不同风险： + +- `config/config.sh` 是可执行 shell，不是声明式配置；直接 `source` 会把命令替换、文件读取和进程启动引入迁移 authority； +- Keyv v4 的 `keyv(key,value)` 同时缓存 `authInfo`、`apps` 和 `lang`；其中认证材料必须作废，`apps/lang` 又与主数据库事实重复； +- `ssh.d` 同时保存私钥和可执行 SSH client policy，历史配置可能含 `ProxyCommand` 与 `StrictHostKeyChecking no`。 + +QingLong 3.0 同时面向低配路由设备和集群节点。转换必须是短生命周期、有硬预算、无网络、无常驻缓存的本地 ceremony；同时产物要绑定 +Project，以便后续由 Local Owner 或 Cluster separation-of-duty 流程认证提交，而不是让迁移工具直接获得目标数据库写 authority。 + +## 决策 + +### 1. 扩展既有领域而不增加 workspace package + +在 `@qinglong/local-owner-cli` 的 `lifecycle/data-directory-adoption/transformation/` 增加两个 exact operation： + +- `local-data-directory.adoption.transform`； +- `local-data-directory.adoption.transform.verify`。 + +实现按 `config`、`keyv`、`ssh`、私有文件系统、目标模型、manifest 与 orchestration 分层，但仍属于同一个一次性 Local Owner capability。 +不新增 workspace package、第三方依赖、binary、daemon、listener、watcher、timer、网络请求或目标数据库连接。 + +### 2. 只转换已验证的 D-385 snapshot + +命令提交完整 D-385 binding：`deploymentRoot`、`dataRoot`、`stagingRoot`、`profile`、SQLite binding 与 +`expectedManifestDigest`，并增加 `transformationRoot` 和 `projectId`。transform 在写目标前验证一次 D-385 stage，转换后再次验证并要求 +低敏 evidence 逐字段相同；transform.verify 在目标校验前后执行同样的源验证。转换只读取 +`stagingRoot/payload/transform-input`，不从当前 2.x `dataRoot` 直接解析秘密。 + +`transformationRoot` 必须是私有 `deploymentRoot` 内、`dataRoot` 与 `stagingRoot` 外的不存在路径；其父目录为当前 UID 拥有的 canonical +`0700` 目录。所有目标目录为 `0700`、文件为 `0600`，写入均 no-replace。 + +### 3. `config.sh` 永不执行 + +转换器不调用 shell，也不展开变量、命令替换、转义或重定向。它只识别单行的简单 assignment：可选 `export`、受限 identifier，以及 +简单单引号、无插值双引号或安全 unquoted literal。 + +- 非空 exported assignment 生成 Project-scoped Secret import draft 和独立 `qinglong3-local-secret-value` 私有文件； +- 空 export 只计数并省略； +- 非 export assignment 作为已退役 legacy setting，只记录名称和值摘要； +- 重复 assignment、无法声明式解析的行和其他 config 资产都使 assessment 变为 `manual_required`，不会被执行或激活。 + +单个 `config.sh` 上限 256 KiB，单个 Secret 明文上限 16 KiB。 + +### 4. Keyv 只读、防御式读取并消除旧认证 authority + +只接受 reviewed Keyv v4 shape：`keyv(key VARCHAR(255) PRIMARY KEY, value TEXT)`。SQLite 以 read-only、defensive、 +`trusted_schema=OFF`、`query_only=ON`、`mmap_size=0` 打开，执行有界 `integrity_check(1)`；打开前后文件身份必须稳定。 + +- `keyv:authInfo` 映射为 `credential_reissue/retired`，只保留值摘要，不复制认证材料； +- `keyv:apps` 映射为 `main_database_apps_reconciliation`； +- `keyv:lang` 只接受 Keyv envelope 中的 `zh|en`，映射为主数据库 system setting reconciliation； +- 未知 row、未知 schema object、非法 locale 或其他数据库资产只计数/摘要并转人工复核。 + +Edge 最多读取 256 rows/4 MiB，Standalone 最多 2,048 rows/16 MiB；单 row 最高 1 MiB。Keyv 模型始终 +`activation=disabled`，不会写目标数据库。 + +### 5. SSH 私钥与历史 client policy 分离 + +只将同目录 `` 私钥与 `.config` 的严格配对转成 Secret draft 和 disabled binding。私钥限制 16 KiB，config 限制 64 KiB； +config 必须恰有一个 `Host` 和一个 basename 与 alias 相同的 `IdentityFile`。 + +目标只保存 host pattern、配置摘要和风险布尔值。历史 `ProxyCommand` 文本和 `StrictHostKeyChecking no` 不会继承；所有 binding 固定 +`hostKeyPolicy=operator_verification_required`、`activation=disabled`。无法配对、无法解析或嵌套的条目只进入人工复核证据。 + +### 6. 目标模型、内容无关 manifest 与恢复状态分离 + +成功产物固定为： + +```text +transformationRoot/ + manifest.json + model/ + config.json + keyv.json + ssh.json + secret-imports.json + manual-review.json + secret-values/*.json +``` + +`model/` 是私有准备材料，会包含 Project ID、环境变量名、SSH alias 和待认证导入的明文 Secret；它不是日志、普通备份或可发布制品。 +根 `manifest.json` 只包含 Profile、时间、Project/path 摘要、D-385 manifest digest、assessment、三类源的计数/摘要和目标模型树摘要， +不包含 Project ID、用户名称、文件名或秘密内容。transform.verify 要求根与模型 exact shape、所有模式/摘要/Secret value digest 和 +`expectedTransformationDigest` 精确匹配。 + +transform 创建根后先持久化 `.incomplete`，只有 model、manifest 和最终静态校验完成后才删除。失败不会自动删除或覆盖残留；D-385 +staging 继续是转换输入和恢复权威。成功状态仅为 `prepared`，不授权把 Secret、settings 或 SSH binding 写入/激活到 3.0。 + +### 7. Profile 资源边界 + +转换器不扫描 D-385 未纳入的资产，不启动 worker，不联网。Secret draft 总数在 Edge 最多 128、Standalone 最多 512；各输入还有独立 +文件、row 和字节上限。超限失败时保留 `.incomplete`，operator 应保留现场、选择新的空路径重试，不能放宽成无界导入。 + +## 被拒绝的替代方案 + +### 执行或 source `config.sh` + +拒绝。迁移输入不是可信程序，执行会把本机命令、文件和网络 authority 隐式授予旧配置。 + +### 原样复制 Keyv 或复用 `authInfo` + +拒绝。Keyv 是重复 cache，不是 3.0 认证事实源；携带旧 token 会绕过 credential reissue，并可能覆盖主 SQLite 中更新的 App/System 事实。 + +### 原样启用 SSH config + +拒绝。`ProxyCommand` 是进程执行面，host-key bypass 会把历史不安全策略升级为 3.0 默认策略。私钥、host identity 与 client policy 必须分离。 + +### 转换时直接提交目标数据库 + +拒绝。解析 legacy 输入和认证目标 mutation 是两个 authority 阶段。先发布可验证 prepared model，后续再由受认证、可审计、可重放的 apply +流程提交，才能保留失败恢复点和 Cluster separation-of-duty。 + +### 为每类转换再拆 workspace package + +拒绝。三类转换共享同一个 D-385 source fence、manifest、恢复状态和部署生命周期。按领域子目录内聚能解决 `src` 平铺问题，同时避免 +只有一两个文件、没有独立部署或依赖生命周期的 package。 + +## 影响 + +### 正面 + +- shell 配置、Keyv cache 和 SSH client policy 首次获得显式、版本化的 3.0 处置语义； +- 旧认证材料不会进入目标 Secret，危险 SSH policy 不会被继承； +- 私有 target model 可稳定验证，根 manifest 和 stdout 保持低敏； +- Edge 与 Standalone 使用同一协议、不同硬预算；Cluster 后续可复用 prepared model，但不能绕过审批； +- D-385 staging、失败 `.incomplete` 和成功 prepared model 各自具有清晰恢复角色。 + +### 代价与限制 + +- 简单 parser 会把复杂但可能无害的 shell 行转人工复核，这是有意的失败关闭； +- prepared model 含明文 Secret，operator 必须把整个 transformation root 当作高敏私有材料； +- 本阶段不迁移 `scripts/upload`，不提交目标数据库，不激活 SSH，不删除 legacy data； +- 尚未完成固定物理 Edge 的 RSS/I/O/ENOSPC/断电证明，也未接入 systemd/OpenRC/Compose cutover lineage。 + +## 验证 + +- D-386 data directory 聚焦套件 `10/10`，覆盖真实 ADR-0476 SQLite 链、D-385 stage、三类转换、exact verify/replay； +- 覆盖 stdout/manifest 脱敏、旧认证淘汰、SSH policy 禁用、目标与源 drift、未知内容 manual review、Edge Secret 预算、no-replace + crash residue 与扩权 command； +- Local Owner `205 total / 200 pass / 5 conditional skip / 0 fail`；backend + `1,535 total / 1,533 pass / 2 conditional skip / 0 fail`，`pnpm build:back` 通过； +- 18-package clean build/逐包测试单次退出 0；package boundary、Cluster dependency、Edge import、Service Bridge import、 + Cluster/Worker deployment、Console 与 Console distribution 八项架构审计全部 compatible/passed； +- 14 档 artifact audit 按顺序全部 compatible；基础 Edge/Standalone `2,598,669 / 2,598,747` bytes、316 files、57 modules， + Adopted `2,818,404 / 2,818,527` bytes、336 files、58 modules，Application+AI `4,502,262 / 4,502,394` bytes、511 files、 + 141 modules，MCP `7,324,601 / 7,324,709` bytes、802 files、227 modules； +- workspace 保持 18 packages、`singleSourcePackages=[]`、`shallowSourcePackages=[]`；Local Owner 为 + `129 source / 128 nested / 1 root binary entry`；GitNexus change audit 是提交前最后门禁； +- 本切片不修改 PostgreSQL schema、ACL、repository、role、Pool、连接或 failover 语义，不重新占有 PostgreSQL HA 证明。 + +## 后续 + +- D-387：为 prepared model 设计受认证、可审计、可重放的 apply/commit 与 Secret 零化/回收协议； +- 在固定物理 Edge/NAS 上执行 stage/transform/verify 的 RSS、I/O、磁盘峰值、ENOSPC 与受控断电演练； +- 将 transformation digest 接入 systemd/OpenRC/Compose cutover、rollback 和发布证据 lineage。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 0b2151b6..80b9ae9b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -482,6 +482,7 @@ | [ADR-0476](./ADR-0476-real-legacy-sqlite-upgrade-and-rollback-rehearsal.md) | 真实 Legacy SQLite 升级与回滚演练 | Accepted | | [ADR-0477](./ADR-0477-bounded-legacy-data-directory-inventory.md) | 有界 Legacy Data Directory 盘点 | Accepted | | [ADR-0478](./ADR-0478-private-legacy-data-directory-staging.md) | 私有 Legacy Data Directory 暂存与稳定校验 | Accepted | +| [ADR-0479](./ADR-0479-private-versioned-legacy-config-keyv-ssh-transformation.md) | 私有、版本化的 Legacy Config、Keyv 与 SSH 转换 | Accepted | ## 规则 diff --git a/docs/operations/ql3-local-data-directory-adoption.md b/docs/operations/ql3-local-data-directory-adoption.md index 25b8f0a7..344d507e 100644 --- a/docs/operations/ql3-local-data-directory-adoption.md +++ b/docs/operations/ql3-local-data-directory-adoption.md @@ -1,7 +1,7 @@ # QingLong 2.x Data Directory 接管 本流程先为完整 QingLong 2.x `data` 目录生成一个只读、确定性、按 Profile 有界的 3.0 接管计划，再把审核过的资产 no-replace -暂存并稳定校验。它不会删除或修改源文件，也不替代 [SQLite 接管流程](./ql3-local-sqlite-adoption.md)。 +暂存、转换并稳定校验。它不会删除或修改源文件，也不替代 [SQLite 接管流程](./ql3-local-sqlite-adoption.md)。 ## 1. 前置条件 @@ -13,6 +13,8 @@ - stage 前必须已经完成 SQLite inspect、stage、verify 与 activation，并保留五个绝对路径和 `activationDigest`； - `deploymentRoot` 与 `stagingRoot` 的父目录必须是当前 UID 拥有的 canonical `0700` 目录；`stagingRoot` 必须尚不存在且位于 `deploymentRoot` 内、`dataRoot` 外。 +- transform 前创建私有 canonical `0700` transformation parent；`transformationRoot` 必须尚不存在，位于 `deploymentRoot` 内，且与 + `dataRoot`、`stagingRoot` 互不包含。 ## 2. 执行 inspect @@ -146,16 +148,91 @@ ql3-adoption run --command-file /secure/operator/ql3-data-directory-inspect.json verify 会重新验证目录计划、SQLite activation/source/target、清单 exact shape、私有权限和完整 payload 语义摘要。成功结果为 `status=verified`，且 evidence 应与 stage 的低敏 evidence 一致。 -## 8. 崩溃残留 +## 8. 准备版本化转换模型 -stage 创建暂存根后立即写入 `.incomplete`。只有 payload 和 `manifest.json` 都持久化后才删除它。命令失败或进程崩溃后： +只有 stage verify 成功且 operator 已保存 `manifestDigest` 后才执行 transform。`projectId` 是后续 Secret 与 binding 的目标 Project，不能 +使用占位值或其他环境的 Project： + +```json +{ + "schemaVersion": 1, + "operation": "local-data-directory.adoption.transform", + "options": { + "deploymentRoot": "/opt/qinglong3/adoption", + "dataRoot": "/opt/qinglong/data", + "stagingRoot": "/opt/qinglong3/adoption/staging/reviewed-data", + "transformationRoot": "/opt/qinglong3/adoption/transformations/reviewed-data-v1", + "projectId": "", + "profile": "edge", + "expectedManifestDigest": "<64-hex-directory-manifest-digest>", + "sqlite": { + "sourcePath": "/opt/qinglong/data/db/database.sqlite", + "targetPath": "/opt/qinglong3/adoption/sqlite/qinglong3.sqlite", + "recoveryPath": "/opt/qinglong3/adoption/sqlite/database.pre-ql3.sqlite", + "manifestPath": "/opt/qinglong3/adoption/sqlite/adoption.json", + "activationPath": "/opt/qinglong3/adoption/sqlite/activation.json", + "expectedActivationDigest": "<64-hex-sqlite-activation-digest>" + } + } +} +``` + +成功返回 `status=prepared`、`transformationDigest` 和低敏计数/摘要。私有 `model/` 包含 Project ID、环境变量名、SSH alias 和待导入 +Secret 明文，必须按高敏迁移材料保护，不能写入日志、提交 Git、上传 issue 或复制进普通备份。 + +转换规则固定如下： + +- 不执行 `config.sh`；简单非空 export 生成 Secret input，非 export setting 退役，复杂/重复行要求人工复核； +- `keyv:authInfo` 淘汰且必须重新签发 credential；`apps/lang` 只与主 SQLite authority reconciliation；未知 Keyv 内容人工复核； +- SSH 私钥和 config 分离；历史 `ProxyCommand` 与 `StrictHostKeyChecking no` 不继承，所有 binding 保持 disabled，必须人工核验 host key； +- Edge 最多 128 个 Secret draft、256 个 Keyv row/4 MiB；Standalone 为 512 个 Secret、2,048 row/16 MiB；单 Secret 16 KiB。 + +`assessment=manual_required` 不表示数据丢失：原始内容仍在 D-385 staging 中。operator 应审核 `manual-review.json` 和对应原始 snapshot， +不得手改 manifest 或把 disabled binding 直接启用。 + +## 9. 校验转换模型 + +保存 transform 返回的 `transformationDigest`，在相同 options 上增加该字段并执行： + +```json +{ + "schemaVersion": 1, + "operation": "local-data-directory.adoption.transform.verify", + "options": { + "deploymentRoot": "/opt/qinglong3/adoption", + "dataRoot": "/opt/qinglong/data", + "stagingRoot": "/opt/qinglong3/adoption/staging/reviewed-data", + "transformationRoot": "/opt/qinglong3/adoption/transformations/reviewed-data-v1", + "projectId": "", + "profile": "edge", + "expectedManifestDigest": "<64-hex-directory-manifest-digest>", + "expectedTransformationDigest": "<64-hex-transformation-digest>", + "sqlite": { + "sourcePath": "/opt/qinglong/data/db/database.sqlite", + "targetPath": "/opt/qinglong3/adoption/sqlite/qinglong3.sqlite", + "recoveryPath": "/opt/qinglong3/adoption/sqlite/database.pre-ql3.sqlite", + "manifestPath": "/opt/qinglong3/adoption/sqlite/adoption.json", + "activationPath": "/opt/qinglong3/adoption/sqlite/activation.json", + "expectedActivationDigest": "<64-hex-sqlite-activation-digest>" + } + } +} +``` + +成功返回 `status=verified`，evidence 应与 transform 完全相同。verify 会在目标校验前后重新验证 D-385 staging、当前 2.x source 和 SQLite +activation，并检查目标 exact 文件集、私有 mode、Project/profile/path binding、Secret value digest 和完整模型树摘要。 + +## 10. 崩溃残留与恢复 + +stage 和 transform 都在创建目标根后立即写入 `.incomplete`。只有 payload/model 和 `manifest.json` 都持久化且静态校验通过后才删除。 +命令失败或进程崩溃后： - 不要直接把残留目录当作恢复资产； -- 不要在原路径重试，stage 会 no-replace 拒绝； +- 不要在原路径重试，stage/transform 会 no-replace 拒绝； - 先保存现场用于诊断，再由 operator 显式移走残留目录，并使用一个新的空路径重试； - verify 遇到 `.incomplete`、额外文件或缺失文件一律失败关闭。 -## 9. 常见失败 +## 11. 常见失败 - 根目录或条目 group/world 可写：修正 ownership/permission 后重新盘点； - symlink、硬链接或特殊文件：保留现场，确认来源和目标后人工处置；盘点不会跟随或读取； @@ -165,16 +242,20 @@ stage 创建暂存根后立即写入 `.incomplete`。只有 payload 和 `manifes - activation 不匹配：重新执行 SQLite verify/activation，不能只替换 digest； - `stagingRoot` 已存在：检查是否为崩溃残留，禁止覆盖或合并； - stage/verify 后源或目标 drift：停止所有 writer，回到 inspect，生成并重新审核新的 plan。 +- `transformationRoot` 已存在或有 `.incomplete`：保留诊断现场，显式移走后选择新的空路径，禁止覆盖或续写； +- Keyv schema、row 或字节超限：不要把数据库当 JSON dump 绕过，先确认版本和未知数据归属； +- `manual_required`：从 D-385 staging 审核相应原始输入；不要执行旧 shell、复用 `authInfo` 或启用旧 SSH config； +- transform verify drift：停止后续 apply，保留 D-385 staging 和 transformation root，重新建立新的版本化转换路径。 -## 10. 当前边界 +## 12. 当前边界 -本流程已经提供 inspect、私有 stage 和稳定 verify。它仍不： +本流程已经提供 inspect、私有 stage、版本化 transform 和两级稳定 verify。它仍不： - 删除或修改任何源资产； -- 把 `config`、Keyv 或 `ssh.d` 自动转换为 3.0 目标模型； +- 把 prepared Secret/settings/SSH binding 写入或激活到 3.0 目标数据库； - 把历史日志/备份复制到默认目标，或复用跨架构 repo/dependency cache； - 授权 service-manager/Compose cutover 或 Legacy rollback； - 证明固定物理路由器/NAS 上的耗时、RSS、I/O、磁盘峰值和断电恢复。 -只有 `status=verified` 仍不是 cutover 授权。继续保留原始 2.x data directory、SQLite recovery 和两份 manifest，等待后续转换与部署 -lineage 完成。 +即使 transform 返回 `status=prepared` 且 transform.verify 返回 `status=verified`，仍不是 apply 或 cutover 授权。继续保留原始 2.x +data directory、SQLite recovery、D-385 staging 和两份 manifest，等待受认证 apply/commit 与部署 lineage 完成。 diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/command.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/command.ts index 316764d6..34a27dd6 100644 --- a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/command.ts +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/command.ts @@ -1,6 +1,8 @@ import { LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION, LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION, + LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION, + LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION, normalizeLocalDataDirectoryAdoptionCommand, } from './contract'; import { @@ -12,10 +14,16 @@ import { verifyLocalDataDirectoryAdoption, type LocalDataDirectoryAdoptionMutationResult, } from './staging'; +import { + transformLocalDataDirectoryAdoption, + verifyLocalDataDirectoryAdoptionTransformation, + type LocalDataDirectoryTransformationResult, +} from './transformation/transformation'; export type LocalDataDirectoryAdoptionProductCommandResult = | LocalDataDirectoryAdoptionInspectResult - | LocalDataDirectoryAdoptionMutationResult; + | LocalDataDirectoryAdoptionMutationResult + | LocalDataDirectoryTransformationResult; export async function runLocalDataDirectoryAdoptionProductCommand( value: unknown, @@ -27,5 +35,11 @@ export async function runLocalDataDirectoryAdoptionProductCommand( if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION) { return stageLocalDataDirectoryAdoption(command); } - return verifyLocalDataDirectoryAdoption(command); + if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION) { + return verifyLocalDataDirectoryAdoption(command); + } + if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION) { + return transformLocalDataDirectoryAdoption(command); + } + return verifyLocalDataDirectoryAdoptionTransformation(command); } diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/contract.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/contract.ts index f7447273..618b237d 100644 --- a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/contract.ts +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/contract.ts @@ -2,6 +2,7 @@ import path from 'node:path'; const MAX_PATH_BYTES = 4_096; const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const PROJECT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; export const LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION = 'local-data-directory.adoption.inspect' as const; @@ -9,11 +10,17 @@ export const LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION = 'local-data-directory.adoption.stage' as const; export const LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION = 'local-data-directory.adoption.verify' as const; +export const LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION = + 'local-data-directory.adoption.transform' as const; +export const LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION = + 'local-data-directory.adoption.transform.verify' as const; export type LocalDataDirectoryAdoptionOperation = | typeof LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION | typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION - | typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION; + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION; export interface InspectLocalDataDirectoryAdoptionCommand { readonly schemaVersion: 1; @@ -57,10 +64,33 @@ export interface VerifyLocalDataDirectoryAdoptionCommand { }; } +interface LocalDataDirectoryAdoptionTransformationOptions + extends LocalDataDirectoryAdoptionMutationOptions { + readonly transformationRoot: string; + readonly projectId: string; + readonly expectedManifestDigest: string; +} + +export interface TransformLocalDataDirectoryAdoptionCommand { + readonly schemaVersion: 1; + readonly operation: typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION; + readonly options: LocalDataDirectoryAdoptionTransformationOptions; +} + +export interface VerifyLocalDataDirectoryAdoptionTransformationCommand { + readonly schemaVersion: 1; + readonly operation: typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION; + readonly options: LocalDataDirectoryAdoptionTransformationOptions & { + readonly expectedTransformationDigest: string; + }; +} + export type LocalDataDirectoryAdoptionCommand = | InspectLocalDataDirectoryAdoptionCommand | StageLocalDataDirectoryAdoptionCommand - | VerifyLocalDataDirectoryAdoptionCommand; + | VerifyLocalDataDirectoryAdoptionCommand + | TransformLocalDataDirectoryAdoptionCommand + | VerifyLocalDataDirectoryAdoptionTransformationCommand; export class LocalDataDirectoryAdoptionConfigurationError extends TypeError { readonly code = 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID'; @@ -97,7 +127,9 @@ export function isLocalDataDirectoryAdoptionOperation( return ( value === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION || value === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION || - value === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION + value === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION || + value === LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION || + value === LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION ); } @@ -187,26 +219,37 @@ export function normalizeLocalDataDirectoryAdoptionCommand( ); } const options = candidate.options as Record; - const expectedKeys = - candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION - ? ['dataRoot', 'profile'] - : candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION - ? [ - 'dataRoot', - 'deploymentRoot', - 'expectedPlanDigest', - 'profile', - 'sqlite', - 'stagingRoot', - ] - : [ - 'dataRoot', - 'deploymentRoot', - 'expectedManifestDigest', - 'profile', - 'sqlite', - 'stagingRoot', - ]; + let expectedKeys: readonly string[]; + if (candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) { + expectedKeys = ['dataRoot', 'profile']; + } else if ( + candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION + ) { + expectedKeys = [ + 'dataRoot', + 'deploymentRoot', + 'expectedPlanDigest', + 'profile', + 'sqlite', + 'stagingRoot', + ]; + } else { + expectedKeys = [ + 'dataRoot', + 'deploymentRoot', + 'expectedManifestDigest', + 'profile', + 'sqlite', + 'stagingRoot', + ...(candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION + ? [] + : ['projectId', 'transformationRoot']), + ...(candidate.operation === + LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION + ? ['expectedTransformationDigest'] + : []), + ]; + } if ( !exactKeys(options, expectedKeys) || !normalizedAbsolutePath(options.dataRoot) || @@ -235,6 +278,32 @@ export function normalizeLocalDataDirectoryAdoptionCommand( 'reviewed adoption digest is invalid', ); } + if ( + candidate.operation === + LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION || + candidate.operation === + LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION + ) { + if ( + !normalizedAbsolutePath(options.transformationRoot) || + typeof options.projectId !== 'string' || + !PROJECT_ID_PATTERN.test(options.projectId) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation target binding is invalid', + ); + } + if ( + candidate.operation === + LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION && + (typeof options.expectedTransformationDigest !== 'string' || + !DIGEST_PATTERN.test(options.expectedTransformationDigest)) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation digest is invalid', + ); + } + } } return Object.freeze(value as LocalDataDirectoryAdoptionCommand); } diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/config.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/config.ts new file mode 100644 index 00000000..5400eb69 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/config.ts @@ -0,0 +1,218 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { LocalDataDirectoryAdoptionConfigurationError } from '../contract'; +import { sha256Text } from '../manifest'; +import { + optionalPrivateDirectory, + readStablePrivateUtf8File, + summarizePrivateTree, + type PrivateTreeEvidence, +} from './files'; + +const MAX_CONFIG_BYTES = 256 * 1024; +const MAX_SECRET_BYTES = 16 * 1024; +const NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; +const SAFE_UNQUOTED_PATTERN = /^[A-Za-z0-9_./:@%,+-]*$/; + +export interface SecretImportDraft { + readonly kind: 'environment' | 'ssh_private_key'; + readonly sourceName: string; + readonly targetName: string; + readonly value: string; +} + +export interface ConfigTransformationModel { + readonly schema: 'qinglong/legacy-config-transformation@v1'; + readonly exportedEnvironment: readonly Readonly<{ + environmentName: string; + targetSecretName: string; + }>[]; + readonly retiredSettings: readonly Readonly<{ + name: string; + valueDigest: string; + }>[]; + readonly omittedEmptyExports: number; + readonly duplicateAssignments: number; + readonly unsupportedLines: number; + readonly unsupportedLineDigest: string; + readonly disabledAssetEntries: number; + readonly activation: 'disabled'; +} + +export interface ConfigTransformation { + readonly source: Readonly | null; + readonly model: Readonly; + readonly secrets: readonly Readonly[]; + readonly assessment: 'ready' | 'manual_required'; +} + +function literal(value: string): string | null { + if (value.length === 0) return ''; + if (value.startsWith("'") && value.endsWith("'")) { + const inner = value.slice(1, -1); + return inner.includes("'") ? null : inner; + } + if (value.startsWith('"') && value.endsWith('"')) { + const inner = value.slice(1, -1); + return /["`$\\]/.test(inner) ? null : inner; + } + return SAFE_UNQUOTED_PATTERN.test(value) ? value : null; +} + +function targetSecretName(name: string): string { + return `legacy-env-${sha256Text(name).slice(0, 32)}`; +} + +function emptyModel(): Readonly { + return Object.freeze({ + schema: 'qinglong/legacy-config-transformation@v1', + exportedEnvironment: Object.freeze([]), + retiredSettings: Object.freeze([]), + omittedEmptyExports: 0, + duplicateAssignments: 0, + unsupportedLines: 0, + unsupportedLineDigest: sha256Text(''), + disabledAssetEntries: 0, + activation: 'disabled', + }); +} + +export function transformLegacyConfig( + categoryRoot: string, + uid: number, +): Readonly { + if ( + !optionalPrivateDirectory(categoryRoot, uid, 'config transformation input') + ) { + return Object.freeze({ + source: null, + model: emptyModel(), + secrets: Object.freeze([]), + assessment: 'ready', + }); + } + const source = summarizePrivateTree(categoryRoot, uid); + const configPath = path.join(categoryRoot, 'config.sh'); + let configExists = false; + try { + const stat = fs.lstatSync(configPath); + configExists = stat.isFile() && !stat.isSymbolicLink(); + } catch (error) { + if ( + !error || + typeof error !== 'object' || + !('code' in error) || + error.code !== 'ENOENT' + ) { + throw error; + } + } + if (!configExists) { + const model = Object.freeze({ + ...emptyModel(), + disabledAssetEntries: source.entries, + }); + return Object.freeze({ + source, + model, + secrets: Object.freeze([]), + assessment: source.entries === 0 ? 'ready' : 'manual_required', + }); + } + + const content = readStablePrivateUtf8File( + configPath, + uid, + MAX_CONFIG_BYTES, + 'legacy config.sh', + ); + const recognized = new Map< + string, + { readonly exported: boolean; readonly value: string } + >(); + const duplicates = new Set(); + let unsupportedLines = 0; + const unsupportedHash = crypto.createHash('sha256'); + const lines = content.split('\n'); + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]!.endsWith('\r') + ? lines[index]!.slice(0, -1) + : lines[index]!; + if (line.trim().length === 0 || line.trimStart().startsWith('#')) continue; + const match = + /^(?:(export)[ \t]+)?([A-Za-z_][A-Za-z0-9_]{0,127})=(.*)$/.exec(line); + const parsed = match ? literal(match[3]!) : null; + if (!match || !NAME_PATTERN.test(match[2]!) || parsed === null) { + unsupportedLines += 1; + unsupportedHash.update(`${index + 1}:${sha256Text(line)}\n`, 'utf8'); + continue; + } + const name = match[2]!; + if (recognized.has(name)) duplicates.add(name); + recognized.set(name, { exported: match[1] === 'export', value: parsed }); + } + + const exportedEnvironment: Array<{ + environmentName: string; + targetSecretName: string; + }> = []; + const retiredSettings: Array<{ name: string; valueDigest: string }> = []; + const secrets: SecretImportDraft[] = []; + let omittedEmptyExports = 0; + for (const name of [...recognized.keys()].sort((left, right) => + Buffer.compare(Buffer.from(left), Buffer.from(right)), + )) { + const entry = recognized.get(name)!; + if (duplicates.has(name)) continue; + if (!entry.exported) { + retiredSettings.push({ name, valueDigest: sha256Text(entry.value) }); + continue; + } + if (entry.value.length === 0) { + omittedEmptyExports += 1; + continue; + } + if ( + entry.value.includes('\0') || + Buffer.byteLength(entry.value, 'utf8') > MAX_SECRET_BYTES + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'legacy exported environment value exceeds the Secret budget', + ); + } + const targetName = targetSecretName(name); + exportedEnvironment.push({ + environmentName: name, + targetSecretName: targetName, + }); + secrets.push({ + kind: 'environment', + sourceName: name, + targetName, + value: entry.value, + }); + } + const disabledAssetEntries = Math.max(0, source.entries - 1); + const model = Object.freeze({ + schema: 'qinglong/legacy-config-transformation@v1' as const, + exportedEnvironment: Object.freeze(exportedEnvironment), + retiredSettings: Object.freeze(retiredSettings), + omittedEmptyExports, + duplicateAssignments: duplicates.size, + unsupportedLines, + unsupportedLineDigest: unsupportedHash.digest('hex'), + disabledAssetEntries, + activation: 'disabled' as const, + }); + return Object.freeze({ + source, + model, + secrets: Object.freeze(secrets), + assessment: + duplicates.size > 0 || unsupportedLines > 0 || disabledAssetEntries > 0 + ? 'manual_required' + : 'ready', + }); +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/files.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/files.ts new file mode 100644 index 00000000..bf7f96ee --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/files.ts @@ -0,0 +1,316 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { TextDecoder } from 'node:util'; + +import { LocalDataDirectoryAdoptionConfigurationError } from '../contract'; +import { + assertPrivateDirectory, + rootAuthority, + sameStat, + sortedNames, + stableFileDigest, + syncDirectory, + type RootAuthority, +} from '../filesystem'; +import type { + TransformLocalDataDirectoryAdoptionCommand, + VerifyLocalDataDirectoryAdoptionTransformationCommand, +} from '../contract'; + +const MAX_RELATIVE_PATH_BYTES = 4_096; + +export interface TransformationAuthority extends RootAuthority { + readonly transformationRoot: string; +} + +export interface PrivateTreeEvidence { + readonly entries: number; + readonly directories: number; + readonly files: number; + readonly bytes: number; + readonly digest: string; +} + +function inside(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return ( + relative !== '' && + relative !== '..' && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) + ); +} + +function assertMissing(candidate: string): void { + try { + fs.lstatSync(candidate); + } catch (error) { + if ( + error && + typeof error === 'object' && + 'code' in error && + error.code === 'ENOENT' + ) { + return; + } + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformationRoot cannot be inspected', + error, + ); + } + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformationRoot must not already exist', + ); +} + +export function transformationAuthority( + options: + | TransformLocalDataDirectoryAdoptionCommand['options'] + | VerifyLocalDataDirectoryAdoptionTransformationCommand['options'], + requireMissing: boolean, +): Readonly { + const stage = rootAuthority( + { + deploymentRoot: options.deploymentRoot, + dataRoot: options.dataRoot, + stagingRoot: options.stagingRoot, + profile: options.profile, + sqlite: options.sqlite, + expectedManifestDigest: options.expectedManifestDigest, + }, + false, + ); + const target = options.transformationRoot; + if ( + !inside(stage.deploymentRoot, target) || + target === stage.dataRoot || + target === stage.stagingRoot || + inside(stage.dataRoot, target) || + inside(target, stage.dataRoot) || + inside(stage.stagingRoot, target) || + inside(target, stage.stagingRoot) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformationRoot must be isolated inside deploymentRoot', + ); + } + assertPrivateDirectory( + path.dirname(target), + stage.uid, + 'transformationRoot parent', + ); + if (requireMissing) assertMissing(target); + else assertPrivateDirectory(target, stage.uid, 'transformationRoot'); + return Object.freeze({ ...stage, transformationRoot: target }); +} + +function assertRelativePath(value: string): void { + if ( + value.length < 1 || + path.isAbsolute(value) || + value === '..' || + value.startsWith(`..${path.sep}`) || + Buffer.byteLength(value, 'utf8') > MAX_RELATIVE_PATH_BYTES + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation relative path is invalid', + ); + } +} + +function privateFileStat( + filePath: string, + uid: number, + label: string, +): fs.BigIntStats { + const stat = fs.lstatSync(filePath, { bigint: true }); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1n || + stat.uid !== BigInt(uid) || + (stat.mode & 0o777n) !== 0o600n + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + `${label} identity or mode is invalid`, + ); + } + return stat; +} + +export function optionalPrivateDirectory( + directoryPath: string, + uid: number, + label: string, +): fs.BigIntStats | null { + try { + return assertPrivateDirectory(directoryPath, uid, label); + } catch (error) { + if ( + error instanceof LocalDataDirectoryAdoptionConfigurationError && + error.cause && + typeof error.cause === 'object' && + 'code' in error.cause && + error.cause.code === 'ENOENT' + ) { + return null; + } + throw error; + } +} + +export function readStablePrivateUtf8File( + filePath: string, + uid: number, + maximumBytes: number, + label: string, +): string { + const expected = privateFileStat(filePath, uid, label); + if ( + expected.size < 0n || + expected.size > BigInt(maximumBytes) || + expected.size > BigInt(Number.MAX_SAFE_INTEGER) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + `${label} exceeds its byte budget`, + ); + } + const descriptor = fs.openSync( + filePath, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + let bytes: Buffer | undefined; + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if (!sameStat(expected, before)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + `${label} identity changed before reading`, + ); + } + bytes = fs.readFileSync(descriptor); + if (!sameStat(before, fs.fstatSync(descriptor, { bigint: true }))) { + throw new LocalDataDirectoryAdoptionConfigurationError( + `${label} changed while reading`, + ); + } + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch (error) { + if (error instanceof LocalDataDirectoryAdoptionConfigurationError) { + throw error; + } + throw new LocalDataDirectoryAdoptionConfigurationError( + `${label} is not valid UTF-8`, + error, + ); + } finally { + bytes?.fill(0); + fs.closeSync(descriptor); + } +} + +export function writePrivateJson(filePath: string, value: object): void { + const descriptor = fs.openSync(filePath, 'wx', 0o600); + const bytes = Buffer.from(`${JSON.stringify(value)}\n`, 'utf8'); + try { + let offset = 0; + while (offset < bytes.length) { + offset += fs.writeSync( + descriptor, + bytes, + offset, + bytes.length - offset, + null, + ); + } + fs.fsyncSync(descriptor); + } finally { + bytes.fill(0); + fs.closeSync(descriptor); + } +} + +export function summarizePrivateTree( + root: string, + uid: number, +): Readonly { + const rootStat = assertPrivateDirectory( + root, + uid, + 'transformation input category', + ); + let entries = 0; + let directories = 0; + let files = 0; + let bytes = 0; + const hash = crypto.createHash('sha256'); + const visit = (directoryPath: string, expected: fs.BigIntStats): void => { + for (const name of sortedNames(directoryPath)) { + const entryPath = path.join(directoryPath, name); + const relative = path.relative(root, entryPath); + assertRelativePath(relative); + const stat = fs.lstatSync(entryPath, { bigint: true }); + entries += 1; + if ( + stat.isSymbolicLink() || + stat.uid !== BigInt(uid) || + (stat.mode & 0o777n) !== (stat.isDirectory() ? 0o700n : 0o600n) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation input identity or mode is invalid', + ); + } + const canonical = relative.split(path.sep).join('/'); + if (stat.isDirectory()) { + directories += 1; + hash.update( + `${JSON.stringify({ relative: canonical, kind: 'directory' })}\n`, + ); + visit(entryPath, stat); + } else if (stat.isFile() && stat.nlink === 1n) { + if (stat.size < 0n || stat.size > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation input file size is unsupported', + ); + } + const count = Number(stat.size); + if (!Number.isSafeInteger(bytes + count)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation input byte total is unsupported', + ); + } + bytes += count; + files += 1; + hash.update( + `${JSON.stringify({ + relative: canonical, + kind: 'file', + bytes: count, + contentDigest: stableFileDigest(entryPath, stat), + })}\n`, + ); + } else { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation input entry kind is invalid', + ); + } + } + if (!sameStat(expected, fs.lstatSync(directoryPath, { bigint: true }))) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation input directory changed while reading', + ); + } + }; + visit(root, rootStat); + return Object.freeze({ + entries, + directories, + files, + bytes, + digest: hash.digest('hex'), + }); +} + +export function finishPrivateDirectory(directoryPath: string): void { + syncDirectory(directoryPath); +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/keyv.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/keyv.ts new file mode 100644 index 00000000..ccff2041 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/keyv.ts @@ -0,0 +1,351 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +import { LocalDataDirectoryAdoptionConfigurationError } from '../contract'; +import { sameStat } from '../filesystem'; +import { sha256Text } from '../manifest'; +import { + optionalPrivateDirectory, + summarizePrivateTree, + type PrivateTreeEvidence, +} from './files'; + +const KNOWN_KEYS = Object.freeze([ + Object.freeze({ + key: 'keyv:authInfo', + target: 'credential_reissue' as const, + }), + Object.freeze({ + key: 'keyv:apps', + target: 'main_database_apps_reconciliation' as const, + }), + Object.freeze({ + key: 'keyv:lang', + target: 'main_database_system_settings_reconciliation' as const, + }), +]); + +type KeyvTarget = (typeof KNOWN_KEYS)[number]['target']; + +export interface KeyvTransformationModel { + readonly schema: 'qinglong/legacy-keyv-transformation@v1'; + readonly databasePresent: boolean; + readonly integrity: 'absent' | 'ok'; + readonly mappings: readonly Readonly<{ + legacyKey: string; + target: KeyvTarget; + state: 'absent' | 'retired' | 'reconcile'; + valueDigest: string | null; + }>[]; + readonly cachedLocale: 'zh' | 'en' | null; + readonly unknownEntries: number; + readonly unknownEntryDigest: string; + readonly unknownSchemaObjects: number; + readonly unknownSchemaDigest: string; + readonly disabledAssetEntries: number; + readonly activation: 'disabled'; +} + +export interface KeyvTransformation { + readonly source: Readonly | null; + readonly model: Readonly; + readonly assessment: 'ready' | 'manual_required'; +} + +function emptyModel(): Readonly { + return Object.freeze({ + schema: 'qinglong/legacy-keyv-transformation@v1', + databasePresent: false, + integrity: 'absent', + mappings: Object.freeze( + KNOWN_KEYS.map((entry) => + Object.freeze({ + legacyKey: entry.key, + target: entry.target, + state: 'absent' as const, + valueDigest: null, + }), + ), + ), + cachedLocale: null, + unknownEntries: 0, + unknownEntryDigest: sha256Text(''), + unknownSchemaObjects: 0, + unknownSchemaDigest: sha256Text(''), + disabledAssetEntries: 0, + activation: 'disabled', + }); +} + +function exactKeys(value: object, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const canonical = [...expected].sort(); + return ( + actual.length === canonical.length && + actual.every((key, index) => key === canonical[index]) + ); +} + +function cachedLocale(value: string): 'zh' | 'en' | null { + try { + const envelope = JSON.parse(value) as unknown; + if ( + !envelope || + typeof envelope !== 'object' || + Array.isArray(envelope) || + !exactKeys(envelope, ['expires', 'value']) + ) { + return null; + } + const candidate = envelope as { + readonly expires?: unknown; + value?: unknown; + }; + return candidate.expires === null && + (candidate.value === 'zh' || candidate.value === 'en') + ? candidate.value + : null; + } catch { + return null; + } +} + +export function transformLegacyKeyv( + categoryRoot: string, + uid: number, + profile: 'edge' | 'standalone', +): Readonly { + if ( + !optionalPrivateDirectory(categoryRoot, uid, 'Keyv transformation input') + ) { + return Object.freeze({ + source: null, + model: emptyModel(), + assessment: 'ready', + }); + } + const source = summarizePrivateTree(categoryRoot, uid); + const databasePath = path.join(categoryRoot, 'keyv.sqlite'); + let expected: fs.BigIntStats; + try { + expected = fs.lstatSync(databasePath, { bigint: true }); + } catch (error) { + if ( + error && + typeof error === 'object' && + 'code' in error && + error.code === 'ENOENT' + ) { + return Object.freeze({ + source, + model: Object.freeze({ + ...emptyModel(), + disabledAssetEntries: source.entries, + }), + assessment: source.entries === 0 ? 'ready' : 'manual_required', + }); + } + throw error; + } + if ( + !expected.isFile() || + expected.isSymbolicLink() || + expected.nlink !== 1n || + expected.uid !== BigInt(uid) || + (expected.mode & 0o777n) !== 0o600n + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged Keyv database identity or mode is invalid', + ); + } + + const client = new DatabaseSync(databasePath, { + allowExtension: false, + defensive: true, + enableDoubleQuotedStringLiterals: false, + enableForeignKeyConstraints: true, + readOnly: true, + timeout: 5_000, + }); + const values = new Map(); + let unknownEntries = 0; + const unknownEntryHash = crypto.createHash('sha256'); + let unknownSchemaObjects = 0; + const unknownSchemaHash = crypto.createHash('sha256'); + try { + client.enableDefensive(true); + client.exec( + `PRAGMA trusted_schema = OFF; PRAGMA query_only = ON; PRAGMA mmap_size = 0; PRAGMA cache_size = ${ + profile === 'edge' ? -2048 : -8192 + }`, + ); + const integrity = client.prepare('PRAGMA integrity_check(1)').get() as + | { readonly integrity_check?: unknown } + | undefined; + if (integrity?.integrity_check !== 'ok') { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged Keyv database integrity check failed', + ); + } + const tables = client.prepare(`PRAGMA table_list('keyv')`).all() as Array<{ + readonly schema?: unknown; + readonly name?: unknown; + readonly type?: unknown; + readonly ncol?: unknown; + readonly wr?: unknown; + readonly strict?: unknown; + }>; + if ( + tables.length !== 1 || + tables[0]?.schema !== 'main' || + tables[0]?.name !== 'keyv' || + tables[0]?.type !== 'table' || + tables[0]?.ncol !== 2 || + tables[0]?.wr !== 0 || + tables[0]?.strict !== 0 + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged Keyv database is not a reviewed ordinary table', + ); + } + const columns = client.prepare(`PRAGMA table_info('keyv')`).all() as Array<{ + readonly name?: unknown; + readonly type?: unknown; + readonly pk?: unknown; + }>; + if ( + columns.length !== 2 || + columns[0]?.name !== 'key' || + columns[0]?.type !== 'VARCHAR(255)' || + columns[0]?.pk !== 1 || + columns[1]?.name !== 'value' || + columns[1]?.type !== 'TEXT' || + columns[1]?.pk !== 0 + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged Keyv database schema is not the reviewed v4 shape', + ); + } + const schema = client + .prepare( + `SELECT type, name, tbl_name AS tableName, sql + FROM sqlite_schema + WHERE name NOT LIKE 'sqlite_%' + ORDER BY CAST(name AS BLOB)`, + ) + .all() as Array<{ + readonly type?: unknown; + readonly name?: unknown; + readonly tableName?: unknown; + readonly sql?: unknown; + }>; + for (const object of schema) { + if ( + object.type === 'table' && + object.name === 'keyv' && + object.tableName === 'keyv' + ) { + continue; + } + unknownSchemaObjects += 1; + unknownSchemaHash.update( + `${sha256Text(JSON.stringify(object))}\n`, + 'utf8', + ); + } + const rowBudget = profile === 'edge' ? 256 : 2_048; + const byteBudget = profile === 'edge' ? 4 * 1024 * 1024 : 16 * 1024 * 1024; + let rows = 0; + let bytes = 0; + const statement = client.prepare( + `SELECT key, value, length(CAST(value AS BLOB)) AS valueBytes + FROM keyv + ORDER BY CAST(key AS BLOB)`, + ); + for (const row of statement.iterate() as Iterable<{ + readonly key?: unknown; + readonly value?: unknown; + readonly valueBytes?: unknown; + }>) { + rows += 1; + if ( + rows > rowBudget || + typeof row.key !== 'string' || + Buffer.byteLength(row.key, 'utf8') > 255 || + typeof row.value !== 'string' || + !Number.isSafeInteger(row.valueBytes) || + (row.valueBytes as number) < 0 || + (row.valueBytes as number) > 1024 * 1024 || + !Number.isSafeInteger(bytes + (row.valueBytes as number)) || + bytes + (row.valueBytes as number) > byteBudget + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged Keyv data exceeds the Profile budget', + ); + } + bytes += row.valueBytes as number; + if (KNOWN_KEYS.some((entry) => entry.key === row.key)) { + values.set(row.key, row.value); + } else { + unknownEntries += 1; + unknownEntryHash.update( + `${sha256Text(row.key)}:${sha256Text(row.value)}\n`, + 'utf8', + ); + } + } + } finally { + client.close(); + } + if (!sameStat(expected, fs.lstatSync(databasePath, { bigint: true }))) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged Keyv database changed while transforming', + ); + } + + const localeValue = values.get('keyv:lang'); + const locale = localeValue === undefined ? null : cachedLocale(localeValue); + const mappings = Object.freeze( + KNOWN_KEYS.map((entry) => { + const value = values.get(entry.key); + return Object.freeze({ + legacyKey: entry.key, + target: entry.target, + state: + value === undefined + ? ('absent' as const) + : entry.key === 'keyv:authInfo' + ? ('retired' as const) + : ('reconcile' as const), + valueDigest: value === undefined ? null : sha256Text(value), + }); + }), + ); + const disabledAssetEntries = Math.max(0, source.entries - 1); + const model = Object.freeze({ + schema: 'qinglong/legacy-keyv-transformation@v1' as const, + databasePresent: true, + integrity: 'ok' as const, + mappings, + cachedLocale: locale, + unknownEntries, + unknownEntryDigest: unknownEntryHash.digest('hex'), + unknownSchemaObjects, + unknownSchemaDigest: unknownSchemaHash.digest('hex'), + disabledAssetEntries, + activation: 'disabled' as const, + }); + return Object.freeze({ + source, + model, + assessment: + unknownEntries > 0 || + unknownSchemaObjects > 0 || + disabledAssetEntries > 0 || + (localeValue !== undefined && locale === null) + ? 'manual_required' + : 'ready', + }); +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/manifest.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/manifest.ts new file mode 100644 index 00000000..6d7aac04 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/manifest.ts @@ -0,0 +1,260 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { LocalDataDirectoryAdoptionConfigurationError } from '../contract'; +import { assertPrivateDirectory, sameStat, sortedNames } from '../filesystem'; +import { sha256Text } from '../manifest'; +import { + readStablePrivateUtf8File, + type TransformationAuthority, +} from './files'; +import { + verifyTransformationModel, + type LocalDataDirectoryTransformationManifest, + type TransformationModelEvidence, + type TransformationSourceEvidence, +} from './model'; + +export const TRANSFORMATION_MANIFEST_NAME = 'manifest.json'; +const MAX_MANIFEST_BYTES = 64 * 1024; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const SOURCE_NAMES = Object.freeze(['config', 'keyv', 'ssh'] as const); + +function exactKeys(value: object, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const canonical = [...expected].sort(); + return ( + actual.length === canonical.length && + actual.every((key, index) => key === canonical[index]) + ); +} + +function safeCount(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + +function treeEvidence( + value: unknown, + extraKeys: readonly string[], +): value is Record { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + !exactKeys(value, [ + 'bytes', + 'digest', + 'directories', + 'entries', + 'files', + ...extraKeys, + ]) + ) { + return false; + } + const candidate = value as Record; + return ( + [ + candidate.entries, + candidate.directories, + candidate.files, + candidate.bytes, + ...extraKeys.map((key) => candidate[key]), + ].every(safeCount) && + candidate.entries === + (candidate.directories as number) + (candidate.files as number) && + typeof candidate.digest === 'string' && + DIGEST_PATTERN.test(candidate.digest) + ); +} + +function sourceEvidence( + value: unknown, + expectedName: (typeof SOURCE_NAMES)[number], +): value is TransformationSourceEvidence { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + !exactKeys(value, [ + 'assessment', + 'bytes', + 'digest', + 'directories', + 'entries', + 'files', + 'name', + 'present', + ]) + ) { + return false; + } + const candidate = value as Record; + return ( + treeEvidence( + { + bytes: candidate.bytes, + digest: candidate.digest, + directories: candidate.directories, + entries: candidate.entries, + files: candidate.files, + }, + [], + ) && + candidate.name === expectedName && + typeof candidate.present === 'boolean' && + (candidate.assessment === 'ready' || + candidate.assessment === 'manual_required') && + (candidate.present || candidate.entries === 0) + ); +} + +function parseManifest( + value: unknown, +): LocalDataDirectoryTransformationManifest { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + !exactKeys(value, [ + 'assessment', + 'createdAtMs', + 'kind', + 'model', + 'profile', + 'projectIdDigest', + 'schemaVersion', + 'sourceStageManifestDigest', + 'sources', + 'state', + 'transformationDigest', + 'transformationRootPathDigest', + ]) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation manifest shape is invalid', + ); + } + const candidate = value as Record; + if ( + candidate.schemaVersion !== 1 || + candidate.kind !== 'qinglong3-legacy-data-directory-transformation' || + candidate.state !== 'prepared' || + (candidate.profile !== 'edge' && candidate.profile !== 'standalone') || + (candidate.assessment !== 'ready' && + candidate.assessment !== 'manual_required') || + !safeCount(candidate.createdAtMs) || + ![ + candidate.projectIdDigest, + candidate.sourceStageManifestDigest, + candidate.transformationDigest, + candidate.transformationRootPathDigest, + ].every( + (digest) => typeof digest === 'string' && DIGEST_PATTERN.test(digest), + ) || + !Array.isArray(candidate.sources) || + candidate.sources.length !== SOURCE_NAMES.length || + !candidate.sources.every((entry, index) => + sourceEvidence(entry, SOURCE_NAMES[index]!), + ) || + !treeEvidence(candidate.model, [ + 'environmentSecrets', + 'manualCategories', + 'sshSecrets', + ]) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation manifest value is invalid', + ); + } + const { transformationDigest, ...payload } = + candidate as unknown as LocalDataDirectoryTransformationManifest; + if (sha256Text(JSON.stringify(payload)) !== transformationDigest) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation manifest digest does not match', + ); + } + return candidate as unknown as LocalDataDirectoryTransformationManifest; +} + +function readManifest( + root: string, + uid: number, +): Readonly { + try { + return parseManifest( + JSON.parse( + readStablePrivateUtf8File( + path.join(root, TRANSFORMATION_MANIFEST_NAME), + uid, + MAX_MANIFEST_BYTES, + 'transformation manifest', + ), + ), + ); + } catch (error) { + if (error instanceof LocalDataDirectoryAdoptionConfigurationError) { + throw error; + } + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation manifest JSON is invalid', + error, + ); + } +} + +export function verifyStaticTransformation(options: { + readonly authority: Readonly; + readonly profile: 'edge' | 'standalone'; + readonly projectId: string; + readonly sourceStageManifestDigest: string; + readonly expectedTransformationDigest: string; +}): Readonly { + const before = assertPrivateDirectory( + options.authority.transformationRoot, + options.authority.uid, + 'transformationRoot', + ); + if ( + JSON.stringify(sortedNames(options.authority.transformationRoot)) !== + JSON.stringify([TRANSFORMATION_MANIFEST_NAME, 'model'].sort()) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation root is incomplete or contains unexpected entries', + ); + } + const manifest = readManifest( + options.authority.transformationRoot, + options.authority.uid, + ); + if ( + manifest.transformationDigest !== options.expectedTransformationDigest || + manifest.profile !== options.profile || + manifest.projectIdDigest !== sha256Text(options.projectId) || + manifest.sourceStageManifestDigest !== options.sourceStageManifestDigest || + manifest.transformationRootPathDigest !== + sha256Text(options.authority.transformationRoot) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation manifest authority binding is invalid', + ); + } + verifyTransformationModel({ + modelRoot: path.join(options.authority.transformationRoot, 'model'), + uid: options.authority.uid, + projectId: options.projectId, + profile: options.profile, + expected: manifest.model as Readonly, + }); + if ( + !sameStat( + before, + fs.lstatSync(options.authority.transformationRoot, { bigint: true }), + ) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation root changed during verification', + ); + } + return manifest; +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/model.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/model.ts new file mode 100644 index 00000000..23d7b9c5 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/model.ts @@ -0,0 +1,430 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { LocalDataDirectoryAdoptionConfigurationError } from '../contract'; +import { sortedNames, syncDirectory } from '../filesystem'; +import { sha256Text } from '../manifest'; +import type { ConfigTransformation, SecretImportDraft } from './config'; +import { + readStablePrivateUtf8File, + summarizePrivateTree, + writePrivateJson, + type PrivateTreeEvidence, +} from './files'; +import type { KeyvTransformation } from './keyv'; +import type { SshTransformation } from './ssh'; + +const MAX_MODEL_FILE_BYTES = 1024 * 1024; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const SECRET_FILE_PATTERN = /^secret-values\/[0-9a-f]{64}\.json$/; + +export interface TransformationSourceEvidence extends PrivateTreeEvidence { + readonly name: 'config' | 'keyv' | 'ssh'; + readonly present: boolean; + readonly assessment: 'ready' | 'manual_required'; +} + +export interface TransformationModelEvidence extends PrivateTreeEvidence { + readonly environmentSecrets: number; + readonly sshSecrets: number; + readonly manualCategories: number; +} + +export interface LocalDataDirectoryTransformationManifestPayload { + readonly schemaVersion: 1; + readonly kind: 'qinglong3-legacy-data-directory-transformation'; + readonly state: 'prepared'; + readonly profile: 'edge' | 'standalone'; + readonly createdAtMs: number; + readonly projectIdDigest: string; + readonly sourceStageManifestDigest: string; + readonly transformationRootPathDigest: string; + readonly assessment: 'ready' | 'manual_required'; + readonly sources: readonly TransformationSourceEvidence[]; + readonly model: Readonly; +} + +export interface LocalDataDirectoryTransformationManifest + extends LocalDataDirectoryTransformationManifestPayload { + readonly transformationDigest: string; +} + +interface SecretImportEntry { + readonly kind: SecretImportDraft['kind']; + readonly sourceName: string; + readonly targetName: string; + readonly expectedCurrentVersion: 0; + readonly valueFile: string; + readonly valueDigest: string; +} + +function exactKeys(value: object, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const canonical = [...expected].sort(); + return ( + actual.length === canonical.length && + actual.every((key, index) => key === canonical[index]) + ); +} + +function emptyEvidence(): Readonly { + return Object.freeze({ + entries: 0, + directories: 0, + files: 0, + bytes: 0, + digest: sha256Text(''), + }); +} + +function sourceEvidence( + name: TransformationSourceEvidence['name'], + transformation: + | Readonly + | Readonly + | Readonly, +): Readonly { + return Object.freeze({ + name, + present: transformation.source !== null, + ...(transformation.source ?? emptyEvidence()), + assessment: transformation.assessment, + }); +} + +function secretId(entry: Readonly): string { + return sha256Text(`${entry.kind}\0${entry.sourceName}\0${entry.targetName}`); +} + +export function writeTransformationModel(options: { + readonly modelRoot: string; + readonly uid: number; + readonly projectId: string; + readonly profile: 'edge' | 'standalone'; + readonly config: Readonly; + readonly keyv: Readonly; + readonly ssh: Readonly; +}): Readonly<{ + sources: readonly TransformationSourceEvidence[]; + model: Readonly; + assessment: 'ready' | 'manual_required'; +}> { + fs.mkdirSync(options.modelRoot, { mode: 0o700 }); + const secretRoot = path.join(options.modelRoot, 'secret-values'); + fs.mkdirSync(secretRoot, { mode: 0o700 }); + const drafts = [...options.config.secrets, ...options.ssh.secrets].sort( + (left, right) => + Buffer.compare( + Buffer.from(`${left.kind}\0${left.sourceName}`, 'utf8'), + Buffer.from(`${right.kind}\0${right.sourceName}`, 'utf8'), + ), + ); + const maximumSecrets = options.profile === 'edge' ? 128 : 512; + if (drafts.length > maximumSecrets) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation Secret count exceeds the Profile budget', + ); + } + const targets = new Set(); + const files = new Set(); + const imports: SecretImportEntry[] = []; + for (const draft of drafts) { + const id = secretId(draft); + const relative = `secret-values/${id}.json`; + if (targets.has(draft.targetName) || files.has(relative)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation Secret identity collides', + ); + } + targets.add(draft.targetName); + files.add(relative); + writePrivateJson(path.join(options.modelRoot, relative), { + schemaVersion: 1, + kind: 'qinglong3-local-secret-value', + value: draft.value, + }); + imports.push({ + kind: draft.kind, + sourceName: draft.sourceName, + targetName: draft.targetName, + expectedCurrentVersion: 0, + valueFile: relative, + valueDigest: sha256Text(draft.value), + }); + } + syncDirectory(secretRoot); + writePrivateJson( + path.join(options.modelRoot, 'config.json'), + options.config.model, + ); + writePrivateJson( + path.join(options.modelRoot, 'keyv.json'), + options.keyv.model, + ); + writePrivateJson(path.join(options.modelRoot, 'ssh.json'), options.ssh.model); + writePrivateJson(path.join(options.modelRoot, 'secret-imports.json'), { + schema: 'qinglong/local-secret-import-plan@v1', + projectId: options.projectId, + state: 'prepared', + imports, + }); + const sources = Object.freeze([ + sourceEvidence('config', options.config), + sourceEvidence('keyv', options.keyv), + sourceEvidence('ssh', options.ssh), + ]); + const manualCategories = sources.filter( + ({ assessment }) => assessment === 'manual_required', + ).length; + writePrivateJson(path.join(options.modelRoot, 'manual-review.json'), { + schema: 'qinglong/legacy-data-directory-manual-review@v1', + required: manualCategories > 0, + categories: sources.map(({ name, present, assessment, ...evidence }) => ({ + name, + present, + assessment, + evidence, + })), + activation: 'disabled', + }); + syncDirectory(options.modelRoot); + const tree = summarizePrivateTree(options.modelRoot, options.uid); + const environmentSecrets = imports.filter( + ({ kind }) => kind === 'environment', + ).length; + const sshSecrets = imports.length - environmentSecrets; + return Object.freeze({ + sources, + model: Object.freeze({ + ...tree, + environmentSecrets, + sshSecrets, + manualCategories, + }), + assessment: manualCategories > 0 ? 'manual_required' : 'ready', + }); +} + +function readJson(filePath: string, uid: number): unknown { + try { + return JSON.parse( + readStablePrivateUtf8File( + filePath, + uid, + MAX_MODEL_FILE_BYTES, + 'transformation model file', + ), + ); + } catch (error) { + if (error instanceof LocalDataDirectoryAdoptionConfigurationError) { + throw error; + } + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation model JSON is invalid', + error, + ); + } +} + +function assertSchemaFile( + filePath: string, + uid: number, + schema: string, +): Record { + const value = readJson(filePath, uid); + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + (value as { readonly schema?: unknown }).schema !== schema || + (value as { readonly activation?: unknown }).activation !== 'disabled' + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation target model schema is invalid', + ); + } + return value as Record; +} + +export function verifyTransformationModel(options: { + readonly modelRoot: string; + readonly uid: number; + readonly projectId: string; + readonly profile: 'edge' | 'standalone'; + readonly expected: Readonly; +}): void { + if ( + JSON.stringify(sortedNames(options.modelRoot)) !== + JSON.stringify( + [ + 'config.json', + 'keyv.json', + 'manual-review.json', + 'secret-imports.json', + 'secret-values', + 'ssh.json', + ].sort(), + ) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation model root contains unexpected entries', + ); + } + assertSchemaFile( + path.join(options.modelRoot, 'config.json'), + options.uid, + 'qinglong/legacy-config-transformation@v1', + ); + assertSchemaFile( + path.join(options.modelRoot, 'keyv.json'), + options.uid, + 'qinglong/legacy-keyv-transformation@v1', + ); + assertSchemaFile( + path.join(options.modelRoot, 'ssh.json'), + options.uid, + 'qinglong/legacy-ssh-transformation@v1', + ); + const manual = assertSchemaFile( + path.join(options.modelRoot, 'manual-review.json'), + options.uid, + 'qinglong/legacy-data-directory-manual-review@v1', + ); + if ( + !exactKeys(manual, ['activation', 'categories', 'required', 'schema']) || + typeof manual.required !== 'boolean' || + !Array.isArray(manual.categories) || + manual.categories.length !== 3 + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'manual-review model is invalid', + ); + } + const plan = readJson( + path.join(options.modelRoot, 'secret-imports.json'), + options.uid, + ); + if ( + !plan || + typeof plan !== 'object' || + Array.isArray(plan) || + !exactKeys(plan, ['imports', 'projectId', 'schema', 'state']) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'Secret import plan shape is invalid', + ); + } + const candidate = plan as Record; + const maximumSecrets = options.profile === 'edge' ? 128 : 512; + if ( + candidate.schema !== 'qinglong/local-secret-import-plan@v1' || + candidate.projectId !== options.projectId || + candidate.state !== 'prepared' || + !Array.isArray(candidate.imports) || + candidate.imports.length > maximumSecrets + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'Secret import plan value is invalid', + ); + } + const expectedFiles: string[] = []; + const targets = new Set(); + let environmentSecrets = 0; + let sshSecrets = 0; + for (const value of candidate.imports) { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + !exactKeys(value, [ + 'expectedCurrentVersion', + 'kind', + 'sourceName', + 'targetName', + 'valueDigest', + 'valueFile', + ]) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'Secret import entry shape is invalid', + ); + } + const entry = value as Record; + if ( + (entry.kind !== 'environment' && entry.kind !== 'ssh_private_key') || + typeof entry.sourceName !== 'string' || + typeof entry.targetName !== 'string' || + entry.expectedCurrentVersion !== 0 || + typeof entry.valueFile !== 'string' || + !SECRET_FILE_PATTERN.test(entry.valueFile) || + typeof entry.valueDigest !== 'string' || + !DIGEST_PATTERN.test(entry.valueDigest) || + targets.has(entry.targetName) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'Secret import entry value is invalid', + ); + } + targets.add(entry.targetName); + expectedFiles.push(path.basename(entry.valueFile)); + const secret = readJson( + path.join(options.modelRoot, entry.valueFile), + options.uid, + ); + if ( + !secret || + typeof secret !== 'object' || + Array.isArray(secret) || + !exactKeys(secret, ['kind', 'schemaVersion', 'value']) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'Secret value file shape is invalid', + ); + } + const secretValue = secret as Record; + if ( + secretValue.schemaVersion !== 1 || + secretValue.kind !== 'qinglong3-local-secret-value' || + typeof secretValue.value !== 'string' || + secretValue.value.includes('\0') || + Buffer.byteLength(secretValue.value, 'utf8') > 16 * 1024 || + sha256Text(secretValue.value) !== entry.valueDigest + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'Secret value file is invalid', + ); + } + if (entry.kind === 'environment') environmentSecrets += 1; + else sshSecrets += 1; + } + expectedFiles.sort(); + if ( + JSON.stringify( + sortedNames(path.join(options.modelRoot, 'secret-values')), + ) !== JSON.stringify(expectedFiles) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'Secret value file set is invalid', + ); + } + const actual = summarizePrivateTree(options.modelRoot, options.uid); + if ( + JSON.stringify({ + ...actual, + environmentSecrets, + sshSecrets, + manualCategories: (manual.categories as unknown[]).filter( + (entry) => + !!entry && + typeof entry === 'object' && + !Array.isArray(entry) && + (entry as { readonly assessment?: unknown }).assessment === + 'manual_required', + ).length, + }) !== JSON.stringify(options.expected) + ) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'transformation model no longer matches the manifest', + ); + } +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/ssh.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/ssh.ts new file mode 100644 index 00000000..8eae29a5 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/ssh.ts @@ -0,0 +1,224 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { LocalDataDirectoryAdoptionConfigurationError } from '../contract'; +import { sortedNames } from '../filesystem'; +import { sha256Text } from '../manifest'; +import { + optionalPrivateDirectory, + readStablePrivateUtf8File, + summarizePrivateTree, + type PrivateTreeEvidence, +} from './files'; +import type { SecretImportDraft } from './config'; + +const MAX_KEY_BYTES = 16 * 1024; +const MAX_CONFIG_BYTES = 64 * 1024; +const MAX_ALIAS_BYTES = 128; +const KEY_KINDS = new Set([ + 'OPENSSH PRIVATE KEY', + 'RSA PRIVATE KEY', + 'EC PRIVATE KEY', + 'PRIVATE KEY', +]); + +export interface SshTransformationModel { + readonly schema: 'qinglong/legacy-ssh-transformation@v1'; + readonly bindings: readonly Readonly<{ + alias: string; + legacyHostPattern: string; + targetSecretName: string; + legacyConfigDigest: string; + legacyProxyCommandPresent: boolean; + legacyHostKeyBypassPresent: boolean; + hostKeyPolicy: 'operator_verification_required'; + activation: 'disabled'; + }>[]; + readonly manualEntries: number; + readonly manualEntryDigest: string; + readonly activation: 'disabled'; +} + +export interface SshTransformation { + readonly source: Readonly | null; + readonly model: Readonly; + readonly secrets: readonly Readonly[]; + readonly assessment: 'ready' | 'manual_required'; +} + +function targetSecretName(alias: string): string { + return `legacy-ssh-${sha256Text(alias).slice(0, 32)}`; +} + +function privateKey(value: string): boolean { + if (value.includes('\0')) return false; + const lines = value.trimEnd().split('\n'); + if (lines.length < 3) return false; + const begin = /^-----BEGIN ([A-Z0-9 ]+)-----$/.exec(lines[0]!); + const end = /^-----END ([A-Z0-9 ]+)-----$/.exec(lines.at(-1)!); + return !!begin && !!end && begin[1] === end[1] && KEY_KINDS.has(begin[1]!); +} + +function configBinding( + value: string, + alias: string, +): Readonly<{ + hostPattern: string; + proxyCommandPresent: boolean; + hostKeyBypassPresent: boolean; +}> | null { + if ( + value.includes('\0') || + /[\u0001-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(value) + ) { + return null; + } + const lines = value.split(/\r?\n/); + const hosts = lines + .map((line) => /^Host[ \t]+([^\s]+)[ \t]*$/.exec(line)?.[1]) + .filter((entry): entry is string => entry !== undefined); + const identities = lines + .map((line) => /^[ \t]+IdentityFile[ \t]+(.+?)[ \t]*$/.exec(line)?.[1]) + .filter((entry): entry is string => entry !== undefined); + if ( + hosts.length !== 1 || + identities.length !== 1 || + Buffer.byteLength(hosts[0]!, 'utf8') > 255 || + path.basename(identities[0]!) !== alias + ) { + return null; + } + return Object.freeze({ + hostPattern: hosts[0]!, + proxyCommandPresent: lines.some((line) => + /^[ \t]+ProxyCommand[ \t]+/.test(line), + ), + hostKeyBypassPresent: lines.some((line) => + /^[ \t]+StrictHostKeyChecking[ \t]+no[ \t]*$/i.test(line), + ), + }); +} + +function emptyModel(): Readonly { + return Object.freeze({ + schema: 'qinglong/legacy-ssh-transformation@v1', + bindings: Object.freeze([]), + manualEntries: 0, + manualEntryDigest: sha256Text(''), + activation: 'disabled', + }); +} + +export function transformLegacySsh( + categoryRoot: string, + uid: number, +): Readonly { + if ( + !optionalPrivateDirectory(categoryRoot, uid, 'SSH transformation input') + ) { + return Object.freeze({ + source: null, + model: emptyModel(), + secrets: Object.freeze([]), + assessment: 'ready', + }); + } + const source = summarizePrivateTree(categoryRoot, uid); + const names = sortedNames(categoryRoot); + const nameSet = new Set(names); + const consumed = new Set(); + const bindings: Array<{ + alias: string; + legacyHostPattern: string; + targetSecretName: string; + legacyConfigDigest: string; + legacyProxyCommandPresent: boolean; + legacyHostKeyBypassPresent: boolean; + hostKeyPolicy: 'operator_verification_required'; + activation: 'disabled'; + }> = []; + const secrets: SecretImportDraft[] = []; + const manualHash = crypto.createHash('sha256'); + for (const configName of names.filter((name) => name.endsWith('.config'))) { + const alias = configName.slice(0, -'.config'.length); + if ( + alias.length === 0 || + Buffer.byteLength(alias, 'utf8') > MAX_ALIAS_BYTES || + /[\u0000-\u001f\u007f]/.test(alias) || + !nameSet.has(alias) + ) { + continue; + } + const keyPath = path.join(categoryRoot, alias); + const configPath = path.join(categoryRoot, configName); + let key: string; + let config: string; + try { + key = readStablePrivateUtf8File( + keyPath, + uid, + MAX_KEY_BYTES, + 'legacy SSH private key', + ); + config = readStablePrivateUtf8File( + configPath, + uid, + MAX_CONFIG_BYTES, + 'legacy SSH config', + ); + } catch { + continue; + } + const parsed = configBinding(config, alias); + if (!privateKey(key) || !parsed) continue; + const secretName = targetSecretName(alias); + bindings.push({ + alias, + legacyHostPattern: parsed.hostPattern, + targetSecretName: secretName, + legacyConfigDigest: sha256Text(config), + legacyProxyCommandPresent: parsed.proxyCommandPresent, + legacyHostKeyBypassPresent: parsed.hostKeyBypassPresent, + hostKeyPolicy: 'operator_verification_required', + activation: 'disabled', + }); + secrets.push({ + kind: 'ssh_private_key', + sourceName: alias, + targetName: secretName, + value: key, + }); + consumed.add(alias); + consumed.add(configName); + } + for (const name of names) { + if (consumed.has(name)) continue; + const entryPath = path.join(categoryRoot, name); + const stat = fs.lstatSync(entryPath, { bigint: true }); + manualHash.update( + `${sha256Text(name)}:${stat.isDirectory() ? 'directory' : 'file'}\n`, + 'utf8', + ); + } + const after = summarizePrivateTree(categoryRoot, uid); + if (JSON.stringify(after) !== JSON.stringify(source)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'SSH transformation input changed while reading', + ); + } + const manualEntries = Math.max(0, source.entries - consumed.size); + const model = Object.freeze({ + schema: 'qinglong/legacy-ssh-transformation@v1' as const, + bindings: Object.freeze(bindings), + manualEntries, + manualEntryDigest: manualHash.digest('hex'), + activation: 'disabled' as const, + }); + return Object.freeze({ + source, + model, + secrets: Object.freeze(secrets), + assessment: manualEntries > 0 ? 'manual_required' : 'ready', + }); +} diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/transformation.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/transformation.ts new file mode 100644 index 00000000..7d408654 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/transformation/transformation.ts @@ -0,0 +1,248 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { + LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION, + LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION, + LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION, + LocalDataDirectoryAdoptionConfigurationError, + type TransformLocalDataDirectoryAdoptionCommand, + type VerifyLocalDataDirectoryAdoptionCommand, + type VerifyLocalDataDirectoryAdoptionTransformationCommand, +} from '../contract'; +import { syncDirectory } from '../filesystem'; +import { sha256Text } from '../manifest'; +import { verifyLocalDataDirectoryAdoption } from '../staging'; +import { transformLegacyConfig } from './config'; +import { + finishPrivateDirectory, + transformationAuthority, + writePrivateJson, +} from './files'; +import { transformLegacyKeyv } from './keyv'; +import { + TRANSFORMATION_MANIFEST_NAME, + verifyStaticTransformation, +} from './manifest'; +import { + writeTransformationModel, + type LocalDataDirectoryTransformationManifest, + type LocalDataDirectoryTransformationManifestPayload, + type TransformationModelEvidence, + type TransformationSourceEvidence, +} from './model'; +import { transformLegacySsh } from './ssh'; + +const INCOMPLETE_NAME = '.incomplete'; + +export interface LocalDataDirectoryTransformationResult { + readonly schemaVersion: 1; + readonly operation: + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION; + readonly status: 'prepared' | 'verified'; + readonly evidence: Readonly<{ + profile: 'edge' | 'standalone'; + createdAtMs: number; + sourceStageManifestDigest: string; + transformationDigest: string; + assessment: 'ready' | 'manual_required'; + sources: readonly TransformationSourceEvidence[]; + model: Readonly; + }>; +} + +type TransformationCommand = + | Readonly + | Readonly; + +async function verifySource(command: TransformationCommand) { + const sourceCommand: VerifyLocalDataDirectoryAdoptionCommand = { + schemaVersion: 1, + operation: LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION, + options: { + deploymentRoot: command.options.deploymentRoot, + dataRoot: command.options.dataRoot, + stagingRoot: command.options.stagingRoot, + profile: command.options.profile, + sqlite: command.options.sqlite, + expectedManifestDigest: command.options.expectedManifestDigest, + }, + }; + return verifyLocalDataDirectoryAdoption(sourceCommand); +} + +function unchangedSource( + before: Awaited>, + after: Awaited>, +): void { + if (JSON.stringify(before.evidence) !== JSON.stringify(after.evidence)) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'staged source changed during transformation', + ); + } +} + +function createdAtMs(): number { + const value = Date.now(); + if (!Number.isSafeInteger(value) || value < 0) { + throw new LocalDataDirectoryAdoptionConfigurationError( + 'system clock returned an invalid timestamp', + ); + } + return value; +} + +function result( + operation: + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION + | typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION, + status: 'prepared' | 'verified', + manifest: Readonly, +): Readonly { + return Object.freeze({ + schemaVersion: 1, + operation, + status, + evidence: Object.freeze({ + profile: manifest.profile, + createdAtMs: manifest.createdAtMs, + sourceStageManifestDigest: manifest.sourceStageManifestDigest, + transformationDigest: manifest.transformationDigest, + assessment: manifest.assessment, + sources: manifest.sources, + model: manifest.model, + }), + }); +} + +export async function transformLocalDataDirectoryAdoption( + command: Readonly, +): Promise> { + try { + const authority = transformationAuthority(command.options, true); + const before = await verifySource(command); + fs.mkdirSync(authority.transformationRoot, { mode: 0o700 }); + writePrivateJson(path.join(authority.transformationRoot, INCOMPLETE_NAME), { + schemaVersion: 1, + kind: 'qinglong3-legacy-data-directory-transformation-incomplete', + }); + finishPrivateDirectory(authority.transformationRoot); + syncDirectory(path.dirname(authority.transformationRoot)); + + const inputRoot = path.join( + authority.stagingRoot, + 'payload', + 'transform-input', + ); + const config = transformLegacyConfig( + path.join(inputRoot, 'config'), + authority.uid, + ); + const keyv = transformLegacyKeyv( + path.join(inputRoot, 'db'), + authority.uid, + command.options.profile, + ); + const ssh = transformLegacySsh( + path.join(inputRoot, 'ssh.d'), + authority.uid, + ); + const prepared = writeTransformationModel({ + modelRoot: path.join(authority.transformationRoot, 'model'), + uid: authority.uid, + projectId: command.options.projectId, + profile: command.options.profile, + config, + keyv, + ssh, + }); + const after = await verifySource(command); + unchangedSource(before, after); + + const payload: LocalDataDirectoryTransformationManifestPayload = { + schemaVersion: 1, + kind: 'qinglong3-legacy-data-directory-transformation', + state: 'prepared', + profile: command.options.profile, + createdAtMs: createdAtMs(), + projectIdDigest: sha256Text(command.options.projectId), + sourceStageManifestDigest: command.options.expectedManifestDigest, + transformationRootPathDigest: sha256Text(authority.transformationRoot), + assessment: prepared.assessment, + sources: prepared.sources, + model: prepared.model, + }; + const manifest: LocalDataDirectoryTransformationManifest = { + ...payload, + transformationDigest: sha256Text(JSON.stringify(payload)), + }; + writePrivateJson( + path.join(authority.transformationRoot, TRANSFORMATION_MANIFEST_NAME), + manifest, + ); + finishPrivateDirectory(authority.transformationRoot); + fs.unlinkSync(path.join(authority.transformationRoot, INCOMPLETE_NAME)); + finishPrivateDirectory(authority.transformationRoot); + const verified = verifyStaticTransformation({ + authority, + profile: command.options.profile, + projectId: command.options.projectId, + sourceStageManifestDigest: command.options.expectedManifestDigest, + expectedTransformationDigest: manifest.transformationDigest, + }); + return result( + LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION, + 'prepared', + verified, + ); + } catch (error) { + if (error instanceof LocalDataDirectoryAdoptionConfigurationError) { + throw error; + } + throw new LocalDataDirectoryAdoptionConfigurationError( + 'data directory transformation failed', + error, + ); + } +} + +export async function verifyLocalDataDirectoryAdoptionTransformation( + command: Readonly, +): Promise> { + try { + const authority = transformationAuthority(command.options, false); + const before = await verifySource(command); + const manifest = verifyStaticTransformation({ + authority, + profile: command.options.profile, + projectId: command.options.projectId, + sourceStageManifestDigest: command.options.expectedManifestDigest, + expectedTransformationDigest: + command.options.expectedTransformationDigest, + }); + const after = await verifySource(command); + unchangedSource(before, after); + verifyStaticTransformation({ + authority, + profile: command.options.profile, + projectId: command.options.projectId, + sourceStageManifestDigest: command.options.expectedManifestDigest, + expectedTransformationDigest: + command.options.expectedTransformationDigest, + }); + return result( + LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION, + 'verified', + manifest, + ); + } catch (error) { + if (error instanceof LocalDataDirectoryAdoptionConfigurationError) { + throw error; + } + throw new LocalDataDirectoryAdoptionConfigurationError( + 'data directory transformation verification failed', + error, + ); + } +} diff --git a/packages/ql3-local-owner-cli/test/dataDirectoryAdoptionStageCli.test.cjs b/packages/ql3-local-owner-cli/test/dataDirectoryAdoptionStageCli.test.cjs index 914cfeed..b74a3f9a 100644 --- a/packages/ql3-local-owner-cli/test/dataDirectoryAdoptionStageCli.test.cjs +++ b/packages/ql3-local-owner-cli/test/dataDirectoryAdoptionStageCli.test.cjs @@ -10,6 +10,9 @@ const BINARY = path.join(__dirname, '../dist/lifecycle/adoptionCli.js'); const DIRECTORY_INSPECT = 'local-data-directory.adoption.inspect'; const DIRECTORY_STAGE = 'local-data-directory.adoption.stage'; const DIRECTORY_VERIFY = 'local-data-directory.adoption.verify'; +const DIRECTORY_TRANSFORM = 'local-data-directory.adoption.transform'; +const DIRECTORY_TRANSFORM_VERIFY = + 'local-data-directory.adoption.transform.verify'; function privateDirectory(directoryPath) { fs.mkdirSync(directoryPath, { recursive: true, mode: 0o700 }); @@ -275,6 +278,99 @@ function verifyOptions(value, prepared, manifestDigest) { }; } +function createKeyvDatabase(databasePath, values = {}) { + fs.rmSync(databasePath, { force: true }); + const database = new DatabaseSync(databasePath); + database.exec('CREATE TABLE keyv(key VARCHAR(255) PRIMARY KEY, value TEXT)'); + const insert = database.prepare('INSERT INTO keyv(key, value) VALUES (?, ?)'); + const entries = { + 'keyv:authInfo': { + value: { token: values.authSecret ?? 'legacy-auth-token-never-carried' }, + expires: null, + }, + 'keyv:apps': { value: [{ id: 'legacy-app' }], expires: null }, + 'keyv:lang': { value: 'en', expires: null }, + ...(values.extra ?? {}), + }; + for (const [key, value] of Object.entries(entries)) { + insert.run(key, JSON.stringify(value)); + } + database.close(); + fs.chmodSync(databasePath, 0o600); +} + +function configureTransformationInput(value) { + const secrets = { + environmentName: 'D386_API_TOKEN', + environmentValue: 'd386-environment-secret', + projectId: 'project-d386', + sshAlias: 'repository-key', + sshValue: + '-----BEGIN OPENSSH PRIVATE KEY-----\nZDM4Ni1wcml2YXRlLWtleQ==\n-----END OPENSSH PRIVATE KEY-----\n', + authValue: 'legacy-auth-token-never-carried', + }; + privateFile( + path.join(value.dataRoot, 'config', 'config.sh'), + [ + `export ${secrets.environmentName}='${secrets.environmentValue}'`, + 'AutoStartBot=false', + 'export EMPTY_VALUE=', + '', + ].join('\n'), + ); + createKeyvDatabase(path.join(value.dataRoot, 'db', 'keyv.sqlite'), { + authSecret: secrets.authValue, + }); + privateFile( + path.join(value.dataRoot, 'ssh.d', secrets.sshAlias), + secrets.sshValue, + ); + privateFile( + path.join(value.dataRoot, 'ssh.d', `${secrets.sshAlias}.config`), + [ + `Host ${secrets.sshAlias}`, + ` IdentityFile /root/.ssh/${secrets.sshAlias}`, + ' StrictHostKeyChecking no', + ' ProxyCommand nc -x legacy-proxy:1080 %h %p', + '', + ].join('\n'), + ); + value.transformationParent = path.join( + value.deploymentRoot, + 'transformations', + ); + value.transformationRoot = path.join( + value.transformationParent, + 'reviewed-data-v1', + ); + privateDirectory(value.transformationParent); + return secrets; +} + +function stageForTransformation(value) { + const prepared = prepare(value); + const staged = run( + value, + 'directory-stage-for-transformation', + DIRECTORY_STAGE, + stageOptions(value, prepared), + ).result; + return { prepared, staged }; +} + +function transformationOptions(value, prepared, staged, projectId) { + return { + deploymentRoot: value.deploymentRoot, + dataRoot: value.dataRoot, + stagingRoot: value.stagingRoot, + transformationRoot: value.transformationRoot, + projectId, + profile: 'edge', + expectedManifestDigest: staged.evidence.manifestDigest, + sqlite: sqliteBinding(value, prepared.activationDigest), + }; +} + test('stages only reviewed payloads behind the real SQLite activation fence', (t) => { const value = fixture(t); const prepared = prepare(value); @@ -479,3 +575,355 @@ test('widened directory staging commands fail closed before source access', (t) 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID', ); }); + +test('prepares and exactly verifies a disabled versioned legacy transformation', (t) => { + const value = fixture(t); + const secrets = configureTransformationInput(value); + const { prepared, staged } = stageForTransformation(value); + const options = transformationOptions( + value, + prepared, + staged, + secrets.projectId, + ); + const transformed = run( + value, + 'directory-transform', + DIRECTORY_TRANSFORM, + options, + ); + + assert.equal(transformed.result.status, 'prepared'); + assert.equal(transformed.result.evidence.assessment, 'ready'); + assert.match( + transformed.result.evidence.transformationDigest, + /^[0-9a-f]{64}$/, + ); + for (const sensitive of [ + value.transformationRoot, + secrets.projectId, + secrets.environmentName, + secrets.environmentValue, + secrets.sshAlias, + secrets.sshValue, + secrets.authValue, + ]) { + assert.equal(transformed.child.stdout.includes(sensitive), false); + } + assert.deepEqual(fs.readdirSync(value.transformationRoot).sort(), [ + 'manifest.json', + 'model', + ]); + const modelRoot = path.join(value.transformationRoot, 'model'); + const manifestText = fs.readFileSync( + path.join(value.transformationRoot, 'manifest.json'), + 'utf8', + ); + for (const sensitive of [ + secrets.projectId, + secrets.environmentName, + secrets.environmentValue, + secrets.sshAlias, + secrets.sshValue, + secrets.authValue, + ]) { + assert.equal(manifestText.includes(sensitive), false); + } + const config = JSON.parse( + fs.readFileSync(path.join(modelRoot, 'config.json'), 'utf8'), + ); + assert.deepEqual( + config.exportedEnvironment.map((entry) => entry.environmentName), + [secrets.environmentName], + ); + assert.equal(config.retiredSettings[0].name, 'AutoStartBot'); + assert.equal(config.omittedEmptyExports, 1); + assert.equal(config.activation, 'disabled'); + + const keyv = JSON.parse( + fs.readFileSync(path.join(modelRoot, 'keyv.json'), 'utf8'), + ); + assert.equal(keyv.integrity, 'ok'); + assert.equal(keyv.cachedLocale, 'en'); + assert.equal( + keyv.mappings.find((entry) => entry.legacyKey === 'keyv:authInfo').state, + 'retired', + ); + assert.equal(JSON.stringify(keyv).includes(secrets.authValue), false); + + const ssh = JSON.parse( + fs.readFileSync(path.join(modelRoot, 'ssh.json'), 'utf8'), + ); + assert.equal(ssh.bindings[0].activation, 'disabled'); + assert.equal(ssh.bindings[0].hostKeyPolicy, 'operator_verification_required'); + assert.equal(ssh.bindings[0].legacyProxyCommandPresent, true); + assert.equal(ssh.bindings[0].legacyHostKeyBypassPresent, true); + assert.equal(JSON.stringify(ssh).includes('legacy-proxy'), false); + assert.equal(JSON.stringify(ssh).includes('StrictHostKeyChecking'), false); + + const importPlan = JSON.parse( + fs.readFileSync(path.join(modelRoot, 'secret-imports.json'), 'utf8'), + ); + assert.equal(importPlan.state, 'prepared'); + assert.equal(importPlan.projectId, secrets.projectId); + assert.equal(importPlan.imports.length, 2); + const secretValues = importPlan.imports.map((entry) => { + const secretPath = path.join(modelRoot, entry.valueFile); + assert.equal(fs.statSync(secretPath).mode & 0o777, 0o600); + return JSON.parse(fs.readFileSync(secretPath, 'utf8')).value; + }); + assert.deepEqual( + secretValues.sort(), + [secrets.environmentValue, secrets.sshValue].sort(), + ); + const fullTarget = + fs.readFileSync(path.join(modelRoot, 'secret-imports.json'), 'utf8') + + fs.readFileSync(path.join(modelRoot, 'keyv.json'), 'utf8'); + assert.equal(fullTarget.includes(secrets.authValue), false); + + const verified = run( + value, + 'directory-transform-verify', + DIRECTORY_TRANSFORM_VERIFY, + { + ...options, + expectedTransformationDigest: + transformed.result.evidence.transformationDigest, + }, + ).result; + assert.equal(verified.status, 'verified'); + assert.deepEqual(verified.evidence, transformed.result.evidence); + const replayed = run( + value, + 'directory-transform-verify-replay', + DIRECTORY_TRANSFORM_VERIFY, + { + ...options, + expectedTransformationDigest: + transformed.result.evidence.transformationDigest, + }, + ).result; + assert.deepEqual(replayed, verified); +}); + +test('transformation verification rejects target and current source drift', (t) => { + const target = fixture(t); + const targetSecrets = configureTransformationInput(target); + const targetStage = stageForTransformation(target); + const targetOptions = transformationOptions( + target, + targetStage.prepared, + targetStage.staged, + targetSecrets.projectId, + ); + const transformed = run( + target, + 'directory-transform-before-target-drift', + DIRECTORY_TRANSFORM, + targetOptions, + ).result; + const secretFile = fs.readdirSync( + path.join(target.transformationRoot, 'model', 'secret-values'), + )[0]; + privateFile( + path.join(target.transformationRoot, 'model', 'secret-values', secretFile), + '{"schemaVersion":1,"kind":"qinglong3-local-secret-value","value":"tampered"}\n', + ); + const targetDrift = runRaw( + target, + 'directory-transform-target-drift', + DIRECTORY_TRANSFORM_VERIFY, + { + ...targetOptions, + expectedTransformationDigest: transformed.evidence.transformationDigest, + }, + ); + assert.equal(targetDrift.status, 1); + assert.equal( + JSON.parse(targetDrift.stderr).code, + 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID', + ); + + const source = fixture(t); + const sourceSecrets = configureTransformationInput(source); + const sourceStage = stageForTransformation(source); + const sourceOptions = transformationOptions( + source, + sourceStage.prepared, + sourceStage.staged, + sourceSecrets.projectId, + ); + const sourceTransformed = run( + source, + 'directory-transform-before-source-drift', + DIRECTORY_TRANSFORM, + sourceOptions, + ).result; + privateFile( + path.join(source.dataRoot, 'config', 'config.sh'), + 'export D386_API_TOKEN=source-drift\n', + ); + const sourceDrift = runRaw( + source, + 'directory-transform-source-drift', + DIRECTORY_TRANSFORM_VERIFY, + { + ...sourceOptions, + expectedTransformationDigest: + sourceTransformed.evidence.transformationDigest, + }, + ); + assert.equal(sourceDrift.status, 1); + assert.equal( + JSON.parse(sourceDrift.stderr).code, + 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID', + ); +}); + +test('unknown legacy behavior is retained as disabled manual-review evidence', (t) => { + const value = fixture(t); + const secrets = configureTransformationInput(value); + privateFile( + path.join(value.dataRoot, 'config', 'config.sh'), + `export ${secrets.environmentName}=${secrets.environmentValue}\neval dangerous\n`, + ); + const keyvPath = path.join(value.dataRoot, 'db', 'keyv.sqlite'); + const keyv = new DatabaseSync(keyvPath); + keyv + .prepare('INSERT INTO keyv(key, value) VALUES (?, ?)') + .run('keyv:unknown', JSON.stringify({ value: 'retain', expires: null })); + keyv.close(); + privateFile(path.join(value.dataRoot, 'ssh.d', 'unpaired-key'), 'manual'); + const { prepared, staged } = stageForTransformation(value); + const transformed = run( + value, + 'directory-transform-manual', + DIRECTORY_TRANSFORM, + transformationOptions(value, prepared, staged, secrets.projectId), + ).result; + + assert.equal(transformed.evidence.assessment, 'manual_required'); + assert.equal(transformed.evidence.model.manualCategories, 3); + const modelRoot = path.join(value.transformationRoot, 'model'); + assert.equal( + JSON.parse(fs.readFileSync(path.join(modelRoot, 'config.json'), 'utf8')) + .unsupportedLines, + 1, + ); + assert.equal( + JSON.parse(fs.readFileSync(path.join(modelRoot, 'keyv.json'), 'utf8')) + .unknownEntries, + 1, + ); + assert.equal( + JSON.parse(fs.readFileSync(path.join(modelRoot, 'ssh.json'), 'utf8')) + .manualEntries, + 1, + ); + const manual = JSON.parse( + fs.readFileSync(path.join(modelRoot, 'manual-review.json'), 'utf8'), + ); + assert.equal(manual.required, true); + assert.equal(manual.activation, 'disabled'); +}); + +test('edge Secret budget leaves no-replace recovery residue', (t) => { + const value = fixture(t); + const secrets = configureTransformationInput(value); + privateFile( + path.join(value.dataRoot, 'config', 'config.sh'), + `${Array.from( + { length: 129 }, + (_, index) => `export D386_${index}=value`, + ).join('\n')}\n`, + ); + const { prepared, staged } = stageForTransformation(value); + const options = transformationOptions( + value, + prepared, + staged, + secrets.projectId, + ); + const overBudget = runRaw( + value, + 'directory-transform-over-budget', + DIRECTORY_TRANSFORM, + options, + ); + assert.equal(overBudget.status, 1); + assert.equal( + JSON.parse(overBudget.stderr).code, + 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID', + ); + const marker = path.join(value.transformationRoot, '.incomplete'); + const residue = fs.readFileSync(marker, 'utf8'); + const replay = runRaw( + value, + 'directory-transform-over-budget-replay', + DIRECTORY_TRANSFORM, + options, + ); + assert.equal(replay.status, 1); + assert.equal(fs.readFileSync(marker, 'utf8'), residue); +}); + +test('widened transformation commands fail closed before source access', (t) => { + const value = fixture(t); + const missingData = path.join(value.deploymentRoot, 'missing-source'); + const child = runRaw(value, 'widened-transform', DIRECTORY_TRANSFORM, { + deploymentRoot: value.deploymentRoot, + dataRoot: missingData, + stagingRoot: path.join(value.deploymentRoot, 'missing-stage'), + transformationRoot: path.join(value.deploymentRoot, 'missing-transform'), + projectId: 'project-d386', + profile: 'edge', + expectedManifestDigest: '0'.repeat(64), + sqlite: { + sourcePath: path.join(missingData, 'db', 'database.sqlite'), + targetPath: path.join(value.artifactsDirectory, 'missing-target'), + recoveryPath: path.join(value.artifactsDirectory, 'missing-recovery'), + manifestPath: path.join(value.artifactsDirectory, 'missing-manifest'), + activationPath: path.join(value.artifactsDirectory, 'missing-activation'), + expectedActivationDigest: '0'.repeat(64), + }, + extraAuthority: true, + }); + assert.equal(child.status, 1); + assert.equal(child.stdout, ''); + assert.equal( + JSON.parse(child.stderr).code, + 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID', + ); + + const invalidProject = runRaw( + value, + 'invalid-transform-project', + DIRECTORY_TRANSFORM, + { + deploymentRoot: value.deploymentRoot, + dataRoot: missingData, + stagingRoot: path.join(value.deploymentRoot, 'missing-stage'), + transformationRoot: path.join(value.deploymentRoot, 'missing-transform'), + projectId: 'project with spaces', + profile: 'edge', + expectedManifestDigest: '0'.repeat(64), + sqlite: { + sourcePath: path.join(missingData, 'db', 'database.sqlite'), + targetPath: path.join(value.artifactsDirectory, 'missing-target'), + recoveryPath: path.join(value.artifactsDirectory, 'missing-recovery'), + manifestPath: path.join(value.artifactsDirectory, 'missing-manifest'), + activationPath: path.join( + value.artifactsDirectory, + 'missing-activation', + ), + expectedActivationDigest: '0'.repeat(64), + }, + }, + ); + assert.equal(invalidProject.status, 1); + assert.equal(invalidProject.stdout, ''); + assert.equal( + JSON.parse(invalidProject.stderr).code, + 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID', + ); +}); diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index ecca580b..442a0784 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -207,10 +207,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: localOwnerCli.rootSourceFileRoles, }, { - sourceFiles: 122, + sourceFiles: 129, rootSourceFiles: 1, rootSourceLines: 50, - nestedSourceFiles: 121, + nestedSourceFiles: 128, rootSourceFileRoles: { 'cli.ts': 'binary_entry' }, }, );