From dbd219bedcf50d940a46d3427f19f179508fc99b Mon Sep 17 00:00:00 2001 From: whyour Date: Sun, 23 Aug 2026 15:29:00 +0800 Subject: [PATCH] feat(local): classify legacy secret config migration --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 17 + ...-config-reconciliation-and-task-binding.md | 134 +++++ docs/adr/README.md | 9 +- packages/ql3-local-admin/package.json | 5 + .../environmentInspection.ts | 536 ++++++++++++++++++ .../test/legacyEnvironmentInspection.test.cjs | 214 +++++++ 6 files changed, 914 insertions(+), 1 deletion(-) create mode 100644 docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md create mode 100644 packages/ql3-local-admin/src/legacy-adoption/secret-and-config/environmentInspection.ts create mode 100644 packages/ql3-local-admin/test/legacyEnvironmentInspection.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index b9809e6d..42d371fe 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,23 @@ 最新增量证据（2026-08-23）： +- D-397/ADR-0491（进行中）：Secret/Config reconciliation 已先冻结行为保持边界，不能把“密文已保存”冒充“任务已迁移”。2.x `Envs` + 的 active 行必须按 `isPinned DESC、position DESC、createdAt ASC、id ASC` 重放旧顺序，同名值用 `&` 形成唯一 effective Secret；该 + Secret 后续必须在同一事务中绑定到经 Automation adoption ledger 证明的全部 Legacy Task 新修订，并同步追加指向新 Task revision 的 Trigger/dispatch + 修订。disabled 行逐行加密保全但不激活；非法/保留名称、异常 status/ordering、单值或总字节超限、部分组失败均进入 manual，不能静默丢行。 + + 第一切片已在既有 `@qinglong/local-admin/src/legacy-adoption/secret-and-config/` 落地 content-free inspection 与精确私有 subpath，没有新增 package、 + dependency、daemon 或 `src` 根平铺。Edge/Standalone 行数上限分别为 10,000/100,000，disabled preservation 为 128/512；共同受 256 个 active + binding、单值 16 KiB 与总 effective 64 KiB 限制。实现逐行扫描，active 在途 value 有固定内存上限，disabled 以第二遍逐项交付；inventory/row + diagnostics 不含 Env name/value/row body。absent、unsupported schema、Edge over-budget、旧顺序、同名连接、disabled、保留 `QL3_`、异常状态与 + overflow 均已覆盖，Local Admin 完整测试 `95/95`。 + + D-385～D-388 的 `config.sh`/Keyv/SSH data-directory lineage 与 SQLite `Envs` 保持分离；当前无稳定生产 schema 的历史 `Configs` 表继续 sealed+manual， + 不猜字段。后续切片必须完成独立 signed decision、Secret envelope + audit + Task/Trigger/dispatch + receipt ledger 的单事务发布、prepared/apply/rollback + lineage、completion 下一 schema 与备份回收。D-397 apply 只声明 sealed source retained 且 `physicalErasureGuaranteed=false`；明文销毁必须在 restart/ + readiness、观察窗和 rollback retention 之后另行强认证。Cluster 必须使用 PostgreSQL SERIALIZABLE ledger、外部 KMS/Secret provider 与 HA evidence， + 不复用 Local SQLite/POSIX authority，也不得把明文写入 PostgreSQL、ConfigMap、Pod env 或 Job command。 + - D-396/ADR-0490（已验收）：Run History 不再只有永久 `manual_external`，但也没有被错误实现为 Legacy 日志到 3.0 Run ledger 的回灌。 新的 Local adapter 以 ADR-0482 sealed capture bundle 作为 append-only 保全资产：Legacy history 必须逐事实选择 `retain_both`，Target history 必须选择 `retain_target`；receipt 只绑定 signed review、application、bundle fingerprint、领域 inventory 与有界 fact counts，不保存表名、Run ID、 diff --git a/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md b/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md new file mode 100644 index 00000000..7414266f --- /dev/null +++ b/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md @@ -0,0 +1,134 @@ +# ADR-0491：有界 Secret/Config Reconciliation 与任务环境绑定 + +- 状态：Proposed（D-397 第一切片已实现 Legacy Env inspection，原子 application 尚未完成） +- 日期：2026-08-23 +- 决策：D-397 +- 关联：ADR-0073、ADR-0074、ADR-0092、ADR-0094、ADR-0480、ADR-0482、ADR-0483、ADR-0484、ADR-0485、ADR-0486、ADR-0487、ADR-0488、ADR-0490 + +## 背景 + +QingLong 2.x 的 `Envs` 不是简单的 `name → value` 字典。运行时只使用 `status=normal` 的行，先按 `isPinned DESC、position DESC、createdAt ASC` 排序，再把同名行的值用 `&` 连接后注入所有 Legacy Task。停用行仍属于用户数据，但不应被激活。直接把每一行转换成一个 3.0 Secret 会改变同名变量语义；只把数据加密保存而不更新 TaskDefinition，又会让迁移在形式上完成、实际任务却失去环境变量。 + +3.0 已具有项目内版本化 SecretRef、TaskDefinition `command@v1` 的 secret environment binding、本地按引用解密注入和 Cluster remote Secret delivery。D-397 应复用这些边界，不能恢复 2.x 的全局明文 `export` 文件，也不能让 Secret 值进入 plan、review、receipt、日志、命令文件或 stdout。 + +此外，D-385～D-388 已处理数据目录中的 `config.sh`、Keyv 与 SSH material，但没有处理 Legacy SQLite 的 `Envs`。二者是不同 source lineage，不能因为目标端已经存在 Data Directory Adoption Secret 就把数据库 Env 判定为已迁移。当前 2.x 生产代码也没有稳定的 SQLite `Configs` 模型；捕获中出现的历史 `Configs` 表必须按未知 schema 失败关闭，不能套用 `Envs` 规则。 + +## 决策 + +### 1. 固定执行顺序与领域授权 + +Local reconciliation 顺序固定为： + +```text +application plan + → Automation plan / review / apply（若需要） + → Secret/Config plan / review / apply（若需要） + → Run History preservation（若需要，绑定最新 head） + → cross-domain completion +``` + +Secret/Config 不消费 Automation decision 作为自身授权。它必须重新绑定 D-391 的 `secret_and_config` facts、同一 sealed bundle、D-392 application plan、当前 target snapshot 与独立的逐候选 signed decision。强认证 User、Project Policy、Secret custody 与 Task mutation authority 都要在写事务前及事务内重新验证。 + +存在 active Env 时，Automation 必须已经完成，且至少一个经 `QingLong3LegacyAdoptions` 证明的 Legacy Task 可绑定；否则不得用“Secret 已保存”冒充行为迁移。只有停用 Env 的场景可以在 Automation `no_effect` 后做纯保全。 + +### 2. Legacy Env 的确定性语义 + +`Envs` inspection 要求 `id/name/value`，并兼容缺省的 `status/isPinned/position/createdAt`。合法环境变量名必须符合 shell 与 3.0 `command@v1` 的共同子集，且不得使用 `QL3_` 保留前缀。 + +active 行按以下顺序处理： + +```text +COALESCE(isPinned, 0) DESC, +position DESC, +createdAt ASC, +id ASC +``` + +同名行按该顺序用单个 `&` 连接，形成一个 effective value；它只产生一个 active Secret 和一个 Task environment binding。停用行不参与连接，每行形成一个 disabled-preservation candidate，后续可由 Owner 显式恢复或重建，但不会绑定到任务。 + +诊断只保存 row ordinal、source digest、disposition 与 reason。inventory 只保存计数、状态与 digest。变量名、值、remarks、labels、原始排序字段和 row body 均不得进入公开 evidence。candidate 只能通过 Local Admin 的精确私有子路径流式交给短生命周期 planner/publisher。 + +以下情况必须 `manual_required`：非法/保留名称、非文本或含 NUL 的值、异常 status/ordering metadata、同组 effective value 超过 16 KiB、全部 active effective values 超过 64 KiB、active binding 超过 256、停用保全数量超预算、必要列缺失或 source 漂移。不能丢弃异常行后对剩余行做“部分成功”绑定。 + +### 3. Edge 与 Standalone 预算 + +第一切片固定： + +| 预算 | Edge | Standalone | +| --- | ---: | ---: | +| Legacy Env 行数 | 10,000 | 100,000 | +| disabled preservation candidates | 128 | 512 | +| active bindings | 256 | 256 | +| 单个 effective value | 16 KiB | 16 KiB | +| 全部 active effective values | 64 KiB | 64 KiB | + +实现逐行读取，不把整张 `Envs` 或全部停用值加载到内存；active value 的在途内存由 64 KiB 合同封顶，停用值通过第二次有界扫描逐个交付。它位于既有 `@qinglong/local-admin/src/legacy-adoption/secret-and-config/`，不新增 workspace package、production dependency、daemon、timer、watcher、listener、socket、数据库连接池或 `src` 根平铺文件。 + +### 4. 原子 application 必须同时完成 custody 与行为绑定 + +后续 D-397 application 必须在一个 `BEGIN IMMEDIATE` 事务内完成： + +1. 复验 Project/RoleBinding fence、signed decision、sealed source、target snapshot 与当前 instance head； +2. 为每个 active effective Env 写入加密 Local Secret envelope、content-free `secret.create` audit 与 adoption item； +3. 为每个 disabled candidate 写入加密但未绑定的 Secret，并记录 disabled preservation disposition； +4. 为每个经 Automation adoption ledger 证明的 Legacy Task 追加 TaskDefinition revision，把 active Env 绑定为 `kind=secret` 的 environment； +5. 为指向旧 Task revision 的 Legacy Trigger 追加 Trigger revision，并更新 local schedule/dispatch revision； +6. 写入不可变 Secret/Config reconciliation receipt ledger 后一次提交。 + +任一 Secret、Task、Trigger、dispatch、audit、ledger 或 fence 冲突都回滚整个事务。禁止先提交 Secret 再逐任务修补，也禁止在现有 Task revision 上原地改 JSON。目标已有同名/同源 Secret、非 Legacy Task、Plugin-owned Task 或用户在 stopped window 中产生的 revision 都按冲突处理，不自动覆盖或重命名。 + +由于该 adapter 执行 DML，它需要独立的 prepared/applied/rolled-back lineage 与写前 target backup；无 DML 的 Run History preservation 继续只绑定最新 head。跨领域 completion 后才可回收 Automation 与 Secret/Config 两份 rollback material。空间不足必须在 prepare 前失败，不得在低配设备上边写边赌。 + +### 5. preserve、destroy 与 completion + +D-397 application 的成功终态只声明： + +- active Env 已加密并绑定到全部已采纳 Legacy Task； +- disabled Env 已加密保全且保持未激活； +- sealed source database 与 rollback backup 仍然存在； +- `physicalErasureGuaranteed=false`。 + +它不删除 Legacy SQLite、capture bundle、Data Directory Adoption Secret 或外部备份。明文销毁必须是目标 restart/readiness、观察窗口和 rollback 保留策略之后的独立强认证 ceremony，并明确列出可删除对象；闪存、CoW 文件系统与备份介质不能声称物理擦除保证。 + +Completion 需要演进为兼容旧 receipt 的下一 schema：`secret_and_config + adapter_required` 必须携带 exact Secret/Config application evidence；active Env 的 custody-only receipt 不合法。`adapterCount` 由实际 Automation、Secret/Config、Run History 三类证明推导，调用方不能自报。任何未知 `Configs`、manual Env、未绑定 Task、stale run-history proof、rolled-back state 或 target drift 都继续拒绝 `reconciliation_completed`。 + +### 6. `Configs` 与 D-385～D-388 的边界 + +- 数据目录 `config.sh`、Keyv、SSH 继续由 ADR-0477～ADR-0481 的 lineage 管理，默认 activation 仍为 disabled； +- SQLite `Envs` 由本 ADR 管理，active effective value 必须绑定 Task 才能终态； +- SQLite `Configs` 在没有版本化 schema transformer 前只能保留在 sealed bundle 并进入 manual，不自动解析字段或猜测文件路径； +- 两条 lineage 可以在 target Secret store 共存，但发布器必须检查 Secret name/source digest 冲突，不能重复接管同一 material。 + +### 7. Cluster 是独立实现 + +Cluster migration 不复用 Local SQLite handle、POSIX keyring、instance head 或本机 backup。它需要独立的 PostgreSQL SERIALIZABLE ledger、外部 KMS/Secret provider custody、exact Project/Task revision fence、Trigger revision 与 remote delivery authority，以及 HA promotion 后仍可验证的 receipt。 + +Cluster 不得把 Legacy Env 明文写入 PostgreSQL、ConfigMap、Job command、Pod environment 或迁移日志。一次性 migration Job 应使用精确投影、短期身份、零 watcher/controller，并在事务提交前后绑定 PostgreSQL timeline/role readiness。Edge 不安装或加载这些 Cluster 依赖；Cluster 也不能把本机 10,000 行预算冒充集群容量证明。 + +## 被拒绝的替代方案 + +### 恢复全局 `export` 文件 + +拒绝。它重新引入明文落盘、所有 Task 隐式继承、无法审计的动态作用域和本地/Cluster 行为分叉。 + +### 每个 Legacy Env 行直接创建一个 active Secret + +拒绝。同名行在 2.x 中按顺序用 `&` 合并；逐行绑定既无法表达该语义，也会产生重复 environment name。 + +### 只做 Secret custody，不更新任务 + +拒绝。数据存在不等于任务行为已迁移；active Env 未绑定时 completion 必须失败。 + +### 在 Automation apply 中顺便导入 Env + +拒绝。Automation decision 没有 Secret custody authority，也没有逐 Env 决策；把两个领域塞入同一签名会扩大权限并破坏独立 rollback/evidence。 + +### 自动猜测 `Configs` 表结构 + +拒绝。历史版本和第三方 fork 可能有同名异义表；字段猜测会把未知数据变成错误配置或 Secret。 + +## 当前验证与后续门禁 + +D-397 第一切片已经实现并测试：absent、unsupported、Edge over-budget、2.x 顺序、同名连接、disabled preservation、保留前缀、异常状态、effective overflow、candidate digest 与 content-free diagnostics。Local Admin 完整测试为 95/95。 + +转为 Accepted 前仍必须完成：私有 row plan 与 signed decision、原子 Secret/Task/Trigger/dispatch publisher、prepared/apply/rollback response-loss、completion schema 演进、完整 Local Owner/18-package/boundary/artifact gates、真实 Edge 空间预算、PostgreSQL HA 与 Cluster Secret provider live gate。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 853bacc7..ef0d628a 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -487,7 +487,14 @@ | [ADR-0481](./ADR-0481-committed-legacy-data-deployment-lineage.md) | Committed Legacy Data Receipt 的本机部署 Lineage | Accepted | | [ADR-0482](./ADR-0482-stopped-local-sqlite-reconciliation-recovery-bundle.md) | 停止态 Local SQLite Reconciliation Recovery Bundle | Accepted | | [ADR-0483](./ADR-0483-sealed-reconciliation-bundle-domain-plan.md) | 密封 Reconciliation Bundle 的有界数据域计划 | Accepted | -| [ADR-0484](./ADR-0484-private-reconciliation-diagnostics-and-reviewed-decision.md) | 私有 Reconciliation 诊断与受认证人工裁决 | Proposed（D-391 契约冻结） | +| [ADR-0484](./ADR-0484-private-reconciliation-diagnostics-and-reviewed-decision.md) | 私有 Reconciliation 诊断与受认证人工裁决 | Accepted | +| [ADR-0485](./ADR-0485-reconciliation-application-coordinator.md) | Reconciliation 应用协调器与领域 Adapter 边界 | Accepted | +| [ADR-0486](./ADR-0486-bounded-automation-row-reconciliation-plan.md) | 有界 Automation 行级 Reconciliation Plan | Accepted | +| [ADR-0487](./ADR-0487-reconciliation-apply-storage-lifecycle.md) | Reconciliation Apply 证据封存与回滚存储生命周期 | Accepted | +| [ADR-0488](./ADR-0488-cross-domain-reconciliation-completion-fence.md) | 跨领域 Reconciliation 完成围栏与目标重启授权 | Accepted | +| [ADR-0489](./ADR-0489-service-manager-completion-restart-lineage.md) | Service Manager 完成围栏重启谱系 | Accepted | +| [ADR-0490](./ADR-0490-run-history-terminal-preservation.md) | Run History 终态保全与跨领域完成证明 | Accepted | +| [ADR-0491](./ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md) | 有界 Secret/Config Reconciliation 与任务环境绑定 | Proposed（D-397 第一切片） | ## 规则 diff --git a/packages/ql3-local-admin/package.json b/packages/ql3-local-admin/package.json index 12807feb..8c0d3174 100644 --- a/packages/ql3-local-admin/package.json +++ b/packages/ql3-local-admin/package.json @@ -45,6 +45,11 @@ "require": "./dist/legacy-adoption/legacyCrontabInspection.js", "default": "./dist/legacy-adoption/legacyCrontabInspection.js" }, + "./reconciliation-secret-and-config-inspection": { + "types": "./dist/legacy-adoption/secret-and-config/environmentInspection.d.ts", + "require": "./dist/legacy-adoption/secret-and-config/environmentInspection.js", + "default": "./dist/legacy-adoption/secret-and-config/environmentInspection.js" + }, "./reconciliation-automation-decision": { "types": "./dist/legacy-adoption/reconciliationAutomationDecision.d.ts", "require": "./dist/legacy-adoption/reconciliationAutomationDecision.js", diff --git a/packages/ql3-local-admin/src/legacy-adoption/secret-and-config/environmentInspection.ts b/packages/ql3-local-admin/src/legacy-adoption/secret-and-config/environmentInspection.ts new file mode 100644 index 00000000..4587c07c --- /dev/null +++ b/packages/ql3-local-admin/src/legacy-adoption/secret-and-config/environmentInspection.ts @@ -0,0 +1,536 @@ +import { createHash } from 'node:crypto'; +import type { DatabaseSync } from 'node:sqlite'; + +export const MAX_EDGE_LEGACY_ENVIRONMENT_ROWS = 10_000; +export const MAX_STANDALONE_LEGACY_ENVIRONMENT_ROWS = 100_000; +export const MAX_LEGACY_ENVIRONMENT_BINDINGS = 256; +export const MAX_LEGACY_ENVIRONMENT_VALUE_BYTES = 16 * 1024; +export const MAX_LEGACY_ENVIRONMENT_EFFECTIVE_BYTES = 64 * 1024; +export const MAX_EDGE_LEGACY_DISABLED_ENVIRONMENTS = 128; +export const MAX_STANDALONE_LEGACY_DISABLED_ENVIRONMENTS = 512; + +const ENVIRONMENT_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; +const COLUMNS = Object.freeze([ + 'id', + 'name', + 'value', + 'status', + 'isPinned', + 'position', + 'createdAt', +] as const); + +type Column = (typeof COLUMNS)[number]; +type LegacyRow = Record; + +export type LegacyEnvironmentTableState = + | 'absent' + | 'supported' + | 'unsupported_schema' + | 'budget_exceeded'; + +export type LegacyEnvironmentRowDisposition = + | 'active_member' + | 'preserve_disabled' + | 'manual_required'; + +export type LegacyEnvironmentRowReason = + | 'identifier_invalid' + | 'name_invalid' + | 'value_invalid' + | 'status_invalid' + | 'ordering_metadata_invalid'; + +export interface LegacyEnvironmentRowInspection { + readonly rowOrdinal: number; + readonly sourceDigest: string; + readonly disposition: LegacyEnvironmentRowDisposition; + readonly reasons: readonly LegacyEnvironmentRowReason[]; +} + +export interface LegacyEnvironmentActiveBindingCandidate { + readonly kind: 'active_binding'; + readonly environmentName: string; + readonly value: string; + readonly sourceRowCount: number; + readonly sourceSetDigest: string; + readonly candidateDigest: string; +} + +export interface LegacyEnvironmentDisabledPreservationCandidate { + readonly kind: 'disabled_preservation'; + readonly environmentName: string; + readonly value: string; + readonly sourceRowOrdinal: number; + readonly sourceDigest: string; + readonly candidateDigest: string; +} + +export type LegacyEnvironmentCandidate = + | LegacyEnvironmentActiveBindingCandidate + | LegacyEnvironmentDisabledPreservationCandidate; + +export interface LegacyEnvironmentInventory { + readonly schemaVersion: 1; + readonly kind: 'qinglong3-legacy-environment-inventory'; + readonly profile: 'edge' | 'standalone'; + readonly tableState: LegacyEnvironmentTableState; + readonly rowCount: number; + readonly activeRowCount: number; + readonly disabledRowCount: number; + readonly manualRowCount: number; + readonly activeGroupCount: number; + readonly bindingReadyCount: number; + readonly preservationReadyCount: number; + readonly manualGroupCount: number; + readonly mutationReady: boolean; + readonly inventoryDigest: string; +} + +export interface VisitLegacyEnvironmentOptions { + readonly profile: 'edge' | 'standalone'; + readonly visitRow?: (row: Readonly) => void; + readonly visitCandidate?: ( + candidate: Readonly, + ) => void; +} + +export class LegacyEnvironmentInspectionError extends Error { + constructor(message: string, readonly cause?: unknown) { + super(`Legacy environment inspection failed: ${message}`); + this.name = 'LegacyEnvironmentInspectionError'; + } +} + +interface ActiveGroup { + readonly name: string; + readonly rowDigests: string[]; + readonly values: string[]; + valueBytes: number; + valid: boolean; +} + +function digest(domain: string, value: unknown): string { + return createHash('sha256') + .update(domain) + .update('\0') + .update(JSON.stringify(value)) + .digest('hex'); +} + +function scalarEvidence(value: unknown): readonly unknown[] { + if (value === null) return Object.freeze(['null']); + if (value === undefined) return Object.freeze(['missing']); + if (typeof value === 'string') { + return Object.freeze([ + 'text', + Buffer.byteLength(value, 'utf8'), + createHash('sha256').update(value).digest('hex'), + ]); + } + if (typeof value === 'number') { + return Object.freeze([ + 'number', + Number.isFinite(value) + ? String(Object.is(value, -0) ? 0 : value) + : 'invalid', + ]); + } + if (typeof value === 'bigint') { + return Object.freeze(['bigint', value.toString()]); + } + if (value instanceof Uint8Array) { + return Object.freeze([ + 'blob', + value.byteLength, + createHash('sha256').update(value).digest('hex'), + ]); + } + return Object.freeze(['unsupported', typeof value]); +} + +function tableNames(client: DatabaseSync): readonly string[] { + return Object.freeze( + ( + client + .prepare( + `SELECT "name" FROM "sqlite_schema" + WHERE "type" = 'table' AND "name" NOT LIKE 'sqlite_%' + ORDER BY "name"`, + ) + .all() as { readonly name?: unknown }[] + ) + .map(({ name }) => name) + .filter((name): name is string => typeof name === 'string'), + ); +} + +function columns(client: DatabaseSync): ReadonlySet { + return new Set( + ( + client.prepare('PRAGMA table_info("Envs")').all() as { + readonly name?: unknown; + }[] + ) + .map(({ name }) => name) + .filter((name): name is string => typeof name === 'string'), + ); +} + +function rowCount(client: DatabaseSync): number { + const row = client.prepare('SELECT COUNT(*) AS "count" FROM "Envs"').get() as + | { readonly count?: unknown } + | undefined; + if (!row || !Number.isSafeInteger(row.count) || (row.count as number) < 0) { + throw new LegacyEnvironmentInspectionError('row count is invalid'); + } + return row.count as number; +} + +function emptyInventory( + profile: 'edge' | 'standalone', + tableState: Exclude, + count: number, + schemaColumns: readonly string[], +): Readonly { + const payload = Object.freeze({ + schemaVersion: 1 as const, + kind: 'qinglong3-legacy-environment-inventory' as const, + profile, + tableState, + rowCount: count, + activeRowCount: 0, + disabledRowCount: 0, + manualRowCount: count, + activeGroupCount: 0, + bindingReadyCount: 0, + preservationReadyCount: 0, + manualGroupCount: 0, + mutationReady: tableState === 'absent', + }); + return Object.freeze({ + ...payload, + inventoryDigest: digest( + 'qinglong3.legacy-environment-inventory.v1', + Object.freeze({ ...payload, schemaColumns }), + ), + }); +} + +function selectSql(schema: ReadonlySet): string { + const projection = COLUMNS.map((column) => { + if (schema.has(column)) return `"${column}"`; + if (column === 'status' || column === 'isPinned') { + return `0 AS "${column}"`; + } + return `NULL AS "${column}"`; + }); + const pinned = schema.has('isPinned') ? 'COALESCE("isPinned", 0)' : '0'; + const position = schema.has('position') ? '"position"' : 'NULL'; + const createdAt = schema.has('createdAt') ? '"createdAt"' : 'NULL'; + return `SELECT ${projection.join(', ')} FROM "Envs" + ORDER BY ${pinned} DESC, ${position} DESC, ${createdAt} ASC, "id" ASC`; +} + +function reasons(row: LegacyRow): readonly LegacyEnvironmentRowReason[] { + const selected: LegacyEnvironmentRowReason[] = []; + if (!Number.isSafeInteger(row.id) || (row.id as number) < 1) { + selected.push('identifier_invalid'); + } + if ( + typeof row.name !== 'string' || + !ENVIRONMENT_NAME.test(row.name) || + row.name.startsWith('QL3_') + ) { + selected.push('name_invalid'); + } + if ( + typeof row.value !== 'string' || + row.value.includes('\0') || + Buffer.byteLength(row.value, 'utf8') > MAX_LEGACY_ENVIRONMENT_VALUE_BYTES + ) { + selected.push('value_invalid'); + } + if (row.status !== 0 && row.status !== 1) selected.push('status_invalid'); + if ( + (row.isPinned !== 0 && row.isPinned !== 1) || + (row.position !== null && + (typeof row.position !== 'number' || !Number.isFinite(row.position))) || + (row.createdAt !== null && + (typeof row.createdAt !== 'string' || + row.createdAt.includes('\0') || + Buffer.byteLength(row.createdAt, 'utf8') > 128)) + ) { + selected.push('ordering_metadata_invalid'); + } + return Object.freeze(selected); +} + +function sourceDigest(row: LegacyRow): string { + return digest( + 'qinglong3.legacy-environment-row.v1', + COLUMNS.map((column) => [column, scalarEvidence(row[column])]), + ); +} + +function candidateDigest( + candidate: Omit, + value: string, +): string { + return digest('qinglong3.legacy-environment-candidate.v1', { + ...candidate, + environmentNameDigest: digest( + 'qinglong3.legacy-environment-name.v1', + candidate.environmentName, + ), + environmentName: undefined, + valueBytes: Buffer.byteLength(value, 'utf8'), + valueDigest: createHash('sha256').update(value).digest('hex'), + }); +} + +export function visitLegacyEnvironmentAdoption( + client: DatabaseSync, + options: Readonly, +): Readonly { + if ( + !client || + typeof client !== 'object' || + !options || + typeof options !== 'object' || + (options.profile !== 'edge' && options.profile !== 'standalone') || + (options.visitRow !== undefined && + typeof options.visitRow !== 'function') || + (options.visitCandidate !== undefined && + typeof options.visitCandidate !== 'function') + ) { + throw new LegacyEnvironmentInspectionError('options are invalid'); + } + const names = tableNames(client); + if (!names.includes('Envs')) { + return emptyInventory(options.profile, 'absent', 0, []); + } + const schema = columns(client); + const orderedSchema = [...schema].sort(); + let count: number; + try { + count = rowCount(client); + } catch (error) { + if (error instanceof LegacyEnvironmentInspectionError) throw error; + throw new LegacyEnvironmentInspectionError( + 'row count is unavailable', + error, + ); + } + if (!['id', 'name', 'value'].every((column) => schema.has(column))) { + return emptyInventory( + options.profile, + 'unsupported_schema', + count, + orderedSchema, + ); + } + const maximumRows = + options.profile === 'edge' + ? MAX_EDGE_LEGACY_ENVIRONMENT_ROWS + : MAX_STANDALONE_LEGACY_ENVIRONMENT_ROWS; + if (count > maximumRows) { + return emptyInventory( + options.profile, + 'budget_exceeded', + count, + orderedSchema, + ); + } + + const inventoryHash = createHash('sha256') + .update('qinglong3.legacy-environment-inventory.v1\0') + .update( + JSON.stringify({ profile: options.profile, schema: orderedSchema }), + ); + const groups = new Map(); + const invalidActiveGroupDigests = new Set(); + const disabledLimit = + options.profile === 'edge' + ? MAX_EDGE_LEGACY_DISABLED_ENVIRONMENTS + : MAX_STANDALONE_LEGACY_DISABLED_ENVIRONMENTS; + let rowOrdinal = 0; + let activeRowCount = 0; + let disabledRowCount = 0; + let manualRowCount = 0; + let preservationReadyCount = 0; + let activeValueBytes = 0; + + try { + for (const raw of client + .prepare(selectSql(schema)) + .iterate() as Iterable) { + rowOrdinal += 1; + const digestValue = sourceDigest(raw); + const rowReasons = reasons(raw); + let disposition: LegacyEnvironmentRowDisposition = 'manual_required'; + if (rowReasons.length === 0 && raw.status === 0) { + disposition = 'active_member'; + activeRowCount += 1; + } else if (rowReasons.length === 0 && raw.status === 1) { + disposition = 'preserve_disabled'; + disabledRowCount += 1; + preservationReadyCount += 1; + } else { + manualRowCount += 1; + if (raw.status === 0) activeRowCount += 1; + if (raw.status === 1) disabledRowCount += 1; + } + const inspection = Object.freeze({ + rowOrdinal, + sourceDigest: digestValue, + disposition, + reasons: rowReasons, + }); + options.visitRow?.(inspection); + inventoryHash.update('\0row\0').update(JSON.stringify(inspection)); + + if (raw.status !== 0) continue; + if ( + typeof raw.name !== 'string' || + !ENVIRONMENT_NAME.test(raw.name) || + raw.name.startsWith('QL3_') + ) { + invalidActiveGroupDigests.add( + digest( + 'qinglong3.legacy-environment-invalid-name.v1', + scalarEvidence(raw.name), + ), + ); + continue; + } + let group = groups.get(raw.name); + if (!group) { + group = { + name: raw.name, + rowDigests: [], + values: [], + valueBytes: 0, + valid: true, + }; + groups.set(raw.name, group); + } + group.rowDigests.push(digestValue); + if (rowReasons.length > 0 || typeof raw.value !== 'string') { + group.valid = false; + group.values.length = 0; + continue; + } + const valueBytes = Buffer.byteLength(raw.value, 'utf8'); + const separatorBytes = group.values.length === 0 ? 0 : 1; + group.valueBytes += valueBytes + separatorBytes; + activeValueBytes += valueBytes + separatorBytes; + if ( + group.valueBytes > MAX_LEGACY_ENVIRONMENT_VALUE_BYTES || + activeValueBytes > MAX_LEGACY_ENVIRONMENT_EFFECTIVE_BYTES + ) { + group.valid = false; + group.values.length = 0; + } else if (group.valid) { + group.values.push(raw.value); + } + } + } catch (error) { + if (error instanceof LegacyEnvironmentInspectionError) throw error; + throw new LegacyEnvironmentInspectionError( + 'rows cannot be inspected', + error, + ); + } + if (rowOrdinal !== count) { + throw new LegacyEnvironmentInspectionError('row count drifted'); + } + + const activeGroupCount = groups.size + invalidActiveGroupDigests.size; + const globalBudgetExceeded = + activeGroupCount > MAX_LEGACY_ENVIRONMENT_BINDINGS || + activeValueBytes > MAX_LEGACY_ENVIRONMENT_EFFECTIVE_BYTES || + preservationReadyCount > disabledLimit; + let bindingReadyCount = 0; + let manualGroupCount = invalidActiveGroupDigests.size; + if (!globalBudgetExceeded) { + for (const group of [...groups.values()].sort((left, right) => + left.name.localeCompare(right.name), + )) { + if (!group.valid || group.values.length !== group.rowDigests.length) { + manualGroupCount += 1; + continue; + } + const value = group.values.join('&'); + const base = Object.freeze({ + kind: 'active_binding' as const, + environmentName: group.name, + sourceRowCount: group.rowDigests.length, + sourceSetDigest: digest( + 'qinglong3.legacy-environment-source-set.v1', + group.rowDigests, + ), + }); + const candidate = Object.freeze({ + ...base, + value, + candidateDigest: candidateDigest(base, value), + }); + bindingReadyCount += 1; + options.visitCandidate?.(candidate); + inventoryHash.update('\0candidate\0').update(candidate.candidateDigest); + } + } else { + manualGroupCount = activeGroupCount; + bindingReadyCount = 0; + preservationReadyCount = 0; + } + + if (!globalBudgetExceeded && preservationReadyCount > 0) { + rowOrdinal = 0; + for (const raw of client + .prepare(selectSql(schema)) + .iterate() as Iterable) { + rowOrdinal += 1; + if (raw.status !== 1 || reasons(raw).length !== 0) continue; + const digestValue = sourceDigest(raw); + const base = Object.freeze({ + kind: 'disabled_preservation' as const, + environmentName: raw.name as string, + sourceRowOrdinal: rowOrdinal, + sourceDigest: digestValue, + }); + const value = raw.value as string; + const candidate = Object.freeze({ + ...base, + value, + candidateDigest: candidateDigest(base, value), + }); + options.visitCandidate?.(candidate); + inventoryHash.update('\0candidate\0').update(candidate.candidateDigest); + } + } + + const mutationReady = + !globalBudgetExceeded && manualRowCount === 0 && manualGroupCount === 0; + const payload = Object.freeze({ + schemaVersion: 1 as const, + kind: 'qinglong3-legacy-environment-inventory' as const, + profile: options.profile, + tableState: 'supported' as const, + rowCount: count, + activeRowCount, + disabledRowCount, + manualRowCount, + activeGroupCount, + bindingReadyCount, + preservationReadyCount, + manualGroupCount, + mutationReady, + }); + inventoryHash.update('\0summary\0').update(JSON.stringify(payload)); + return Object.freeze({ + ...payload, + inventoryDigest: inventoryHash.digest('hex'), + }); +} diff --git a/packages/ql3-local-admin/test/legacyEnvironmentInspection.test.cjs b/packages/ql3-local-admin/test/legacyEnvironmentInspection.test.cjs new file mode 100644 index 00000000..98abea5a --- /dev/null +++ b/packages/ql3-local-admin/test/legacyEnvironmentInspection.test.cjs @@ -0,0 +1,214 @@ +const assert = require('node:assert/strict'); +const { DatabaseSync } = require('node:sqlite'); +const { test } = require('node:test'); + +const { + MAX_EDGE_LEGACY_ENVIRONMENT_ROWS, + visitLegacyEnvironmentAdoption, +} = require('@qinglong/local-admin/reconciliation-secret-and-config-inspection'); + +function memoryDatabase(sql = '') { + const database = new DatabaseSync(':memory:'); + if (sql) database.exec(sql); + return database; +} + +function inspect(database, profile = 'edge') { + const rows = []; + const candidates = []; + const inventory = visitLegacyEnvironmentAdoption(database, { + profile, + visitRow: (row) => rows.push(row), + visitCandidate: (candidate) => candidates.push(candidate), + }); + return { inventory, rows, candidates }; +} + +test('treats an absent Envs table as a stable no-effect inventory', () => { + const database = memoryDatabase( + 'CREATE TABLE "Crontabs" (id INTEGER PRIMARY KEY)', + ); + try { + const first = inspect(database); + const second = inspect(database); + assert.equal(first.inventory.tableState, 'absent'); + assert.equal(first.inventory.mutationReady, true); + assert.equal(first.inventory.rowCount, 0); + assert.equal( + first.inventory.inventoryDigest, + second.inventory.inventoryDigest, + ); + assert.deepEqual(first.rows, []); + assert.deepEqual(first.candidates, []); + } finally { + database.close(); + } +}); + +test('reproduces legacy ordering and joins active values without exposing them in diagnostics', () => { + const database = memoryDatabase(` + CREATE TABLE "Envs" ( + id INTEGER PRIMARY KEY, + name TEXT, + value TEXT, + status INTEGER, + position REAL, + "isPinned" INTEGER, + "createdAt" TEXT + ); + INSERT INTO "Envs" VALUES + (1, 'TOKEN', 'later-value', 0, 10, 0, '2026-01-01'), + (2, 'TOKEN', 'pinned-value', 0, 1, 1, '2026-01-02'), + (3, 'PLAIN', 'plain-value', 0, 9, 0, '2026-01-03'), + (4, 'DISABLED', 'disabled-value', 1, 8, 0, '2026-01-04'); + `); + try { + const { inventory, rows, candidates } = inspect(database); + assert.deepEqual( + { + rowCount: inventory.rowCount, + activeRowCount: inventory.activeRowCount, + disabledRowCount: inventory.disabledRowCount, + activeGroupCount: inventory.activeGroupCount, + bindingReadyCount: inventory.bindingReadyCount, + preservationReadyCount: inventory.preservationReadyCount, + mutationReady: inventory.mutationReady, + }, + { + rowCount: 4, + activeRowCount: 3, + disabledRowCount: 1, + activeGroupCount: 2, + bindingReadyCount: 2, + preservationReadyCount: 1, + mutationReady: true, + }, + ); + assert.deepEqual( + candidates.map(({ kind, environmentName, value }) => ({ + kind, + environmentName, + value, + })), + [ + { + kind: 'active_binding', + environmentName: 'PLAIN', + value: 'plain-value', + }, + { + kind: 'active_binding', + environmentName: 'TOKEN', + value: 'pinned-value&later-value', + }, + { + kind: 'disabled_preservation', + environmentName: 'DISABLED', + value: 'disabled-value', + }, + ], + ); + const publicEvidence = JSON.stringify({ inventory, rows }); + for (const secret of [ + 'later-value', + 'pinned-value', + 'plain-value', + 'disabled-value', + 'TOKEN', + 'PLAIN', + 'DISABLED', + ]) { + assert.equal(publicEvidence.includes(secret), false); + } + assert.equal( + new Set(candidates.map((value) => value.candidateDigest)).size, + 3, + ); + } finally { + database.close(); + } +}); + +test('fails closed for malformed rows, reserved names and effective-value overflow', () => { + const oversized = 'x'.repeat(12 * 1024); + const database = memoryDatabase(` + CREATE TABLE "Envs" ( + id INTEGER PRIMARY KEY, + name TEXT, + value TEXT, + status INTEGER, + position REAL, + "isPinned" INTEGER, + "createdAt" TEXT + ); + `); + const insert = database.prepare( + 'INSERT INTO "Envs" VALUES (?, ?, ?, ?, ?, ?, ?)', + ); + insert.run(1, 'QL3_FORBIDDEN', 'secret-a', 0, 3, 0, '2026-01-01'); + insert.run(2, 'TOKEN', oversized, 0, 2, 0, '2026-01-02'); + insert.run(3, 'TOKEN', oversized, 0, 1, 0, '2026-01-03'); + insert.run(4, 'BROKEN', 'secret-b', 7, 0, 0, '2026-01-04'); + try { + const { inventory, rows, candidates } = inspect(database); + assert.equal(inventory.mutationReady, false); + assert.equal(inventory.manualRowCount, 2); + assert.equal(inventory.manualGroupCount, 2); + assert.equal(inventory.bindingReadyCount, 0); + assert.deepEqual(candidates, []); + assert.deepEqual( + rows.map(({ disposition, reasons }) => ({ disposition, reasons })), + [ + { disposition: 'manual_required', reasons: ['name_invalid'] }, + { disposition: 'active_member', reasons: [] }, + { disposition: 'active_member', reasons: [] }, + { disposition: 'manual_required', reasons: ['status_invalid'] }, + ], + ); + assert.equal( + JSON.stringify({ inventory, rows }).includes('secret-a'), + false, + ); + assert.equal( + JSON.stringify({ inventory, rows }).includes('secret-b'), + false, + ); + } finally { + database.close(); + } +}); + +test('rejects unsupported schemas and over-budget Edge tables without scanning rows', () => { + const unsupported = memoryDatabase( + 'CREATE TABLE "Envs" (id INTEGER PRIMARY KEY, name TEXT)', + ); + try { + const value = inspect(unsupported); + assert.equal(value.inventory.tableState, 'unsupported_schema'); + assert.equal(value.inventory.mutationReady, false); + } finally { + unsupported.close(); + } + + const overBudget = memoryDatabase(` + CREATE TABLE "Envs" (id INTEGER PRIMARY KEY, name TEXT, value TEXT); + WITH RECURSIVE rows(id) AS ( + SELECT 1 UNION ALL SELECT id + 1 FROM rows + WHERE id < ${MAX_EDGE_LEGACY_ENVIRONMENT_ROWS + 1} + ) + INSERT INTO "Envs" SELECT id, 'TOKEN_' || id, 'value' FROM rows; + `); + try { + const value = inspect(overBudget); + assert.equal(value.inventory.tableState, 'budget_exceeded'); + assert.equal( + value.inventory.rowCount, + MAX_EDGE_LEGACY_ENVIRONMENT_ROWS + 1, + ); + assert.equal(value.inventory.mutationReady, false); + assert.deepEqual(value.rows, []); + assert.deepEqual(value.candidates, []); + } finally { + overBudget.close(); + } +});