From 3c02ae2020e2714410b0fbcc12e7214c1d8a7375 Mon Sep 17 00:00:00 2001 From: whyour Date: Thu, 20 Aug 2026 18:24:52 +0800 Subject: [PATCH] feat(ql3): prove legacy rollback readiness --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 21 + ...474-bounded-legacy-core-readiness-proof.md | 119 ++++ docs/adr/README.md | 1 + docs/operations/ql3-local-deployment.md | 46 ++ .../src/deployment/cutover/instanceLineage.ts | 7 +- .../cutover/legacy-readiness/contract.ts | 198 +++++++ .../cutover/legacy-readiness/probe.ts | 538 ++++++++++++++++++ .../src/deployment/localDeployment.ts | 17 + .../src/deployment/localDeploymentCli.ts | 6 +- .../test/legacyReadinessProbe.test.cjs | 321 +++++++++++ ...ql3-service-manager-bridge-docker-gate.cjs | 8 + .../ql3-service-manager-bridge-live-actor.cjs | 91 ++- ...l3-service-manager-legacy-live-service.cjs | 44 ++ ...egacySystemReadinessCompatibility.test.cjs | 81 +++ test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 15 files changed, 1491 insertions(+), 11 deletions(-) create mode 100644 docs/adr/ADR-0474-bounded-legacy-core-readiness-proof.md create mode 100644 packages/ql3-local-owner-cli/src/deployment/cutover/legacy-readiness/contract.ts create mode 100644 packages/ql3-local-owner-cli/src/deployment/cutover/legacy-readiness/probe.ts create mode 100644 packages/ql3-local-owner-cli/test/legacyReadinessProbe.test.cjs create mode 100644 scripts/ql3-service-manager-legacy-live-service.cjs create mode 100644 test/back/ql3LegacySystemReadinessCompatibility.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 3fb72310..69bd7c26 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,27 @@ 最新增量证据（2026-08-20）： +- D-381/ADR-0474（已接受；OpenRC live actor 待镜像基础设施恢复后补跑）：把 service-manager adopted rollback 从仅证明 + init/process 的 `legacy_running` 推进到有界、可重放的 2.x core readiness。新增显式 Owner 私有命令 + `cutover-legacy-readiness-probe`，绑定 exact cutover/profile/instance/generation、activation、当前 head、legacy-running source + record、port 与精确 2.x version；只允许 `GET 127.0.0.1:/api/system`，没有 caller URL/header/credential、redirect、proxy + 或 keep-alive agent。每次请求最多 2 秒/32 KiB，Edge 总预算 30 秒/最多 60 次，Standalone 60 秒/最多 120 次；只有 HTTP/envelope + 成功、`isInitialized=true` 且 version 精确相等才 no-replace 发布 `0600` 收据并 CAS 到 `legacy_ready`，exact replay 发起零次网络 + 请求，所有 not-ready reason 保持 `legacy_running`。该状态只证明正确 2.x 版本的本机 HTTP core 与初始化，不宣称任务、订阅、 + provider、通知、Worker 或全部 API 健康。实现内聚在现有 Local Owner 的 `cutover/legacy-readiness/`，不新增 package、production + dependency、binary、daemon、listener、watcher、timer、数据库连接或部署对象。readiness 聚焦 `6/6`，生产 `/api/system` Router + 兼容门 `2/2`；Local Owner 全量 `187 total / 182 pass / 5 conditional skip / 0 fail`；backend 全量 + `1,525 total / 1,523 pass / 2 conditional skip / 0 fail`；18-package clean build/逐包测试单次退出 0。package boundary、Service + Bridge import、Edge import、Cluster dependency、Cluster/Worker deployment、Console 与 Console distribution 八项审计全部 + compatible/passed；workspace 保持 18 packages、`singleSourcePackages=[]`、`shallowSourcePackages=[]`，Local Owner 为 + `113 source / 112 nested / 1 root binary entry`。14 档 Local artifact audit 全部 compatible；基础 Edge/Standalone 保持 + `2,598,669 / 2,598,747` bytes、316 files、57 loaded modules，Adopted 为 `2,817,964 / 2,818,087` bytes、58 loaded modules， + Application+AI 为 `4,501,822 / 4,501,954` bytes，MCP 为 `7,324,601 / 7,324,709` bytes、227 loaded modules，证明 ceremony + 未进入低配运行闭包。systemd Docker live gate 已覆盖 root/non-root success，真实 `/api/system` 推进到 `legacy_ready` 并验证收据/head + exact replay；barrier-crash 保持 `manual_required` 且不执行 readiness。OpenRC 因 `node:24-alpine` 拉取卡在 Docker credential + helper 而未执行，不能宣称全组合门闭合；挂起拉取已终止。本切片不改变 PostgreSQL schema、ACL、repository、role、Pool、连接或 + failover 语义，因此不重跑且不重新占有 HA 证明。D-382 应扩展 System/Script/Open API/鉴权错误 envelope 兼容矩阵与真实升级/回滚 + rehearsal；OpenRC 门在镜像基础设施恢复后补跑。 - D-380/ADR-0473（已接受；OpenRC live actor 待镜像基础设施恢复后补跑）：完成 systemd/OpenRC service-manager rollback 的 安全 commit 协议。`rollback_prepared` 不再被误当作 root 授权；Owner 重新绑定当前 head、preparation、Application 与 legacy-silence commitment 原始摘要、descriptor 摘要后，才 no-replace 发布 `legacy_restart_requested`。短生命周期 root bridge diff --git a/docs/adr/ADR-0474-bounded-legacy-core-readiness-proof.md b/docs/adr/ADR-0474-bounded-legacy-core-readiness-proof.md new file mode 100644 index 00000000..8321f4dd --- /dev/null +++ b/docs/adr/ADR-0474-bounded-legacy-core-readiness-proof.md @@ -0,0 +1,119 @@ +# ADR-0474：有界 Legacy Core Readiness Proof + +- 状态：Accepted +- 日期：2026-08-20 +- 关联 RFC：QL-RFC-0001 D-64、D-274、D-275、D-380、D-381 +- 关联 ADR：ADR-0315、ADR-0471、ADR-0472、ADR-0473 + +## 背景 + +ADR-0473 已把 adopted systemd/OpenRC 回滚安全地收敛到 `legacy_running`，但该状态只证明 init/process running、 +target stopped 以及进程身份绑定。它不能证明 QingLong 2.x 已完成数据库初始化并能通过 HTTP 服务核心请求。把 +`legacy_running` 直接当作回滚完成，会在服务仍启动中、初始化失败或错误版本占用端口时产生假成功。 + +readiness 证明同时必须适配低配路由设备与集群节点：它不能引入常驻 watcher、连接池、队列或新部署对象，也不能把 +调用方提供的 URL、header、代理或重定向变成新的网络能力。编辑前 GitNexus 对 instance lineage 返回 MEDIUM：11 个累计影响、 +9 个直接 importer、0 条已识别 execution flow；Local Owner CLI、live actor、Docker gate 和 package boundary 返回 LOW，未发现 +HIGH/CRITICAL 风险。 + +## 决策 + +### 1. readiness 是显式、一次性 Owner ceremony + +新增私有命令文件操作： + +```text +ql3-local-deploy cutover-legacy-readiness-probe --command-file +``` + +命令必须绑定 exact cutover/profile/instance/generation、activation digest、当前 `legacy_running` head digest、产生该 head 的 +source record digest、legacy HTTP port、预期精确 2.x version 和请求时间。Owner 在任何网络请求前完成 instance head +compare-and-swap 复验；stale、漂移或非 `legacy_running|legacy_ready` 状态失败关闭。 + +该 ceremony 复用于 Docker、systemd 与 OpenRC，因为 readiness 所证明的是同一个 Owner instance lineage，而不是某个 init +manager 的私有事实。它仍是显式短生命周期进程，不自动挂接到 root bridge，也不在后台周期执行。 + +### 2. 网络能力固定且有界 + +生产探针只允许： + +- `GET http://127.0.0.1:/api/system`； +- `Accept: application/json` 与 `Connection: close`； +- 每次请求最多 2 秒、响应最多 32 KiB； +- Edge 总预算 30 秒、最多 60 次，Standalone 总预算 60 秒、最多 120 次，固定 500 ms 间隔； +- 不使用 keep-alive agent、不跟随 redirect、不接受 caller URL/path/header/credential/proxy。 + +这些边界使失败成本由 Profile 固定，内存消耗为常数；集群节点不会因此获得新的外部网络、Worker 或控制面权限，低配设备也 +不会新增常驻 RSS。 + +### 3. 只证明 2.x core readiness + +成功必须同时满足 HTTP 200、现行 JSON envelope `code=200`、`data.isInitialized=true`，以及 `data.version` 与命令绑定的 +精确 2.x version 相同。生产 Router 兼容测试直接运行 `/api/system`，证明当前 2.x 在已初始化与默认未初始化账户下保持该契约。 + +该结果称为 **core readiness**：它证明正确版本的本机 2.x HTTP core 已响应并完成初始化。它不证明任务执行、订阅、外部 provider、 +通知、脚本依赖、Cluster Worker 或全部业务 API 健康，因此不得命名为 full health。 + +### 4. 成功持久化，失败不推进 lineage + +成功后 Owner 以 no-replace、`0600` 发布 `legacy-readiness-gN.json`，收据绑定 prior head、legacy-running source record、endpoint、 +预期/观察 version、attempts 与观察时间，再把 instance head 唯一合法地从 `legacy_running` CAS 到 `legacy_ready`。 + +相同命令的 exact replay 读取并复验收据，返回相同 `legacy_ready` 事实且发起零次网络请求。`unavailable`、`http_rejected`、 +`response_too_large`、`response_invalid`、`not_initialized` 或 `version_mismatch` 都返回 `not_ready`，保持 head 为 +`legacy_running`，不写失败收据，也不自动重启任何服务。 + +### 5. 保持 package 与部署边界 + +实现内聚在现有 `@qinglong/local-owner-cli`： + +```text +deployment/cutover/legacy-readiness/ + contract.ts + probe.ts +``` + +不新增 workspace package、production dependency、binary、daemon、listener、timer、watcher、数据库连接、systemd/OpenRC unit、 +容器或 Kubernetes workload。用于 Docker live gate 的 HTTP 服务只是测试 fixture，不进入产品制品。 + +## 被否决方案 + +1. **把 `legacy_running` 当作健康**：进程存在不能证明 HTTP 与数据库初始化，拒绝。 +2. **探测 `/api/health`**：该端点的进程局部 gRPC 观察在多节点/Worker 语义下容易过度声明，不适合作为 2.x core 回滚证明。 +3. **接受 caller URL、header 或 `curl` 参数**：会扩大 SSRF、凭据与代理面，且无法形成可重放的固定契约，拒绝。 +4. **在 root bridge 内自动探测**：会混合 OS mutation 与 Owner lineage authority，并延长 root ceremony，拒绝。 +5. **后台 watcher 或无限重试**：会给路由设备增加常驻成本，也让失败无法有界，拒绝。 +6. **新建 readiness package**：两个同生命周期源码模块没有独立交付和依赖理由，拒绝。 + +## 升级与回退 + +`legacy_ready` 是 additive lineage state；新版本只在 exact `legacy_running` 后写入。升级前应记录实际 2.x version 与 port，并以 +Owner 私有命令文件传入。若回退到不认识 `legacy_ready` 的旧 3.0 孵化构建，应保留收据和 instance journal、停止自动 ceremony， +由 operator 审核后从受支持版本恢复；不得手工改写 head 伪造旧状态。 + +## 验收证据 + +- readiness 聚焦门 `6/6`，覆盖闭合 contract、成功/零网络重放、Edge 有界失败、version mismatch/stale head、真实固定回环和 + oversized/redirect 拒绝；生产 `/api/system` Router 兼容门 `2/2`。 +- Local Owner 全量 `187 total / 182 pass / 5 conditional skip / 0 fail`；backend 全量 + `1,525 total / 1,523 pass / 2 conditional skip / 0 fail`。 +- 18-package clean build 与逐包测试单次退出 0。package boundary、Service Bridge import、Edge import、Cluster dependency、 + Cluster/Worker deployment、Console 与 Console distribution 八项审计全部 compatible/passed；workspace 仍为 18 packages、 + `singleSourcePackages=[]`、`shallowSourcePackages=[]`，Local Owner 为 `113 source / 112 nested / 1 root binary entry`。 +- 14 档 Local artifact audit 全部 compatible。基础 Edge/Standalone 保持 `2,598,669 / 2,598,747` bytes、316 files、 + 57 loaded modules；Adopted 为 `2,817,964 / 2,818,087` bytes、58 loaded modules；Application+AI 为 + `4,501,822 / 4,501,954` bytes；MCP 为 `7,324,601 / 7,324,709` bytes、227 loaded modules。readiness ceremony + 没有进入基础运行闭包。 +- systemd Docker live gate 覆盖 root/non-root success，真实 `/api/system` 使 Owner 进入 `legacy_ready` 并完成零网络 exact replay； + root barrier-crash 保持 `manual_required`，不会错误执行 readiness。 +- OpenRC actor 因 `node:24-alpine` 拉取卡在本机 Docker credential helper 而未执行，所以本 ADR 不宣称 systemd/OpenRC 全组合门闭合。 +- 本阶段不修改 SQL、migration、PostgreSQL ACL/repository/role/Pool、连接或 failover 语义，因此不重跑且不重新占有 PostgreSQL HA 证明。 + +## 未完成 + +- 镜像基础设施恢复后的 OpenRC root/non-root success 与 barrier-crash live actor； +- 固定物理 Edge 的完整 rollback/readiness 证据； +- System、Script、Open API、鉴权与错误 envelope 的更完整 2.x 兼容矩阵； +- 真实 2.x 数据目录升级、Primary 双态和目标实例 rollback rehearsal。 + +本 ADR 关闭 `legacy_running` 到有界 `legacy_ready` 的 core readiness 证明，不代表 QingLong 3.0 升级/回退 Gate 已全部完成。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 450a2cc3..2830b637 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -477,6 +477,7 @@ | [ADR-0471](./ADR-0471-legacy-core-api-compatibility-baseline.md) | Legacy 核心执行 API 兼容基线 | Accepted | | [ADR-0472](./ADR-0472-service-manager-legacy-rollback-preparation.md) | Service Manager Legacy Rollback Preparation | Accepted | | [ADR-0473](./ADR-0473-service-manager-legacy-rollback-commit.md) | Service Manager Legacy Rollback Commit | Accepted（OpenRC live actor 待补） | +| [ADR-0474](./ADR-0474-bounded-legacy-core-readiness-proof.md) | 有界 Legacy Core Readiness Proof | Accepted（OpenRC live actor 待补） | ## 规则 diff --git a/docs/operations/ql3-local-deployment.md b/docs/operations/ql3-local-deployment.md index 953b2233..a758492a 100644 --- a/docs/operations/ql3-local-deployment.md +++ b/docs/operations/ql3-local-deployment.md @@ -454,6 +454,52 @@ barrier 后崩溃时原样重放 commit。重放只 inspect，绝不盲目重复 补写 exact outcome；若无法证明，则安全收敛为 `manual_required`。整个流程不删除/覆盖 target，不把 target 数据库写回 source。`reconciliation_required` 或 `manual_review` 必须停止在 2.5，等待独立数据恢复流程。 +### 2.7 Legacy core readiness proof + +`legacy_running` 只证明 legacy init/process running 与 target stopped。要把回滚实例推进为 core ready，必须创建独立的私有 +`0600` command file；不要把探针塞进 root bridge，也不要手工用可变 URL 的 `curl` 结果替代： + +```json +{ + "schemaVersion": 1, + "operation": "local.deployment.cutover.legacy-readiness-probe", + "options": { + "deploymentRoot": "/opt/qinglong3", + "allowRootService": false + }, + "request": { + "cutoverId": "cutover-20260820-01", + "profile": "edge", + "instanceId": "local-default", + "generation": 1, + "expectedActivationDigest": "REPLACE_WITH_64_HEX_ACTIVATION_DIGEST", + "expectedInstanceHeadDigest": "REPLACE_WITH_LEGACY_RUNNING_HEAD_DIGEST", + "expectedLegacyRunningRecordDigest": "REPLACE_WITH_LEGACY_RUNNING_SOURCE_RECORD_DIGEST", + "legacyHttpPort": 5700, + "expectedLegacyVersion": "2.21.0", + "requestedAtMs": 1787236200000 + } +} +``` + +```sh +ql3-local-deploy cutover-legacy-readiness-probe \ + --command-file /secure/operator/qinglong3-legacy-readiness.json +``` + +`expectedInstanceHeadDigest` 取 2.6 Owner consume 返回的 `legacy_running` head； +`expectedLegacyRunningRecordDigest` 取该 head 绑定的 source record digest。Docker 与 systemd/OpenRC 使用相同命令，因为它只操作 +共同的 Owner instance lineage。 + +探针固定请求 `GET http://127.0.0.1:/api/system`，不接受其他 host/path/header/credential，不跟随重定向； +每次请求最多 2 秒、响应最多 32 KiB。Edge 总预算 30 秒/最多 60 次，Standalone 总预算 60 秒/最多 120 次。 +只有 HTTP/envelope 成功、`isInitialized=true` 且 version 精确等于命令中的 2.x version 才会发布 +`legacy-readiness-gN.json` 并 CAS 到 `legacy_ready`。 + +`not_ready` 会给出 `unavailable|http_rejected|response_too_large|response_invalid|not_initialized|version_mismatch`，保持 +`legacy_running` 不变；处理原因后必须原样重放，不要换 cutover/generation 绕过证据链。成功命令的 exact replay 只读取既有收据， +不会再次发网络请求。`legacy_ready` 是 2.x HTTP core 与初始化证明，不代表任务、订阅、外部 provider、通知或 Cluster Worker 全健康。 + ## 3. systemd 命令中使用 `"kind": "systemd"`。成功后检查： diff --git a/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts b/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts index 64157f3b..05718bd7 100644 --- a/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts +++ b/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts @@ -26,6 +26,7 @@ export type LocalCutoverInstanceHeadState = | 'rollback_prepared' | 'legacy_restart_requested' | 'legacy_running' + | 'legacy_ready' | 'manual_required' | 'resolution_authorized'; @@ -156,6 +157,7 @@ function parseHead(value: unknown): Readonly { head.state !== 'rollback_prepared' && head.state !== 'legacy_restart_requested' && head.state !== 'legacy_running' && + head.state !== 'legacy_ready' && head.state !== 'manual_required' && head.state !== 'resolution_authorized') || !Number.isSafeInteger(head.generation) || @@ -321,6 +323,7 @@ export function advanceLocalCutoverInstanceHead( | 'rollback_prepared' | 'legacy_restart_requested' | 'legacy_running' + | 'legacy_ready' | 'manual_required', generation: number, sourceRecordDigest: string, @@ -356,7 +359,8 @@ export function advanceLocalCutoverInstanceHead( current.generation === generation && (current.state === 'rollback_prepared' || current.state === 'legacy_restart_requested' || - current.state === 'legacy_running') + current.state === 'legacy_running' || + current.state === 'legacy_ready') ) { return current; } @@ -374,6 +378,7 @@ export function advanceLocalCutoverInstanceHead( current.state === 'rollback_prepared') || (state === 'legacy_running' && current.state === 'legacy_restart_requested') || + (state === 'legacy_ready' && current.state === 'legacy_running') || (state === 'manual_required' && (current.state === 'legacy_stopped' || current.state === 'target_active' || diff --git a/packages/ql3-local-owner-cli/src/deployment/cutover/legacy-readiness/contract.ts b/packages/ql3-local-owner-cli/src/deployment/cutover/legacy-readiness/contract.ts new file mode 100644 index 00000000..06b93e69 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/cutover/legacy-readiness/contract.ts @@ -0,0 +1,198 @@ +import path from 'node:path'; + +import { + currentIdentity, + LocalDeploymentConfigurationError, + type LocalDeploymentProfile, +} from '../../foundation/contract'; + +const MAX_PATH_BYTES = 4_096; +const SAFE_PATH_PATTERN = /^\/[A-Za-z0-9._/@-]+$/; +const INSTANCE_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/; +const CUTOVER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const LEGACY_VERSION_PATTERN = + /^2\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/; + +export interface LocalDeploymentLegacyReadinessCommand { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.cutover.legacy-readiness-probe'; + readonly options: Readonly<{ + deploymentRoot: string; + allowRootService: boolean; + }>; + readonly request: Readonly<{ + cutoverId: string; + profile: LocalDeploymentProfile; + instanceId: string; + generation: number; + expectedActivationDigest: string; + expectedInstanceHeadDigest: string; + expectedLegacyRunningRecordDigest: string; + legacyHttpPort: number; + expectedLegacyVersion: string; + requestedAtMs: number; + }>; +} + +function object(value: unknown, label: string): Record { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + (Object.getPrototypeOf(value) !== Object.prototype && + Object.getPrototypeOf(value) !== null) + ) { + throw new LocalDeploymentConfigurationError(`${label} must be an object`); + } + return value as Record; +} + +function exact( + value: Record, + keys: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + throw new LocalDeploymentConfigurationError(`${label} shape is invalid`); + } +} + +function safeAbsolutePath(value: unknown, label: string): string { + if ( + typeof value !== 'string' || + !path.isAbsolute(value) || + path.normalize(value) !== value || + path.parse(value).root === value || + value.includes('\0') || + value.includes('//') || + !SAFE_PATH_PATTERN.test(value) || + Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES + ) { + throw new LocalDeploymentConfigurationError( + `${label} must be a supervisor-safe normalized absolute non-root path`, + ); + } + return value; +} + +function integer( + value: unknown, + minimum: number, + maximum: number, + label: string, +): number { + if ( + !Number.isSafeInteger(value) || + (value as number) < minimum || + (value as number) > maximum + ) { + throw new LocalDeploymentConfigurationError(`${label} is invalid`); + } + return value as number; +} + +export function normalizeLocalDeploymentLegacyReadinessCommand( + value: unknown, +): Readonly { + const command = object(value, 'command'); + exact( + command, + ['operation', 'options', 'request', 'schemaVersion'], + 'command', + ); + if ( + command.schemaVersion !== 1 || + command.operation !== 'local.deployment.cutover.legacy-readiness-probe' + ) { + throw new LocalDeploymentConfigurationError( + 'legacy readiness schemaVersion or operation is invalid', + ); + } + const identity = currentIdentity(); + const options = object(command.options, 'options'); + exact(options, ['allowRootService', 'deploymentRoot'], 'options'); + if ( + typeof options.allowRootService !== 'boolean' || + (identity.uid === 0) !== options.allowRootService + ) { + throw new LocalDeploymentConfigurationError( + 'allowRootService does not match the current identity', + ); + } + const request = object(command.request, 'request'); + exact( + request, + [ + 'cutoverId', + 'expectedActivationDigest', + 'expectedInstanceHeadDigest', + 'expectedLegacyRunningRecordDigest', + 'expectedLegacyVersion', + 'generation', + 'instanceId', + 'legacyHttpPort', + 'profile', + 'requestedAtMs', + ], + 'request', + ); + if ( + typeof request.cutoverId !== 'string' || + !CUTOVER_ID_PATTERN.test(request.cutoverId) || + (request.profile !== 'edge' && request.profile !== 'standalone') || + typeof request.instanceId !== 'string' || + !INSTANCE_ID_PATTERN.test(request.instanceId) || + typeof request.expectedActivationDigest !== 'string' || + !DIGEST_PATTERN.test(request.expectedActivationDigest) || + typeof request.expectedInstanceHeadDigest !== 'string' || + !DIGEST_PATTERN.test(request.expectedInstanceHeadDigest) || + typeof request.expectedLegacyRunningRecordDigest !== 'string' || + !DIGEST_PATTERN.test(request.expectedLegacyRunningRecordDigest) || + typeof request.expectedLegacyVersion !== 'string' || + !LEGACY_VERSION_PATTERN.test(request.expectedLegacyVersion) + ) { + throw new LocalDeploymentConfigurationError( + 'legacy readiness request identity is invalid', + ); + } + return Object.freeze({ + schemaVersion: 1 as const, + operation: 'local.deployment.cutover.legacy-readiness-probe' as const, + options: Object.freeze({ + deploymentRoot: safeAbsolutePath( + options.deploymentRoot, + 'deploymentRoot', + ), + allowRootService: options.allowRootService, + }), + request: Object.freeze({ + cutoverId: request.cutoverId, + profile: request.profile, + instanceId: request.instanceId, + generation: integer(request.generation, 1, 15, 'generation'), + expectedActivationDigest: request.expectedActivationDigest, + expectedInstanceHeadDigest: request.expectedInstanceHeadDigest, + expectedLegacyRunningRecordDigest: + request.expectedLegacyRunningRecordDigest, + legacyHttpPort: integer( + request.legacyHttpPort, + 1, + 65_535, + 'legacyHttpPort', + ), + expectedLegacyVersion: request.expectedLegacyVersion, + requestedAtMs: integer( + request.requestedAtMs, + 0, + Number.MAX_SAFE_INTEGER, + 'requestedAtMs', + ), + }), + }); +} diff --git a/packages/ql3-local-owner-cli/src/deployment/cutover/legacy-readiness/probe.ts b/packages/ql3-local-owner-cli/src/deployment/cutover/legacy-readiness/probe.ts new file mode 100644 index 00000000..667a069f --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/cutover/legacy-readiness/probe.ts @@ -0,0 +1,538 @@ +import fs from 'node:fs'; +import http from 'node:http'; +import path from 'node:path'; + +import { readPrivateLocalCommandFile } from '@qinglong/local-command-file'; + +import { + currentIdentity, + LocalDeploymentConfigurationError, +} from '../../foundation/contract'; +import { + preflightPublishedFile, + publishExactFile, +} from '../../foundation/files'; +import { + advanceLocalCutoverInstanceHead, + localCutoverInstanceDirectory, + readLocalCutoverInstanceHead, + type LocalCutoverInstanceHead, +} from '../instanceLineage'; +import { cutoverDigest } from '../targetEvidence'; +import { + normalizeLocalDeploymentLegacyReadinessCommand, + type LocalDeploymentLegacyReadinessCommand, +} from './contract'; + +const RECEIPT_SCHEMA = 'qinglong3-local-legacy-readiness-receipt'; +const LOOPBACK_HOST = '127.0.0.1'; +const SYSTEM_PATH = '/api/system'; +const MAX_RESPONSE_BYTES = 32 * 1024; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; + +export type LocalLegacyReadinessReason = + | 'unavailable' + | 'http_rejected' + | 'response_too_large' + | 'response_invalid' + | 'not_initialized' + | 'version_mismatch'; + +export type LocalLegacyReadinessObservation = + | Readonly<{ + ready: true; + initialized: true; + version: string; + }> + | Readonly<{ + ready: false; + reason: LocalLegacyReadinessReason; + }>; + +export interface LocalLegacyReadinessProbeInput { + readonly host: typeof LOOPBACK_HOST; + readonly port: number; + readonly path: typeof SYSTEM_PATH; + readonly timeoutMs: number; + readonly maxResponseBytes: number; +} + +export interface LocalDeploymentLegacyReadinessDependencies { + readonly probe?: ( + input: Readonly, + ) => Promise>; + readonly now?: () => number; + readonly wait?: (milliseconds: number) => Promise; +} + +interface LocalLegacyReadinessReceipt { + readonly schema: typeof RECEIPT_SCHEMA; + readonly schemaVersion: 1; + readonly state: 'legacy_ready'; + readonly cutoverId: string; + readonly profile: 'edge' | 'standalone'; + readonly instanceId: string; + readonly generation: number; + readonly activationDigest: string; + readonly previousHeadDigest: string; + readonly legacyRunningRecordDigest: string; + readonly endpoint: Readonly<{ + host: typeof LOOPBACK_HOST; + port: number; + path: typeof SYSTEM_PATH; + }>; + readonly expectedVersion: string; + readonly observedVersion: string; + readonly initialized: true; + readonly attempts: number; + readonly observedAtMs: number; + readonly receiptDigest: string; +} + +export type LocalDeploymentLegacyReadinessResult = + | Readonly<{ + schemaVersion: 1; + operation: LocalDeploymentLegacyReadinessCommand['operation']; + status: 'prepared' | 'existing'; + state: 'legacy_ready'; + cutoverId: string; + generation: number; + attempts: number; + receiptDigest: string; + instanceHeadDigest: string; + }> + | Readonly<{ + schemaVersion: 1; + operation: LocalDeploymentLegacyReadinessCommand['operation']; + status: 'not_ready'; + state: 'legacy_running'; + reason: LocalLegacyReadinessReason; + cutoverId: string; + generation: number; + attempts: number; + instanceHeadDigest: string; + }>; + +function configurationError(message: string, cause?: unknown): never { + throw new LocalDeploymentConfigurationError(message, { cause }); +} + +function object(value: unknown, label: string): Record { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + (Object.getPrototypeOf(value) !== Object.prototype && + Object.getPrototypeOf(value) !== null) + ) { + configurationError(`${label} must be an object`); + } + return value as Record; +} + +function exact( + value: Record, + keys: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + configurationError(`${label} shape is invalid`); + } +} + +function rejected(reason: LocalLegacyReadinessReason) { + return Object.freeze({ ready: false as const, reason }); +} + +export function probeLegacySystemEndpoint( + input: Readonly, +): Promise> { + return new Promise((resolve) => { + let settled = false; + const finish = (observation: Readonly) => { + if (settled) return; + settled = true; + resolve(observation); + }; + const request = http.request( + { + agent: false, + host: input.host, + port: input.port, + path: input.path, + method: 'GET', + headers: Object.freeze({ + accept: 'application/json', + connection: 'close', + }), + }, + (response) => { + if (response.statusCode !== 200) { + response.resume(); + finish(rejected('http_rejected')); + return; + } + const chunks: Buffer[] = []; + let size = 0; + response.on('data', (chunk: Buffer | string) => { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += bytes.byteLength; + if (size > input.maxResponseBytes) { + finish(rejected('response_too_large')); + response.destroy(); + return; + } + chunks.push(bytes); + }); + response.on('end', () => { + if (settled) return; + try { + const envelope = object( + JSON.parse(Buffer.concat(chunks).toString('utf8')), + 'legacy system response', + ); + const data = object(envelope.data, 'legacy system response data'); + if (envelope.code !== 200 || typeof data.version !== 'string') { + finish(rejected('response_invalid')); + return; + } + if (data.isInitialized !== true) { + finish(rejected('not_initialized')); + return; + } + finish( + Object.freeze({ + ready: true as const, + initialized: true as const, + version: data.version, + }), + ); + } catch { + finish(rejected('response_invalid')); + } + }); + response.on('error', () => finish(rejected('unavailable'))); + }, + ); + request.setTimeout(input.timeoutMs, () => request.destroy()); + request.on('error', () => finish(rejected('unavailable'))); + request.end(); + }); +} + +function receiptPath( + command: Readonly, +): string { + return path.join( + localCutoverInstanceDirectory( + command.options.deploymentRoot, + command.request.instanceId, + ), + `legacy-readiness-g${command.request.generation}.json`, + ); +} + +function receiptContents( + receipt: Readonly, +): string { + return `${JSON.stringify(receipt, null, 2)}\n`; +} + +function parseReceipt(value: unknown): Readonly { + const receipt = object(value, 'legacy readiness receipt'); + exact( + receipt, + [ + 'activationDigest', + 'attempts', + 'cutoverId', + 'endpoint', + 'expectedVersion', + 'generation', + 'initialized', + 'instanceId', + 'legacyRunningRecordDigest', + 'observedAtMs', + 'observedVersion', + 'previousHeadDigest', + 'profile', + 'receiptDigest', + 'schema', + 'schemaVersion', + 'state', + ], + 'legacy readiness receipt', + ); + const endpoint = object(receipt.endpoint, 'legacy readiness endpoint'); + exact(endpoint, ['host', 'path', 'port'], 'legacy readiness endpoint'); + const { receiptDigest, ...payload } = receipt; + if ( + receipt.schema !== RECEIPT_SCHEMA || + receipt.schemaVersion !== 1 || + receipt.state !== 'legacy_ready' || + typeof receipt.cutoverId !== 'string' || + (receipt.profile !== 'edge' && receipt.profile !== 'standalone') || + typeof receipt.instanceId !== 'string' || + !Number.isSafeInteger(receipt.generation) || + (receipt.generation as number) < 1 || + typeof receipt.activationDigest !== 'string' || + !DIGEST_PATTERN.test(receipt.activationDigest) || + typeof receipt.previousHeadDigest !== 'string' || + !DIGEST_PATTERN.test(receipt.previousHeadDigest) || + typeof receipt.legacyRunningRecordDigest !== 'string' || + !DIGEST_PATTERN.test(receipt.legacyRunningRecordDigest) || + endpoint.host !== LOOPBACK_HOST || + endpoint.path !== SYSTEM_PATH || + !Number.isSafeInteger(endpoint.port) || + (endpoint.port as number) < 1 || + (endpoint.port as number) > 65_535 || + typeof receipt.expectedVersion !== 'string' || + typeof receipt.observedVersion !== 'string' || + receipt.initialized !== true || + !Number.isSafeInteger(receipt.attempts) || + (receipt.attempts as number) < 1 || + !Number.isSafeInteger(receipt.observedAtMs) || + (receipt.observedAtMs as number) < 0 || + typeof receiptDigest !== 'string' || + !DIGEST_PATTERN.test(receiptDigest) || + cutoverDigest(payload) !== receiptDigest + ) { + configurationError('legacy readiness receipt drifted'); + } + return receipt as unknown as Readonly; +} + +function verifyReceipt( + command: Readonly, + receipt: Readonly, +): void { + if ( + receipt.cutoverId !== command.request.cutoverId || + receipt.profile !== command.request.profile || + receipt.instanceId !== command.request.instanceId || + receipt.generation !== command.request.generation || + receipt.activationDigest !== command.request.expectedActivationDigest || + receipt.previousHeadDigest !== command.request.expectedInstanceHeadDigest || + receipt.legacyRunningRecordDigest !== + command.request.expectedLegacyRunningRecordDigest || + receipt.endpoint.port !== command.request.legacyHttpPort || + receipt.expectedVersion !== command.request.expectedLegacyVersion || + receipt.observedVersion !== command.request.expectedLegacyVersion + ) { + configurationError('legacy readiness receipt does not match the command'); + } +} + +function verifyHead( + command: Readonly, + head: Readonly, +): void { + const replay = head.state === 'legacy_ready'; + if ( + head.profile !== command.request.profile || + head.cutoverId !== command.request.cutoverId || + head.instanceId !== command.request.instanceId || + head.generation !== command.request.generation || + head.activationDigest !== command.request.expectedActivationDigest || + (replay + ? head.previousHeadDigest !== command.request.expectedInstanceHeadDigest + : head.headDigest !== command.request.expectedInstanceHeadDigest) || + (!replay && + head.sourceRecordDigest !== + command.request.expectedLegacyRunningRecordDigest) || + (head.state !== 'legacy_running' && head.state !== 'legacy_ready') + ) { + configurationError( + 'legacy readiness command lost the instance head compare-and-swap', + ); + } +} + +function successfulResult( + command: Readonly, + status: 'prepared' | 'existing', + receipt: Readonly, + head: Readonly, +): Readonly { + return Object.freeze({ + schemaVersion: 1 as const, + operation: command.operation, + status, + state: 'legacy_ready' as const, + cutoverId: command.request.cutoverId, + generation: command.request.generation, + attempts: receipt.attempts, + receiptDigest: receipt.receiptDigest, + instanceHeadDigest: head.headDigest, + }); +} + +function completeExistingReceipt( + command: Readonly, + uid: number, + current: Readonly, +): Readonly | undefined { + const filePath = receiptPath(command); + if (!fs.existsSync(filePath)) return undefined; + const receipt = parseReceipt(readPrivateLocalCommandFile(filePath)); + verifyReceipt(command, receipt); + const head = + current.state === 'legacy_ready' + ? current + : advanceLocalCutoverInstanceHead( + command, + uid, + 'legacy_ready', + command.request.generation, + receipt.receiptDigest, + ); + if (head.sourceRecordDigest !== receipt.receiptDigest) { + configurationError('legacy-ready instance head is not receipt-bound'); + } + return successfulResult(command, 'existing', receipt, head); +} + +function policy(profile: 'edge' | 'standalone') { + return profile === 'edge' + ? Object.freeze({ + totalTimeoutMs: 30_000, + requestTimeoutMs: 2_000, + pollIntervalMs: 500, + maximumAttempts: 60, + }) + : Object.freeze({ + totalTimeoutMs: 60_000, + requestTimeoutMs: 2_000, + pollIntervalMs: 500, + maximumAttempts: 120, + }); +} + +export async function proveLocalDeploymentLegacyReadiness( + input: unknown, + dependencies: Readonly = {}, +): Promise> { + const command = normalizeLocalDeploymentLegacyReadinessCommand(input); + const identity = currentIdentity(); + const current = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + command.request.instanceId, + identity.uid, + ); + verifyHead(command, current); + const existing = completeExistingReceipt(command, identity.uid, current); + if (existing) return existing; + if (current.state !== 'legacy_running') { + configurationError('legacy-ready instance is missing its receipt'); + } + + const observe = dependencies.probe ?? probeLegacySystemEndpoint; + const now = dependencies.now ?? Date.now; + const wait = + dependencies.wait ?? + ((milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds))); + const limits = policy(command.request.profile); + const deadline = now() + limits.totalTimeoutMs; + let attempts = 0; + let lastReason: LocalLegacyReadinessReason = 'unavailable'; + while (attempts < limits.maximumAttempts && now() <= deadline) { + attempts += 1; + const remaining = Math.max(1, deadline - now()); + const observation = await observe( + Object.freeze({ + host: LOOPBACK_HOST, + port: command.request.legacyHttpPort, + path: SYSTEM_PATH, + timeoutMs: Math.min(limits.requestTimeoutMs, remaining), + maxResponseBytes: MAX_RESPONSE_BYTES, + }), + ); + if (observation.ready === true) { + if (observation.version !== command.request.expectedLegacyVersion) { + lastReason = 'version_mismatch'; + break; + } + const payload = Object.freeze({ + schema: RECEIPT_SCHEMA, + schemaVersion: 1 as const, + state: 'legacy_ready' as const, + cutoverId: command.request.cutoverId, + profile: command.request.profile, + instanceId: command.request.instanceId, + generation: command.request.generation, + activationDigest: command.request.expectedActivationDigest, + previousHeadDigest: command.request.expectedInstanceHeadDigest, + legacyRunningRecordDigest: + command.request.expectedLegacyRunningRecordDigest, + endpoint: Object.freeze({ + host: LOOPBACK_HOST, + port: command.request.legacyHttpPort, + path: SYSTEM_PATH, + }), + expectedVersion: command.request.expectedLegacyVersion, + observedVersion: observation.version, + initialized: true as const, + attempts, + observedAtMs: now(), + }); + const receipt = Object.freeze({ + ...payload, + receiptDigest: cutoverDigest(payload), + }); + const filePath = receiptPath(command); + const contents = receiptContents(receipt); + preflightPublishedFile( + filePath, + contents, + 0o600, + identity.uid, + 'legacy readiness receipt', + ); + const status = publishExactFile( + filePath, + contents, + 0o600, + identity.uid, + 'legacy readiness receipt', + ); + const head = advanceLocalCutoverInstanceHead( + command, + identity.uid, + 'legacy_ready', + command.request.generation, + receipt.receiptDigest, + ); + return successfulResult(command, status, receipt, head); + } + lastReason = observation.reason; + if (attempts >= limits.maximumAttempts || now() >= deadline) break; + await wait(Math.min(limits.pollIntervalMs, deadline - now())); + } + return Object.freeze({ + schemaVersion: 1 as const, + operation: command.operation, + status: 'not_ready' as const, + state: 'legacy_running' as const, + reason: lastReason, + cutoverId: command.request.cutoverId, + generation: command.request.generation, + attempts, + instanceHeadDigest: current.headDigest, + }); +} + +export function proveLocalDeploymentLegacyReadinessCommandFile( + filePath: string, + dependencies: Readonly = {}, +): Promise> { + return proveLocalDeploymentLegacyReadiness( + readPrivateLocalCommandFile(filePath), + dependencies, + ); +} diff --git a/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts b/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts index 731336a1..3f3ad3e4 100644 --- a/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts +++ b/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts @@ -69,6 +69,10 @@ import { runLocalDeploymentLegacyRollback, runLocalDeploymentLegacyRollbackCommandFile, } from './cutover/legacyRollback'; +import { + proveLocalDeploymentLegacyReadiness, + proveLocalDeploymentLegacyReadinessCommandFile, +} from './cutover/legacy-readiness/probe'; import { consumeLocalServiceManagerOutcome, consumeLocalServiceManagerOutcomeCommandFile, @@ -148,6 +152,17 @@ export { type LocalDeploymentLegacyRollbackResult, } from './cutover/legacyRollbackContract'; export { type LocalDeploymentLegacyRollbackDependencies } from './cutover/legacyRollback'; +export { + normalizeLocalDeploymentLegacyReadinessCommand, + type LocalDeploymentLegacyReadinessCommand, +} from './cutover/legacy-readiness/contract'; +export { + type LocalDeploymentLegacyReadinessDependencies, + type LocalDeploymentLegacyReadinessResult, + type LocalLegacyReadinessObservation, + type LocalLegacyReadinessProbeInput, + type LocalLegacyReadinessReason, +} from './cutover/legacy-readiness/probe'; export { EMPTY_RESOLUTION_DIGEST, normalizeLocalDeploymentCutoverManualCommand, @@ -177,6 +192,8 @@ export { runLocalDeploymentCutoverManualCommandFile, runLocalDeploymentLegacyRollback, runLocalDeploymentLegacyRollbackCommandFile, + proveLocalDeploymentLegacyReadiness, + proveLocalDeploymentLegacyReadinessCommandFile, runLocalDeploymentDockerTarget, runLocalDeploymentDockerTargetCommandFile, switchLocalDeploymentComposeRevision, diff --git a/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts b/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts index be0f361a..8f68a9df 100644 --- a/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts +++ b/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts @@ -14,6 +14,7 @@ import { prepareLocalServiceManagerIntentCommandFile, prepareLocalServiceManagerLegacyRollbackCommandFile, prepareLocalDeploymentCommandFile, + proveLocalDeploymentLegacyReadinessCommandFile, restoreLocalDeploymentComposeCommitCommandFile, restoreLocalDeploymentComposePrepareCommandFile, runLocalDeploymentCutoverManualCommandFile, @@ -25,7 +26,7 @@ import { } from './localDeployment'; const USAGE = - 'Usage: ql3-local-deploy --command-file /absolute/private-command.json'; + 'Usage: ql3-local-deploy --command-file /absolute/private-command.json'; async function main(argv: readonly string[]): Promise { if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) { @@ -48,6 +49,7 @@ async function main(argv: readonly string[]): Promise { argv[0] !== 'cutover-target-stop' && argv[0] !== 'cutover-legacy-rollback-prepare' && argv[0] !== 'cutover-legacy-rollback-commit' && + argv[0] !== 'cutover-legacy-readiness-probe' && argv[0] !== 'cutover-manual-diagnose' && argv[0] !== 'cutover-manual-resolution-prepare' && argv[0] !== 'cutover-manual-resolution-commit' && @@ -101,6 +103,8 @@ async function main(argv: readonly string[]): Promise { ? 'local.deployment.cutover.legacy-rollback-prepare' : 'local.deployment.cutover.legacy-rollback-commit', ) + : argv[0] === 'cutover-legacy-readiness-probe' + ? proveLocalDeploymentLegacyReadinessCommandFile(argv[2]!) : argv[0] === 'cutover-manual-diagnose' || argv[0] === 'cutover-manual-resolution-prepare' || argv[0] === 'cutover-manual-resolution-commit' diff --git a/packages/ql3-local-owner-cli/test/legacyReadinessProbe.test.cjs b/packages/ql3-local-owner-cli/test/legacyReadinessProbe.test.cjs new file mode 100644 index 00000000..ab93de71 --- /dev/null +++ b/packages/ql3-local-owner-cli/test/legacyReadinessProbe.test.cjs @@ -0,0 +1,321 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const os = require('node:os'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { + normalizeLocalDeploymentLegacyReadinessCommand, + proveLocalDeploymentLegacyReadiness, +} = require('../dist/deployment/localDeployment.js'); +const { + probeLegacySystemEndpoint, +} = require('../dist/deployment/cutover/legacy-readiness/probe.js'); +const { + advanceLocalCutoverInstanceHead, + claimLocalCutoverInstance, + readLocalCutoverInstanceHead, +} = require('../dist/deployment/cutover/instanceLineage.js'); + +function rootAcknowledgement() { + return typeof process.getuid === 'function' && process.getuid() === 0; +} + +function fixture(t, profile = 'edge', instanceId = `${profile}-legacy-1`) { + const deploymentRoot = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-legacy-readiness-')), + ); + fs.chmodSync(deploymentRoot, 0o700); + fs.mkdirSync(path.join(deploymentRoot, 'service'), { mode: 0o700 }); + t.after(() => fs.rmSync(deploymentRoot, { recursive: true, force: true })); + const identity = { + options: { deploymentRoot }, + request: { + cutoverId: `cutover-${instanceId}`, + profile, + instanceId, + expectedActivationDigest: 'a'.repeat(64), + requestedAtMs: 1_787_200_000_000, + }, + }; + const uid = process.getuid(); + claimLocalCutoverInstance(identity, uid, '0'.repeat(64)); + const transitions = [ + ['legacy_stopped', 0, '1'.repeat(64)], + ['target_active', 1, '2'.repeat(64)], + ['target_stopped', 1, '3'.repeat(64)], + ['rollback_prepared', 1, '4'.repeat(64)], + ['legacy_restart_requested', 1, '5'.repeat(64)], + ['legacy_running', 1, '6'.repeat(64)], + ]; + let head; + for (const [state, generation, sourceDigest] of transitions) { + head = advanceLocalCutoverInstanceHead( + identity, + uid, + state, + generation, + sourceDigest, + ); + } + const command = { + schemaVersion: 1, + operation: 'local.deployment.cutover.legacy-readiness-probe', + options: { + deploymentRoot, + allowRootService: rootAcknowledgement(), + }, + request: { + cutoverId: identity.request.cutoverId, + profile, + instanceId, + generation: 1, + expectedActivationDigest: identity.request.expectedActivationDigest, + expectedInstanceHeadDigest: head.headDigest, + expectedLegacyRunningRecordDigest: '6'.repeat(64), + legacyHttpPort: 5700, + expectedLegacyVersion: '2.21.0', + requestedAtMs: 1_787_200_030_000, + }, + }; + return { command, deploymentRoot, head, uid }; +} + +test('normalizes a closed legacy readiness command', (t) => { + const state = fixture(t); + const normalized = normalizeLocalDeploymentLegacyReadinessCommand( + state.command, + ); + assert.equal(normalized.request.legacyHttpPort, 5700); + assert.equal(normalized.request.expectedLegacyVersion, '2.21.0'); + assert.throws( + () => + normalizeLocalDeploymentLegacyReadinessCommand({ + ...state.command, + request: { ...state.command.request, endpoint: 'http://example.com' }, + }), + /request shape is invalid/, + ); + assert.throws( + () => + normalizeLocalDeploymentLegacyReadinessCommand({ + ...state.command, + request: { ...state.command.request, expectedLegacyVersion: '3.0.0' }, + }), + /request identity is invalid/, + ); + assert.throws( + () => + normalizeLocalDeploymentLegacyReadinessCommand({ + ...state.command, + request: { ...state.command.request, legacyHttpPort: 65_536 }, + }), + /legacyHttpPort is invalid/, + ); +}); + +test('persists a legacy-ready receipt and replays without network authority', async (t) => { + const state = fixture(t); + let attempts = 0; + let clock = state.command.request.requestedAtMs; + const result = await proveLocalDeploymentLegacyReadiness(state.command, { + now: () => clock, + wait: async (milliseconds) => { + clock += milliseconds; + }, + async probe(input) { + attempts += 1; + assert.deepEqual( + { host: input.host, port: input.port, path: input.path }, + { host: '127.0.0.1', port: 5700, path: '/api/system' }, + ); + return attempts < 3 + ? { ready: false, reason: 'not_initialized' } + : { ready: true, initialized: true, version: '2.21.0' }; + }, + }); + assert.equal(result.status, 'prepared'); + assert.equal(result.state, 'legacy_ready'); + assert.equal(result.attempts, 3); + assert.match(result.receiptDigest, /^[0-9a-f]{64}$/); + const receiptPath = path.join( + state.deploymentRoot, + 'service', + 'cutover-instances', + state.command.request.instanceId, + 'legacy-readiness-g1.json', + ); + const receipt = JSON.parse(fs.readFileSync(receiptPath, 'utf8')); + assert.equal(receipt.state, 'legacy_ready'); + assert.equal(receipt.endpoint.host, '127.0.0.1'); + assert.equal(receipt.endpoint.path, '/api/system'); + assert.equal(receipt.observedVersion, '2.21.0'); + assert.equal(fs.statSync(receiptPath).mode & 0o777, 0o600); + const head = readLocalCutoverInstanceHead( + state.deploymentRoot, + state.command.request.instanceId, + state.uid, + ); + assert.equal(head.state, 'legacy_ready'); + assert.equal(head.previousHeadDigest, state.head.headDigest); + assert.equal(head.sourceRecordDigest, result.receiptDigest); + + const replay = await proveLocalDeploymentLegacyReadiness(state.command, { + async probe() { + throw new Error('exact replay must not open loopback HTTP authority'); + }, + }); + assert.equal(replay.status, 'existing'); + assert.equal(replay.receiptDigest, result.receiptDigest); + assert.equal(replay.instanceHeadDigest, result.instanceHeadDigest); +}); + +test('bounds an unavailable edge probe without mutating lineage', async (t) => { + const state = fixture(t); + let clock = state.command.request.requestedAtMs; + let calls = 0; + const result = await proveLocalDeploymentLegacyReadiness(state.command, { + now: () => clock, + wait: async (milliseconds) => { + clock += milliseconds; + }, + async probe() { + calls += 1; + return { ready: false, reason: 'unavailable' }; + }, + }); + assert.equal(result.status, 'not_ready'); + assert.equal(result.state, 'legacy_running'); + assert.equal(result.reason, 'unavailable'); + assert.equal(result.attempts, 60); + assert.equal(calls, 60); + assert.equal( + readLocalCutoverInstanceHead( + state.deploymentRoot, + state.command.request.instanceId, + state.uid, + ).headDigest, + state.head.headDigest, + ); + assert.equal( + fs.existsSync( + path.join( + state.deploymentRoot, + 'service', + 'cutover-instances', + state.command.request.instanceId, + 'legacy-readiness-g1.json', + ), + ), + false, + ); +}); + +test('fails a version mismatch and stale head before durable mutation', async (t) => { + const state = fixture(t); + const mismatch = await proveLocalDeploymentLegacyReadiness(state.command, { + async probe() { + return { ready: true, initialized: true, version: '2.20.0' }; + }, + }); + assert.equal(mismatch.status, 'not_ready'); + assert.equal(mismatch.reason, 'version_mismatch'); + assert.equal(mismatch.attempts, 1); + let calls = 0; + await assert.rejects( + proveLocalDeploymentLegacyReadiness( + { + ...state.command, + request: { + ...state.command.request, + expectedInstanceHeadDigest: 'f'.repeat(64), + }, + }, + { + async probe() { + calls += 1; + return { ready: true, initialized: true, version: '2.21.0' }; + }, + }, + ), + /lost the instance head compare-and-swap/, + ); + assert.equal(calls, 0); +}); + +test('probes only the fixed loopback system endpoint with bounded parsing', async (t) => { + const paths = []; + const server = http.createServer((request, response) => { + paths.push(request.url); + response.setHeader('content-type', 'application/json'); + response.end( + JSON.stringify({ + code: 200, + data: { isInitialized: true, version: '2.21.0' }, + }), + ); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + t.after(() => new Promise((resolve) => server.close(resolve))); + const address = server.address(); + const result = await probeLegacySystemEndpoint({ + host: '127.0.0.1', + port: address.port, + path: '/api/system', + timeoutMs: 2_000, + maxResponseBytes: 32 * 1024, + }); + assert.deepEqual(result, { + ready: true, + initialized: true, + version: '2.21.0', + }); + assert.deepEqual(paths, ['/api/system']); +}); + +test('rejects oversized and redirect responses without following them', async (t) => { + let redirected = false; + const server = http.createServer((request, response) => { + if (request.url === '/redirected') { + redirected = true; + response.end('unexpected'); + return; + } + if (request.url === '/large') { + response.end('x'.repeat(2_048)); + return; + } + response.writeHead(302, { location: '/redirected' }); + response.end(); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + t.after(() => new Promise((resolve) => server.close(resolve))); + const address = server.address(); + const redirect = await probeLegacySystemEndpoint({ + host: '127.0.0.1', + port: address.port, + path: '/api/system', + timeoutMs: 2_000, + maxResponseBytes: 1_024, + }); + assert.deepEqual(redirect, { ready: false, reason: 'http_rejected' }); + assert.equal(redirected, false); + const large = await probeLegacySystemEndpoint({ + host: '127.0.0.1', + port: address.port, + path: '/large', + timeoutMs: 2_000, + maxResponseBytes: 1_024, + }); + assert.deepEqual(large, { + ready: false, + reason: 'response_too_large', + }); +}); diff --git a/scripts/ql3-service-manager-bridge-docker-gate.cjs b/scripts/ql3-service-manager-bridge-docker-gate.cjs index 18549074..871cead0 100644 --- a/scripts/ql3-service-manager-bridge-docker-gate.cjs +++ b/scripts/ql3-service-manager-bridge-docker-gate.cjs @@ -95,6 +95,8 @@ function actorReport(name, kind, identityMode, scenario = 'success') { 'adoptedLegacyStarted', 'adoptedLegacyProcessIdentity', 'adoptedLegacyResponseLossInspected', + 'adoptedLegacyReady', + 'adoptedLegacyReadinessReplay', ] : [ 'adoptedLegacyBarrierCrash', @@ -258,6 +260,12 @@ function main() { adoptedLegacyResponseLossInspected: successReports.every( (report) => report.gates.adoptedLegacyResponseLossInspected, ), + adoptedLegacyReady: successReports.every( + (report) => report.gates.adoptedLegacyReady, + ), + adoptedLegacyReadinessReplay: successReports.every( + (report) => report.gates.adoptedLegacyReadinessReplay, + ), adoptedLegacyBarrierCrash: crashReports.length === 2, adoptedLegacyInspectOnlyConvergence: crashReports.every( (report) => report.gates.adoptedLegacyInspectOnlyConvergence, diff --git a/scripts/ql3-service-manager-bridge-live-actor.cjs b/scripts/ql3-service-manager-bridge-live-actor.cjs index c08c6c0d..90066522 100644 --- a/scripts/ql3-service-manager-bridge-live-actor.cjs +++ b/scripts/ql3-service-manager-bridge-live-actor.cjs @@ -136,7 +136,10 @@ function descriptor(kind, uid, gid, root, adopted = false) { ].join('\n'); } -function legacyDescriptor(kind, uid, gid) { +function legacyDescriptor(kind, uid, gid, port) { + const nodeExecutable = fs.realpathSync(process.execPath); + const commandArgs = + `/workspace/scripts/ql3-service-manager-legacy-live-service.cjs ${port}`; if (kind === 'systemd') { return [ '[Unit]', @@ -146,7 +149,7 @@ function legacyDescriptor(kind, uid, gid) { 'Type=simple', `User=${uid}`, `Group=${gid}`, - 'ExecStart=/bin/sleep 300', + `ExecStart=${nodeExecutable} ${commandArgs}`, 'Restart=no', '', '[Install]', @@ -158,8 +161,8 @@ function legacyDescriptor(kind, uid, gid) { '#!/sbin/openrc-run', 'name="qinglong"', 'description="QingLong 2 legacy rollback live gate"', - 'command="/bin/sleep"', - 'command_args="300"', + `command="${nodeExecutable}"`, + `command_args="${commandArgs}"`, `command_user="${uid}:${gid}"`, 'supervisor="supervise-daemon"', 'retry="TERM/5/KILL/2"', @@ -532,7 +535,7 @@ function authorizeAdoptedRollback(root, adopted, prepared, uid, gid) { }; } -function installLegacyService(kind, managerOptions, uid, gid) { +function installLegacyService(kind, managerOptions, uid, gid, port) { const descriptorPath = kind === 'systemd' ? '/etc/systemd/system/qinglong.service' @@ -540,7 +543,7 @@ function installLegacyService(kind, managerOptions, uid, gid) { fs.rmSync(descriptorPath, { force: true }); writePrivate( descriptorPath, - legacyDescriptor(kind, uid, gid), + legacyDescriptor(kind, uid, gid, port), kind === 'systemd' ? 0o644 : 0o755, 0, 0, @@ -622,6 +625,54 @@ function executeLegacyRollback( return { commandPath, result, replay, consumed, outcome }; } +function proveLegacyReadiness( + root, + adopted, + consumed, + uid, + gid, + legacyHttpPort, +) { + const command = { + schemaVersion: 1, + operation: 'local.deployment.cutover.legacy-readiness-probe', + options: { deploymentRoot: root, allowRootService: uid === 0 }, + request: { + cutoverId: adopted.cutoverId, + profile: 'edge', + instanceId: adopted.instanceId, + generation: 1, + expectedActivationDigest: adopted.activationDigest, + expectedInstanceHeadDigest: consumed.instanceHeadDigest, + expectedLegacyRunningRecordDigest: consumed.completionDigest, + legacyHttpPort, + expectedLegacyVersion: '2.21.0', + requestedAtMs: Date.now(), + }, + }; + const commandPath = path.join(root, 'owner-legacy-readiness-probe.json'); + writePrivate( + commandPath, + `${JSON.stringify(command, null, 2)}\n`, + 0o600, + uid, + gid, + ); + const result = ownerCli( + uid, + gid, + 'cutover-legacy-readiness-probe', + commandPath, + ); + const replay = ownerCli( + uid, + gid, + 'cutover-legacy-readiness-probe', + commandPath, + ); + return { commandPath, result, replay }; +} + function executeLegacyRollbackBarrierCrash( root, controllerRoot, @@ -830,6 +881,7 @@ async function main(argv) { } const uid = identityMode === 'root' ? 0 : NON_ROOT_SERVICE_UID; const gid = uid; + const legacyHttpPort = 15_700; const root = `/var/lib/ql3-service-manager-${kind}-${identityMode}`; const controllerRoot = `/var/lib/ql3-service-bridge-${kind}-${identityMode}`; fs.rmSync(root, { force: true, recursive: true }); @@ -1035,7 +1087,7 @@ async function main(argv) { ) { fail('adopted rollback authorization did not converge exactly'); } - installLegacyService(kind, managerOptions, uid, gid); + installLegacyService(kind, managerOptions, uid, gid, legacyHttpPort); const adoptedLegacyStarted = scenario === 'success' ? executeLegacyRollback( @@ -1058,6 +1110,7 @@ async function main(argv) { uid, gid, ); + let adoptedLegacyReadiness; if (scenario === 'success') { if ( adoptedLegacyStarted.result.state !== 'legacy_running' || @@ -1078,6 +1131,26 @@ async function main(argv) { if (legacyUid !== uid) { fail(`legacy service process UID drifted: ${legacyUid} != ${uid}`); } + adoptedLegacyReadiness = proveLegacyReadiness( + root, + adopted, + adoptedLegacyStarted.consumed, + uid, + gid, + legacyHttpPort, + ); + if ( + adoptedLegacyReadiness.result.status !== 'prepared' || + adoptedLegacyReadiness.result.state !== 'legacy_ready' || + adoptedLegacyReadiness.replay.status !== 'existing' || + adoptedLegacyReadiness.replay.state !== 'legacy_ready' || + adoptedLegacyReadiness.replay.receiptDigest !== + adoptedLegacyReadiness.result.receiptDigest || + adoptedLegacyReadiness.replay.instanceHeadDigest !== + adoptedLegacyReadiness.result.instanceHeadDigest + ) { + fail('legacy readiness proof did not converge exactly'); + } } else if ( adoptedLegacyStarted.result.state !== 'manual_required' || adoptedLegacyStarted.result.status !== 'prepared' || @@ -1116,6 +1189,8 @@ async function main(argv) { legacyStarted: adoptedLegacyStarted.result, legacyStartReplay: adoptedLegacyStarted.replay, legacyConsumed: adoptedLegacyStarted.consumed, + legacyReadiness: adoptedLegacyReadiness?.result ?? null, + legacyReadinessReplay: adoptedLegacyReadiness?.replay ?? null, }, gates: { rootCommandFile: true, @@ -1140,6 +1215,8 @@ async function main(argv) { adoptedTargetRemainedStopped: true, adoptedLegacyProcessIdentity: scenario === 'success', adoptedLegacyResponseLossInspected: scenario === 'success', + adoptedLegacyReady: scenario === 'success', + adoptedLegacyReadinessReplay: scenario === 'success', adoptedLegacyBarrierCrash: scenario === 'barrier-crash', adoptedLegacyInspectOnlyConvergence: scenario === 'barrier-crash', adoptedLegacyRemainedStopped: scenario === 'barrier-crash', diff --git a/scripts/ql3-service-manager-legacy-live-service.cjs b/scripts/ql3-service-manager-legacy-live-service.cjs new file mode 100644 index 00000000..e79a1624 --- /dev/null +++ b/scripts/ql3-service-manager-legacy-live-service.cjs @@ -0,0 +1,44 @@ +#!/usr/bin/env node + +'use strict'; + +const http = require('node:http'); + +const port = Number(process.argv[2]); +if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) { + process.stderr.write('legacy live service port is invalid\n'); + process.exit(64); +} + +const server = http.createServer((request, response) => { + if (request.method !== 'GET' || request.url !== '/api/system') { + response.writeHead(404, { connection: 'close' }); + response.end(); + return; + } + const body = JSON.stringify({ + code: 200, + data: { + isInitialized: true, + version: '2.21.0', + publishTime: 1_787_200_000, + branch: 'next-live-gate', + changeLog: '', + changeLogLink: '', + }, + }); + response.writeHead(200, { + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(body), + connection: 'close', + }); + response.end(body); +}); + +function shutdown() { + server.close(() => process.exit(0)); +} + +process.on('SIGINT', shutdown); +process.on('SIGTERM', shutdown); +server.listen(port, '127.0.0.1'); diff --git a/test/back/ql3LegacySystemReadinessCompatibility.test.cjs b/test/back/ql3LegacySystemReadinessCompatibility.test.cjs new file mode 100644 index 00000000..5fdeef82 --- /dev/null +++ b/test/back/ql3LegacySystemReadinessCompatibility.test.cjs @@ -0,0 +1,81 @@ +require('ts-node/register/transpile-only'); +require('reflect-metadata'); + +const assert = require('node:assert/strict'); +const http = require('node:http'); +const { after, before, test } = require('node:test'); +const express = require('express'); +const { Container } = require('typedi'); + +const config = require('../../back/config').default; +const UserService = require('../../back/services/user').default; +const registerSystemRoutes = require('../../back/api/system').default; + +let authInfo = { + username: 'operator', + password: 'changed', + token: 'redacted', +}; +let origin; +let server; + +const logger = { + debug() {}, + error() {}, + info() {}, + warn() {}, +}; + +before(async () => { + Container.set('logger', logger); + Container.set(UserService, { + async getAuthInfo() { + return authInfo; + }, + }); + const app = express(); + app.set('case sensitive routing', true); + app.set('strict routing', true); + const api = express.Router(); + registerSystemRoutes(api); + app.use('/api', api); + server = http.createServer(app); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + const address = server.address(); + origin = `http://127.0.0.1:${address.port}`; +}); + +after(async () => { + Container.remove(UserService); + Container.remove('logger'); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); +}); + +test('preserves the public 2.x system readiness envelope', async () => { + const response = await fetch(`${origin}/api/system`); + const body = await response.json(); + assert.equal(response.status, 200); + assert.equal(body.code, 200); + assert.equal(body.data.isInitialized, true); + assert.equal(body.data.version, '2.21.0'); + assert.equal(body.data.branch, process.env.QL_BRANCH || 'master'); + assert.equal(Number.isSafeInteger(body.data.publishTime), true); + assert.equal(typeof body.data.changeLog, 'string'); + assert.equal(typeof body.data.changeLogLink, 'string'); + assert.equal(config.apiWhiteList.includes('/api/system'), true); +}); + +test('keeps default admin credentials distinguishable from readiness', async () => { + authInfo = { username: 'admin', password: 'admin' }; + const response = await fetch(`${origin}/api/system`); + const body = await response.json(); + assert.equal(response.status, 200); + assert.equal(body.code, 200); + assert.equal(body.data.isInitialized, false); + assert.equal(body.data.version, '2.21.0'); +}); diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index 928ab5f3..06f37e3b 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -207,10 +207,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: localOwnerCli.rootSourceFileRoles, }, { - sourceFiles: 111, + sourceFiles: 113, rootSourceFiles: 1, rootSourceLines: 50, - nestedSourceFiles: 110, + nestedSourceFiles: 112, rootSourceFileRoles: { 'cli.ts': 'binary_entry' }, }, );