From 2cd6ea1d554b8873cb6c6aa102b7126b9915dfb1 Mon Sep 17 00:00:00 2001 From: whyour Date: Thu, 20 Aug 2026 17:40:29 +0800 Subject: [PATCH] feat(ql3): commit service manager legacy rollback --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 20 + ...-service-manager-legacy-rollback-commit.md | 115 +++ docs/adr/README.md | 1 + .../src/deployment/localDeployment.ts | 36 +- .../src/deployment/localDeploymentCli.ts | 10 +- .../service-manager/legacy-rollback/bridge.ts | 881 ++++++++++++++++++ .../legacy-rollback/consumer.ts | 814 ++++++++++++++++ .../legacy-rollback/contract.ts | 575 ++++++++++++ .../preparation.ts} | 89 +- .../service-manager/serviceBridgeCli.ts | 15 +- .../test/serviceCutoverConsumer.test.cjs | 217 ++++- .../serviceLegacyRollbackContract.test.cjs | 144 +++ scripts/ql3-service-bridge-import-audit.cjs | 27 +- ...ql3-service-manager-bridge-docker-gate.cjs | 84 +- .../ql3-service-manager-bridge-live-actor.cjs | 412 +++++++- test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 16 files changed, 3382 insertions(+), 62 deletions(-) create mode 100644 docs/adr/ADR-0473-service-manager-legacy-rollback-commit.md create mode 100644 packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/bridge.ts create mode 100644 packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/consumer.ts create mode 100644 packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/contract.ts rename packages/ql3-local-owner-cli/src/deployment/service-manager/{serviceLegacyRollback.ts => legacy-rollback/preparation.ts} (90%) create mode 100644 packages/ql3-local-owner-cli/test/serviceLegacyRollbackContract.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index f59c0e64..3fb72310 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,26 @@ 最新增量证据（2026-08-20）： +- D-380/ADR-0473（已接受；OpenRC live actor 待镜像基础设施恢复后补跑）：完成 systemd/OpenRC service-manager rollback 的 + 安全 commit 协议。`rollback_prepared` 不再被误当作 root 授权；Owner 重新绑定当前 head、preparation、Application 与 + legacy-silence commitment 原始摘要、descriptor 摘要后，才 no-replace 发布 `legacy_restart_requested`。短生命周期 root bridge + 只允许固定 `qinglong`/`qinglong3`、固定 descriptor 与固定 systemd/OpenRC argv，启动前证明双服务 inactive，并在任何 start 前写 + durable barrier；barrier 后所有重放只能 inspect，响应丢失可按 legacy active/target inactive 与进程身份收敛，无法证明则由 Owner + 消费为 `manual_required`，永不盲目二次 start。Owner 是 instance head 的唯一终态写者，成功 CAS 到 `legacy_running` 并支持 + exact replay；该状态只证明 init/process running 与 target stopped，不宣称 2.x 业务健康。实现仍在现有 Local Owner package，且将 + 回滚四模块内聚到 `service-manager/legacy-rollback/`，没有新 package、production dependency、binary、daemon、watcher、timer、 + socket、数据库连接或部署对象。聚焦门 `16/16`；Local Owner 全量 + `181 total / 176 pass / 5 conditional skip / 0 fail`；backend 全量 + `1,523 total / 1,521 pass / 2 conditional skip / 0 fail`；18-package clean build/逐包测试单次退出 0。package boundary、 + Service Bridge import、Edge import、Cluster dependency、Cluster/Worker deployment、Console 与 Console distribution 八项审计 + 全部 compatible/passed；workspace 仍为 18 packages、`singleSourcePackages=[]`、`shallowSourcePackages=[]`，Local Owner 为 + `111 source / 110 nested / 1 root binary entry`。14 档 Local artifact audit 全部 compatible；基础 Edge/Standalone 保持 + `2,598,669 / 2,598,747` bytes、316 files、57 loaded modules，Adopted 为 `2,817,964 / 2,818,087` bytes、58 loaded modules， + Application+AI 为 `4,501,822 / 4,501,954` bytes，MCP 为 `7,324,601 / 7,324,709` bytes。systemd live actor 已覆盖 + root/non-root success、真实 manager start、response-loss inspect convergence、Owner consume/replay；root barrier-crash 证明没有 + 第二次 start，双服务均 inactive 并收敛为 manual。OpenRC 因本机缺少基础镜像且 `node:24-alpine` credential helper 挂起未执行， + 不能宣称全组合门已闭合；临时容器/镜像/tag 已清理。本切片不改变 PostgreSQL schema、ACL、repository、role、Pool、连接或 + failover 语义，因此不重跑且不重新占有 HA 证明。D-381 应补 OpenRC 证据与 `legacy_running` 后的有界 2.x readiness/health proof。 - D-379/ADR-0472（已接受；OpenRC live actor 待镜像基础设施恢复后补跑）：为 systemd/OpenRC adopted cutover 增加 Owner-side `service-legacy-rollback-prepare`。命令绑定 exact `target_stopped` record 与 instance head，重读 stop intent、Application v3、 activation 和 legacy-silence commitment，并复用 Docker 路径相同的 stable-fd SHA-256/inode/sidecar reconciliation。 diff --git a/docs/adr/ADR-0473-service-manager-legacy-rollback-commit.md b/docs/adr/ADR-0473-service-manager-legacy-rollback-commit.md new file mode 100644 index 00000000..fb4bcdfb --- /dev/null +++ b/docs/adr/ADR-0473-service-manager-legacy-rollback-commit.md @@ -0,0 +1,115 @@ +# ADR-0473：Service Manager Legacy Rollback Commit + +- 状态：Accepted +- 日期：2026-08-20 +- 关联 RFC：QL-RFC-0001 D-64、D-274、D-275、D-379、D-380 +- 关联 ADR：ADR-0314、ADR-0315、ADR-0362、ADR-0363、ADR-0472 + +## 背景 + +ADR-0472 已使 systemd/OpenRC adopted cutover 能在 Owner authority 内把安全候选推进到 +`rollback_prepared`，但它刻意不授予 root mutation，也不能证明旧服务已经重新运行。若 root 直接把 +`rollback_prepared` 当作启动授权，Owner 的数据判断、init mutation 与最终实例状态会混成一个不可审计步骤；若在响应丢失后 +再次执行 `start`，又可能越过崩溃窗口重复副作用。 + +本阶段还必须同时适配低配路由设备和集群节点：回滚不能引入新 daemon、watcher、连接池或部署对象，也不能为了源码目录变密就 +制造单文件 workspace package。编辑前 GitNexus 对 preparation、service bridge、deployment CLI、live actor 与 Docker gate +涉及的既有符号完成 upstream impact；均为 LOW，最多 3 个直接/累计上游、0 条已识别 execution flow。 + +## 决策 + +### 1. Owner 授权与 root mutation 分离 + +Owner 只能通过私有命令文件执行： + +```text +ql3-local-deploy service-legacy-rollback-authorize --command-file +``` + +授权必须重新验证当前 instance head、`rollback_prepared` record、Application 原始字节摘要、legacy-silence commitment 原始字节 +摘要、目标 descriptor 摘要和 manager kind，并 no-replace 发布 `legacy_restart_requested`。preparation 不是 root 授权，root bridge +只接受这一种当前 head 和 exact authorization digest。 + +### 2. root bridge 只允许固定服务和固定 argv + +root bridge 的唯一新操作是 `local.deployment.service-manager.legacy-rollback.execute`。它只允许： + +- legacy 服务名 `qinglong` 与 target 服务名 `qinglong3`； +- systemd descriptor `/etc/systemd/system/qinglong.service` 或 OpenRC descriptor `/etc/init.d/qinglong`； +- 受审的固定 `systemctl` 或 `rc-service` argv，不接受 shell、任意 executable、服务名或路径； +- 启动前同时证明 legacy inactive、target inactive，并复验 Owner application/commitment/authorization 与 descriptor 原始摘要。 + +bridge 在任何 start 前先 no-replace 写 durable root barrier。barrier 之后的所有重放都只能 inspect，永不再次 start。 + +### 3. 响应丢失按双服务观察收敛 + +若 manager 实际完成 start 但响应丢失，重放读取 barrier 并检查两个固定服务：只有 legacy active、target inactive，且进程身份与 +manager 观察一致时，root outcome 才是 proved running。若 barrier 已存在但无法证明上述事实，结果固定为 unproved，交给 Owner +终结为 `manual_required`；不得猜测、删除 barrier、替换 descriptor 或自动启动 target。 + +root outcome 先于 Owner outcome 持久化，因此 root 成功后 Owner 进程崩溃仍可用相同输入恢复，不重复 init mutation。 + +### 4. Owner 是实例终态唯一写者 + +Owner 通过： + +```text +ql3-local-deploy service-legacy-rollback-consume --command-file +``` + +消费 exact root outcome。只有 root 证明 legacy running、target stopped，且 authorization/head/digest 全部一致时，才 CAS 到 +`legacy_running`；任一不确定、漂移或双服务冲突都进入 `manual_required`。exact replay 返回同一 completion，不覆盖已存在的不同 +Owner outcome。 + +`legacy_running` 只证明 init/process running 与 target stopped，不代表 QingLong 2.x HTTP、任务调度、数据库或业务健康。 + +### 5. 保持部署与 package 边界 + +实现留在现有 `@qinglong/local-owner-cli`。为避免 `service-manager/` 继续平铺超过直接源码文件阈值，回滚子域内聚为： + +```text +service-manager/legacy-rollback/ + preparation.ts + contract.ts + consumer.ts + bridge.ts +``` + +不新增 workspace package、production dependency、binary、daemon、timer、watcher、socket、数据库连接、systemd/OpenRC unit 或 +Kubernetes workload。root bridge 仍是显式短生命周期命令。 + +## 被否决方案 + +1. **把 `rollback_prepared` 直接交给 root**：准备事实不是启动授权，拒绝。 +2. **barrier 重放再次调用 start**：响应丢失会重复副作用，拒绝。 +3. **root 直接写 instance head**：会混淆 OS mutation 与 Owner 状态 authority，拒绝。 +4. **只检查 legacy active**：target 同时运行时会制造双主，拒绝。 +5. **把 `legacy_running` 宣称为业务健康**：init/process 观察不足以证明 2.x readiness，拒绝。 +6. **新建 rollback package 或 daemon**：没有独立交付/依赖/生命周期依据，并放大路由设备闭包，拒绝。 + +## 验收证据 + +- 聚焦 Owner/contract 门 `16/16`；Local Owner 全量 `181 total / 176 pass / 5 conditional skip / 0 fail`。 +- 18-package clean build/逐包测试单次退出 0；backend 全量 + `1,523 total / 1,521 pass / 2 conditional skip / 0 fail`。 +- package boundary、Service Bridge import、Edge import、Cluster dependency、Cluster/Worker deployment、Console 与 Console + distribution 八项审计全部 compatible/passed。workspace 仍为 18 packages、`singleSourcePackages=[]`、 + `shallowSourcePackages=[]`；Local Owner 为 `111 source / 110 nested / 1 root binary entry`。 +- 14 档 Local artifact audit 全部 compatible。基础 Edge/Standalone 为 `2,598,669 / 2,598,747` bytes、316 files、 + 57 loaded modules；Adopted 为 `2,817,964 / 2,818,087` bytes、58 loaded modules；Application+AI 为 + `4,501,822 / 4,501,954` bytes；MCP 为 `7,324,601 / 7,324,709` bytes。回滚 authority 未进入基础运行闭包。 +- systemd live actor 已覆盖 root/non-root 成功、真实 manager start、响应丢失后的 inspect-only 收敛、Owner + `legacy_running` 消费与 exact replay；root barrier-crash 场景证明没有第二次 start，legacy/target 均保持 inactive，Owner 收敛为 + `manual_required`。 +- OpenRC actor 因本机缺少 Alpine/OpenRC 基础镜像，拉取 `node:24-alpine` 时 credential helper 挂起而未执行；因此本 ADR + 不宣称 systemd/OpenRC 全组合门闭合。临时 systemd 容器、测试镜像和临时 tag 均已清理。 +- 本阶段不修改 SQL、migration、PostgreSQL repository/role/Pool 或 HA 语义，因此不重新生成 PostgreSQL HA 证据。 + +## 未完成 + +- OpenRC root/non-root success 与 root barrier-crash live actor； +- `legacy_running` 后有界、可重放的 2.x readiness/health proof； +- 固定物理 Edge 的完整 prepare/authorize/execute/consume 回滚证据； +- `reconciliation_required` 的 export、冲突裁决与受审回灌。 + +本 ADR 关闭 service-manager 回滚的安全 commit 与进程级收敛，不代表 QingLong 3.0 升级/回退 Gate 已全部完成。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 357dfda1..450a2cc3 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -476,6 +476,7 @@ | [ADR-0470](./ADR-0470-session-scoped-console-capability-discovery.md) | Console 会话级能力发现与服务端操作围栏 | Accepted | | [ADR-0471](./ADR-0471-legacy-core-api-compatibility-baseline.md) | Legacy 核心执行 API 兼容基线 | Accepted | | [ADR-0472](./ADR-0472-service-manager-legacy-rollback-preparation.md) | Service Manager Legacy Rollback Preparation | Accepted | +| [ADR-0473](./ADR-0473-service-manager-legacy-rollback-commit.md) | Service Manager Legacy Rollback Commit | Accepted（OpenRC live actor 待补） | ## 规则 diff --git a/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts b/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts index 8dfd2d04..731336a1 100644 --- a/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts +++ b/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts @@ -82,7 +82,13 @@ import { import { prepareLocalServiceManagerLegacyRollback, prepareLocalServiceManagerLegacyRollbackCommandFile, -} from './service-manager/serviceLegacyRollback'; +} from './service-manager/legacy-rollback/preparation'; +import { + authorizeLocalServiceManagerLegacyRollback, + authorizeLocalServiceManagerLegacyRollbackCommandFile, + consumeLocalServiceManagerLegacyRollback, + consumeLocalServiceManagerLegacyRollbackCommandFile, +} from './service-manager/legacy-rollback/consumer'; export { LocalDeploymentConfigurationError, @@ -187,6 +193,10 @@ export { prepareLocalServiceManagerIntentCommandFile, prepareLocalServiceManagerLegacyRollback, prepareLocalServiceManagerLegacyRollbackCommandFile, + authorizeLocalServiceManagerLegacyRollback, + authorizeLocalServiceManagerLegacyRollbackCommandFile, + consumeLocalServiceManagerLegacyRollback, + consumeLocalServiceManagerLegacyRollbackCommandFile, }; export { localServiceManagerIntentDigest, @@ -221,7 +231,29 @@ export { type LocalServiceManagerLegacyRollbackPreparation, type LocalServiceManagerLegacyRollbackPrepareCommand, type LocalServiceManagerLegacyRollbackPrepareResult, -} from './service-manager/serviceLegacyRollback'; +} from './service-manager/legacy-rollback/preparation'; +export { + normalizeLocalServiceManagerLegacyRollbackCompletion, + type LocalServiceManagerLegacyRollbackAuthorizeCommand, + type LocalServiceManagerLegacyRollbackAuthorizeResult, + type LocalServiceManagerLegacyRollbackCompletion, + type LocalServiceManagerLegacyRollbackConsumeCommand, + type LocalServiceManagerLegacyRollbackConsumeResult, +} from './service-manager/legacy-rollback/consumer'; +export { + localServiceManagerLegacyCompletionPath, + localServiceManagerLegacyDescriptorPath, + localServiceManagerLegacyStartAuthorizationPath, + localServiceManagerLegacyStartOutcomePath, + localServiceManagerTargetDescriptorPath, + normalizeLocalServiceManagerLegacyRollbackBridgeCommand, + normalizeLocalServiceManagerLegacyStartAuthorization, + normalizeLocalServiceManagerLegacyStartOutcome, + type LocalServiceManagerLegacyRollbackBridgeCommand, + type LocalServiceManagerLegacyRollbackBridgeResult, + type LocalServiceManagerLegacyStartAuthorization, + type LocalServiceManagerLegacyStartOutcome, +} from './service-manager/legacy-rollback/contract'; export async function prepareLocalDeployment( input: unknown, diff --git a/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts b/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts index 5af8ae55..be0f361a 100644 --- a/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts +++ b/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts @@ -3,10 +3,12 @@ // Keep the deployment binary beside its composition authority. import { applyLocalDeploymentComposeCommandFile, + authorizeLocalServiceManagerLegacyRollbackCommandFile, collectLocalDeploymentComposeEvidenceCommitCommandFile, collectLocalDeploymentComposeEvidencePrepareCommandFile, consumeLocalServiceManagerOutcomeCommandFile, consumeLocalServiceManagerCutoverOutcomeCommandFile, + consumeLocalServiceManagerLegacyRollbackCommandFile, inspectLocalDeploymentStatusCommandFile, preflightLocalDeploymentComposeCommandFile, prepareLocalServiceManagerIntentCommandFile, @@ -23,7 +25,7 @@ import { } from './localDeployment'; const USAGE = - 'Usage: ql3-local-deploy --command-file /absolute/private-command.json'; + 'Usage: ql3-local-deploy --command-file /absolute/private-command.json'; async function main(argv: readonly string[]): Promise { if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) { @@ -38,6 +40,8 @@ async function main(argv: readonly string[]): Promise { argv[0] !== 'service-outcome-consume' && argv[0] !== 'service-cutover-consume' && argv[0] !== 'service-legacy-rollback-prepare' && + argv[0] !== 'service-legacy-rollback-authorize' && + argv[0] !== 'service-legacy-rollback-consume' && argv[0] !== 'cutover-legacy-stop' && argv[0] !== 'cutover-target-start' && argv[0] !== 'cutover-target-restart' && @@ -78,6 +82,10 @@ async function main(argv: readonly string[]): Promise { ? consumeLocalServiceManagerCutoverOutcomeCommandFile(argv[2]!) : argv[0] === 'service-legacy-rollback-prepare' ? prepareLocalServiceManagerLegacyRollbackCommandFile(argv[2]!) + : argv[0] === 'service-legacy-rollback-authorize' + ? authorizeLocalServiceManagerLegacyRollbackCommandFile(argv[2]!) + : argv[0] === 'service-legacy-rollback-consume' + ? consumeLocalServiceManagerLegacyRollbackCommandFile(argv[2]!) : argv[0] === 'cutover-legacy-stop' ? stopLegacyDockerForLocalDeploymentCommandFile(argv[2]!) : argv[0] === 'cutover-target-start' || diff --git a/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/bridge.ts b/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/bridge.ts new file mode 100644 index 00000000..3d58863f --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/bridge.ts @@ -0,0 +1,881 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +import { readPrivateLocalCommandFile } from '@qinglong/local-command-file'; + +import { LocalDeploymentConfigurationError } from '../../foundation/error'; +import type { + LocalServiceBridgeManager, + LocalServiceManagerKind, +} from '../serviceBridgeContract'; +import { + ensureRootServiceBridgeDirectory, + publishServiceBridgeFile, + readOwnerPrivateJsonFile, + readServiceBridgeFile, + validateServiceBridgeDirectory, +} from '../serviceBridgeFiles'; +import type { + LocalServiceManagerRunRequest, + LocalServiceManagerRunResult, +} from '../serviceBridge'; +import { + localServiceManagerLegacyDescriptorPath, + localServiceManagerLegacyStartAuthorizationPath, + localServiceManagerLegacyStartOutcomeDigest, + localServiceManagerLegacyStartOutcomePath, + localServiceManagerRollbackObservationDigest, + localServiceManagerTargetDescriptorPath, + normalizeLocalServiceManagerLegacyRollbackBridgeCommand, + normalizeLocalServiceManagerLegacyStartAuthorization, + normalizeLocalServiceManagerLegacyStartOutcome, + normalizeLocalServiceManagerRollbackObservation, + type LocalServiceManagerLegacyRollbackBridgeCommand, + type LocalServiceManagerLegacyRollbackBridgeResult, + type LocalServiceManagerLegacyStartAuthorization, + type LocalServiceManagerLegacyStartOutcome, + type LocalServiceManagerRollbackManualReason, + type LocalServiceManagerRollbackMutationDisposition, + type LocalServiceManagerRollbackObservation, +} from './contract'; + +const BARRIER_SCHEMA = 'qinglong3-local-service-manager-legacy-start-barrier'; +const MAX_MANAGER_OUTPUT_BYTES = 64 * 1024; +const MANAGER_TIMEOUT_MS = 45_000; + +interface LegacyStartBarrier { + readonly schema: typeof BARRIER_SCHEMA; + readonly schemaVersion: 1; + readonly cutoverId: string; + readonly generation: number; + readonly managerKind: LocalServiceManagerKind; + readonly preparationDigest: string; + readonly authorizationDigest: string; + readonly ownerUid: number; + readonly ownerGid: number; + readonly legacyDescriptorDigest: string; + readonly targetDescriptorDigest: string; + readonly legacyPreObservation: Readonly; + readonly targetPreObservation: Readonly; + readonly createdAtMs: number; + readonly barrierDigest: string; +} + +export interface LocalServiceManagerLegacyRollbackBridgeDependencies { + readonly runManager?: ( + request: Readonly, + ) => Readonly; + readonly now?: () => number; + readonly afterBarrier?: () => void; + readonly afterStart?: () => void; +} + +function configurationError(message: string, cause?: unknown): never { + throw new LocalDeploymentConfigurationError(message, { cause }); +} + +function object(value: unknown, label: string): Record { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + (Object.getPrototypeOf(value) !== Object.prototype && + Object.getPrototypeOf(value) !== null) + ) { + configurationError(`${label} must be an object`); + } + return value as Record; +} + +function exact( + value: Record, + keys: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + configurationError(`${label} shape is invalid`); + } +} + +function digest(value: unknown): string { + return crypto + .createHash('sha256') + .update(JSON.stringify(value)) + .digest('hex'); +} + +function sha256(bytes: Buffer): string { + return crypto.createHash('sha256').update(bytes).digest('hex'); +} + +function assertRootIdentity(): void { + if ( + process.platform === 'win32' || + typeof process.getuid !== 'function' || + typeof process.geteuid !== 'function' || + process.getuid() !== 0 || + process.geteuid() !== 0 + ) { + configurationError('legacy rollback bridge must run as root'); + } +} + +function trustedRootExecutable(filePath: string, label: string): void { + let stat: fs.Stats; + try { + stat = fs.lstatSync(filePath); + } catch (error) { + configurationError(`${label} is unavailable`, error); + } + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.uid !== 0 || + stat.gid !== 0 || + stat.nlink < 1 || + (stat.mode & 0o022) !== 0 || + fs.realpathSync(filePath) !== filePath + ) { + configurationError(`${label} identity is invalid`); + } +} + +function managerExecutables( + manager: LocalServiceBridgeManager, +): readonly string[] { + return manager.kind === 'systemd' + ? Object.freeze([manager.executable]) + : Object.freeze([manager.serviceExecutable, manager.updateExecutable]); +} + +function defaultRunManager( + request: Readonly, +): Readonly { + const result = spawnSync(request.executable, [...request.args], { + encoding: 'utf8', + env: Object.freeze({ + PATH: '/usr/sbin:/usr/bin:/sbin:/bin', + LANG: 'C', + LC_ALL: 'C', + }), + timeout: request.timeoutMs, + maxBuffer: MAX_MANAGER_OUTPUT_BYTES, + shell: false, + windowsHide: true, + }); + return Object.freeze({ + status: result.status, + signal: result.signal, + stdout: result.stdout ?? '', + stderr: result.stderr ?? '', + responseLost: result.error !== undefined || result.status === null, + }); +} + +function run( + runner: NonNullable< + LocalServiceManagerLegacyRollbackBridgeDependencies['runManager'] + >, + executable: string, + args: readonly string[], +): Readonly { + const result = runner({ + executable, + args: Object.freeze([...args]), + timeoutMs: MANAGER_TIMEOUT_MS, + }); + if ( + !result || + typeof result !== 'object' || + (result.status !== null && !Number.isSafeInteger(result.status)) || + (result.signal !== null && typeof result.signal !== 'string') || + typeof result.stdout !== 'string' || + typeof result.stderr !== 'string' || + typeof result.responseLost !== 'boolean' || + Buffer.byteLength(result.stdout, 'utf8') > MAX_MANAGER_OUTPUT_BYTES || + Buffer.byteLength(result.stderr, 'utf8') > MAX_MANAGER_OUTPUT_BYTES + ) { + configurationError('legacy rollback manager runner returned invalid data'); + } + return result; +} + +function systemdObservation( + manager: Extract, + serviceName: 'qinglong' | 'qinglong3', + fragmentPath: string, + runner: NonNullable< + LocalServiceManagerLegacyRollbackBridgeDependencies['runManager'] + >, + now: () => number, +): Readonly { + const result = run(runner, manager.executable, [ + 'show', + `${serviceName}.service`, + '--no-page', + '--property=LoadState,ActiveState,SubState,FragmentPath,MainPID,UnitFileState', + ]); + const fields = new Map(); + if (!result.responseLost && result.status === 0) { + for (const line of result.stdout.split('\n')) { + const separator = line.indexOf('='); + if (separator > 0) { + fields.set(line.slice(0, separator), line.slice(separator + 1)); + } + } + } + const rawPid = Number(fields.get('MainPID') ?? '0'); + const loadState = fields.get('LoadState'); + const activeState = fields.get('ActiveState'); + const enabledState = fields.get('UnitFileState'); + const payload = Object.freeze({ + managerKind: 'systemd' as const, + serviceName, + fragmentPath: fields.get('FragmentPath') || fragmentPath, + loadState: + loadState === 'loaded' + ? ('loaded' as const) + : loadState === 'not-found' + ? ('not-found' as const) + : ('unknown' as const), + activeState: + activeState === 'active' + ? ('active' as const) + : activeState === 'inactive' + ? ('inactive' as const) + : activeState === 'failed' + ? ('failed' as const) + : ('unknown' as const), + subState: (fields.get('SubState') ?? 'unknown').slice(0, 128), + enabledState: + enabledState === 'enabled' + ? ('enabled' as const) + : enabledState === 'disabled' + ? ('disabled' as const) + : enabledState === 'static' + ? ('static' as const) + : ('unknown' as const), + mainPid: + Number.isSafeInteger(rawPid) && rawPid >= 0 && rawPid <= 0x7fffffff + ? rawPid + : 0, + observedAtMs: now(), + }); + return normalizeLocalServiceManagerRollbackObservation({ + ...payload, + observationDigest: localServiceManagerRollbackObservationDigest(payload), + }); +} + +function openrcObservation( + manager: Extract, + serviceName: 'qinglong' | 'qinglong3', + fragmentPath: string, + runner: NonNullable< + LocalServiceManagerLegacyRollbackBridgeDependencies['runManager'] + >, + now: () => number, +): Readonly { + const service = run(runner, manager.serviceExecutable, [ + serviceName, + 'status', + ]); + const enabled = run(runner, manager.updateExecutable, ['show', 'default']); + const active = !service.responseLost && service.status === 0; + const stopped = + !service.responseLost && + service.status !== null && + (service.stdout.includes('stopped') || service.status === 3); + const payload = Object.freeze({ + managerKind: 'openrc' as const, + serviceName, + fragmentPath, + loadState: fs.existsSync(fragmentPath) + ? ('loaded' as const) + : ('not-found' as const), + activeState: active + ? ('active' as const) + : stopped + ? ('inactive' as const) + : ('unknown' as const), + subState: active ? 'started' : stopped ? 'stopped' : 'unknown', + enabledState: + !enabled.responseLost && + enabled.status === 0 && + enabled.stdout + .split('\n') + .some((line) => new RegExp(`\\b${serviceName}\\b`).test(line)) + ? ('enabled' as const) + : ('unknown' as const), + mainPid: 0, + observedAtMs: now(), + }); + return normalizeLocalServiceManagerRollbackObservation({ + ...payload, + observationDigest: localServiceManagerRollbackObservationDigest(payload), + }); +} + +function inspectManager( + manager: LocalServiceBridgeManager, + serviceName: 'qinglong' | 'qinglong3', + fragmentPath: string, + runner: NonNullable< + LocalServiceManagerLegacyRollbackBridgeDependencies['runManager'] + >, + now: () => number, +): Readonly { + return manager.kind === 'systemd' + ? systemdObservation(manager, serviceName, fragmentPath, runner, now) + : openrcObservation(manager, serviceName, fragmentPath, runner, now); +} + +function descriptorDigest( + filePath: string, + kind: LocalServiceManagerKind, +): string { + const bytes = readServiceBridgeFile( + filePath, + { + uid: 0, + gid: 0, + mode: kind === 'systemd' ? 0o644 : 0o755, + maximumBytes: 64 * 1024, + }, + 'installed rollback service descriptor', + ); + try { + return sha256(bytes); + } finally { + bytes.fill(0); + } +} + +function ownerMaterialMatches( + command: Readonly, + authorization: Readonly, + ownerUid: number, + ownerGid: number, +): boolean { + const material = [ + [ + path.join(command.options.deploymentRoot, 'local-application.json'), + authorization.applicationConfigDigest, + 'application configuration', + ], + [ + path.join( + command.options.deploymentRoot, + 'service', + 'cutovers', + authorization.cutoverId, + '0002-legacy-stopped.json', + ), + authorization.commitmentFileDigest, + 'legacy silence commitment', + ], + ] as const; + try { + for (const [filePath, expectedDigest, label] of material) { + const bytes = readServiceBridgeFile( + filePath, + { uid: ownerUid, gid: ownerGid, mode: 0o600 }, + label, + ); + try { + if (sha256(bytes) !== expectedDigest) return false; + } finally { + bytes.fill(0); + } + } + return true; + } catch { + return false; + } +} + +function barrierRecord( + authorization: Readonly, + ownerUid: number, + ownerGid: number, + legacyDescriptorDigest: string, + targetDescriptorDigest: string, + legacyPreObservation: Readonly, + targetPreObservation: Readonly, + now: () => number, +): Readonly { + const payload = Object.freeze({ + schema: BARRIER_SCHEMA, + schemaVersion: 1 as const, + cutoverId: authorization.cutoverId, + generation: authorization.generation, + managerKind: authorization.managerKind, + preparationDigest: authorization.preparationDigest, + authorizationDigest: authorization.authorizationDigest, + ownerUid, + ownerGid, + legacyDescriptorDigest, + targetDescriptorDigest, + legacyPreObservation, + targetPreObservation, + createdAtMs: now(), + }); + return Object.freeze({ ...payload, barrierDigest: digest(payload) }); +} + +function normalizeBarrier( + value: unknown, + authorization: Readonly, +): Readonly { + const barrier = object(value, 'legacy start barrier'); + exact( + barrier, + [ + 'authorizationDigest', + 'barrierDigest', + 'createdAtMs', + 'cutoverId', + 'generation', + 'legacyDescriptorDigest', + 'legacyPreObservation', + 'managerKind', + 'ownerGid', + 'ownerUid', + 'preparationDigest', + 'schema', + 'schemaVersion', + 'targetDescriptorDigest', + 'targetPreObservation', + ], + 'legacy start barrier', + ); + const legacyPreObservation = normalizeLocalServiceManagerRollbackObservation( + barrier.legacyPreObservation, + ); + const targetPreObservation = normalizeLocalServiceManagerRollbackObservation( + barrier.targetPreObservation, + ); + const { barrierDigest, ...rawPayload } = barrier; + const payload = { + ...rawPayload, + legacyPreObservation, + targetPreObservation, + }; + if ( + barrier.schema !== BARRIER_SCHEMA || + barrier.schemaVersion !== 1 || + barrier.cutoverId !== authorization.cutoverId || + barrier.generation !== authorization.generation || + barrier.managerKind !== authorization.managerKind || + barrier.preparationDigest !== authorization.preparationDigest || + barrier.authorizationDigest !== authorization.authorizationDigest || + !Number.isSafeInteger(barrier.ownerUid) || + (barrier.ownerUid as number) < 0 || + !Number.isSafeInteger(barrier.ownerGid) || + (barrier.ownerGid as number) < 0 || + typeof barrier.legacyDescriptorDigest !== 'string' || + !/^[0-9a-f]{64}$/.test(barrier.legacyDescriptorDigest) || + barrier.targetDescriptorDigest !== authorization.targetDescriptorDigest || + legacyPreObservation.managerKind !== authorization.managerKind || + legacyPreObservation.serviceName !== 'qinglong' || + targetPreObservation.managerKind !== authorization.managerKind || + targetPreObservation.serviceName !== 'qinglong3' || + !Number.isSafeInteger(barrier.createdAtMs) || + (barrier.createdAtMs as number) < authorization.requestedAtMs || + typeof barrierDigest !== 'string' || + !/^[0-9a-f]{64}$/.test(barrierDigest) || + digest(payload) !== barrierDigest + ) { + configurationError('legacy start barrier drifted'); + } + return barrier as unknown as Readonly; +} + +function preconditionProved( + observation: Readonly, + expectedPath: string, +): boolean { + return ( + observation.fragmentPath === expectedPath && + observation.loadState === 'loaded' && + observation.activeState === 'inactive' && + observation.mainPid === 0 + ); +} + +function finalStateProved( + managerKind: LocalServiceManagerKind, + legacy: Readonly, + target: Readonly, + legacyPath: string, + targetPath: string, +): boolean { + return ( + legacy.fragmentPath === legacyPath && + legacy.loadState === 'loaded' && + legacy.activeState === 'active' && + (managerKind === 'openrc' || legacy.mainPid > 0) && + target.fragmentPath === targetPath && + target.loadState === 'loaded' && + target.activeState === 'inactive' && + target.mainPid === 0 + ); +} + +function executeStart( + manager: LocalServiceBridgeManager, + runner: NonNullable< + LocalServiceManagerLegacyRollbackBridgeDependencies['runManager'] + >, +): Readonly<{ failed: boolean; responseLost: boolean }> { + const result = + manager.kind === 'systemd' + ? run(runner, manager.executable, ['start', 'qinglong.service']) + : run(runner, manager.serviceExecutable, ['qinglong', 'start']); + return Object.freeze({ + failed: !result.responseLost && result.status !== 0, + responseLost: result.responseLost, + }); +} + +function outcomeRecord( + authorization: Readonly, + barrier: Readonly, + state: LocalServiceManagerLegacyStartOutcome['state'], + disposition: LocalServiceManagerRollbackMutationDisposition, + manualReason: LocalServiceManagerRollbackManualReason | null, + legacyObservation: Readonly, + targetObservation: Readonly, + now: () => number, +): Readonly { + const payload = Object.freeze({ + schema: 'qinglong3-local-service-manager-legacy-start-outcome' as const, + schemaVersion: 1 as const, + state, + cutoverId: authorization.cutoverId, + profile: authorization.profile, + instanceId: authorization.instanceId, + generation: authorization.generation, + activationDigest: authorization.activationDigest, + managerKind: authorization.managerKind, + preparationDigest: authorization.preparationDigest, + authorizationDigest: authorization.authorizationDigest, + barrierDigest: barrier.barrierDigest, + legacyDescriptorDigest: barrier.legacyDescriptorDigest, + targetDescriptorDigest: barrier.targetDescriptorDigest, + mutationDisposition: disposition, + manualReason, + legacyObservation, + targetObservation, + completedAtMs: Math.max( + now(), + legacyObservation.observedAtMs, + targetObservation.observedAtMs, + ), + }); + return normalizeLocalServiceManagerLegacyStartOutcome({ + ...payload, + outcomeDigest: localServiceManagerLegacyStartOutcomeDigest(payload), + }); +} + +function readRootJson(filePath: string, label: string): unknown { + const file = readOwnerPrivateJsonFile(filePath, label); + if (file.uid !== 0 || file.gid !== 0) { + configurationError(`${label} is not root-owned`); + } + return file.value; +} + +function publishOutcome( + actionDirectory: string, + ownerOutcomePath: string, + ownerUid: number, + ownerGid: number, + outcome: Readonly, +): 'prepared' | 'existing' { + const contents = `${JSON.stringify(outcome, null, 2)}\n`; + const status = publishServiceBridgeFile( + path.join(actionDirectory, 'outcome.json'), + contents, + 0o600, + 0, + 0, + 'legacy start root outcome', + ); + publishServiceBridgeFile( + ownerOutcomePath, + contents, + 0o600, + ownerUid, + ownerGid, + 'legacy start Owner outcome', + ); + return status; +} + +function result( + command: Readonly, + status: 'prepared' | 'existing', + outcome: Readonly, +): Readonly { + return Object.freeze({ + schemaVersion: 1 as const, + operation: command.operation, + status, + state: outcome.state, + cutoverId: outcome.cutoverId, + generation: outcome.generation, + outcomeDigest: outcome.outcomeDigest, + }); +} + +export function runLocalServiceManagerLegacyRollbackBridge( + input: unknown, + dependencies: Readonly = {}, +): Readonly { + assertRootIdentity(); + const command = + normalizeLocalServiceManagerLegacyRollbackBridgeCommand(input); + const runner = dependencies.runManager ?? defaultRunManager; + const now = dependencies.now ?? Date.now; + for (const executable of managerExecutables(command.options.manager)) { + trustedRootExecutable(executable, 'service manager executable'); + } + const authorizationPath = localServiceManagerLegacyStartAuthorizationPath( + command.options.deploymentRoot, + command.request.cutoverId, + command.request.generation, + ); + const ownedAuthorization = readOwnerPrivateJsonFile( + authorizationPath, + 'legacy start authorization', + ); + const authorization = normalizeLocalServiceManagerLegacyStartAuthorization( + ownedAuthorization.value, + ); + if ( + authorization.authorizationDigest !== + command.request.expectedAuthorizationDigest || + authorization.cutoverId !== command.request.cutoverId || + authorization.generation !== command.request.generation || + authorization.managerKind !== command.options.manager.kind + ) { + configurationError('legacy rollback root authorization drifted'); + } + const ownerUid = ownedAuthorization.uid; + const ownerGid = ownedAuthorization.gid; + validateServiceBridgeDirectory( + command.options.deploymentRoot, + ownerUid, + ownerGid, + 0o700, + 'deploymentRoot', + ); + validateServiceBridgeDirectory( + path.join(command.options.deploymentRoot, 'service'), + ownerUid, + ownerGid, + 0o700, + 'serviceDescriptorRoot', + ); + const legacyPath = localServiceManagerLegacyDescriptorPath( + authorization.managerKind, + ); + const targetPath = localServiceManagerTargetDescriptorPath( + authorization.managerKind, + ); + ensureRootServiceBridgeDirectory( + command.options.controllerRoot, + 'serviceBridgeControllerRoot', + ); + const rollbackRoot = path.join( + command.options.controllerRoot, + 'legacy-rollbacks', + ); + ensureRootServiceBridgeDirectory(rollbackRoot, 'legacyRollbackRoot'); + const actionDirectory = path.join( + rollbackRoot, + authorization.authorizationDigest, + ); + ensureRootServiceBridgeDirectory(actionDirectory, 'legacyRollbackActionRoot'); + const barrierPath = path.join(actionDirectory, 'barrier.json'); + const rootOutcomePath = path.join(actionDirectory, 'outcome.json'); + const ownerOutcomePath = localServiceManagerLegacyStartOutcomePath( + command.options.deploymentRoot, + authorization.cutoverId, + authorization.generation, + ); + if (fs.existsSync(rootOutcomePath)) { + const existing = normalizeLocalServiceManagerLegacyStartOutcome( + readRootJson(rootOutcomePath, 'legacy start root outcome'), + ); + if ( + existing.authorizationDigest !== authorization.authorizationDigest || + existing.preparationDigest !== authorization.preparationDigest || + existing.managerKind !== authorization.managerKind + ) { + configurationError('legacy start root outcome drifted'); + } + publishOutcome( + actionDirectory, + ownerOutcomePath, + ownerUid, + ownerGid, + existing, + ); + return result(command, 'existing', existing); + } + const legacyDigest = descriptorDigest(legacyPath, authorization.managerKind); + const targetDigest = descriptorDigest(targetPath, authorization.managerKind); + if ( + targetDigest !== authorization.targetDescriptorDigest || + !ownerMaterialMatches(command, authorization, ownerUid, ownerGid) + ) { + configurationError('legacy rollback root preflight material drifted'); + } + const replay = fs.existsSync(barrierPath); + const barrier = replay + ? normalizeBarrier( + readRootJson(barrierPath, 'legacy start barrier'), + authorization, + ) + : barrierRecord( + authorization, + ownerUid, + ownerGid, + legacyDigest, + targetDigest, + inspectManager( + command.options.manager, + 'qinglong', + legacyPath, + runner, + now, + ), + inspectManager( + command.options.manager, + 'qinglong3', + targetPath, + runner, + now, + ), + now, + ); + if (!replay) { + publishServiceBridgeFile( + barrierPath, + `${JSON.stringify(barrier, null, 2)}\n`, + 0o600, + 0, + 0, + 'legacy start barrier', + ); + dependencies.afterBarrier?.(); + } + let mutation: Readonly<{ failed: boolean; responseLost: boolean }> = + Object.freeze({ failed: false, responseLost: false }); + let immediateReason: LocalServiceManagerRollbackManualReason | null = null; + if ( + !preconditionProved(barrier.legacyPreObservation, legacyPath) || + !preconditionProved(barrier.targetPreObservation, targetPath) + ) { + immediateReason = 'start_precondition_unproved'; + } else if ( + descriptorDigest(legacyPath, authorization.managerKind) !== + barrier.legacyDescriptorDigest || + descriptorDigest(targetPath, authorization.managerKind) !== + barrier.targetDescriptorDigest + ) { + immediateReason = 'service_descriptor_drifted'; + } else if ( + !ownerMaterialMatches(command, authorization, ownerUid, ownerGid) + ) { + immediateReason = 'authorization_material_drifted'; + } else if (!replay) { + mutation = executeStart(command.options.manager, runner); + dependencies.afterStart?.(); + } + const legacyObservation = inspectManager( + command.options.manager, + 'qinglong', + legacyPath, + runner, + now, + ); + const targetObservation = inspectManager( + command.options.manager, + 'qinglong3', + targetPath, + runner, + now, + ); + let descriptorDrifted = false; + try { + descriptorDrifted = + descriptorDigest(legacyPath, authorization.managerKind) !== + barrier.legacyDescriptorDigest || + descriptorDigest(targetPath, authorization.managerKind) !== + barrier.targetDescriptorDigest; + } catch { + descriptorDrifted = true; + } + const proved = + immediateReason === null && + !descriptorDrifted && + finalStateProved( + authorization.managerKind, + legacyObservation, + targetObservation, + legacyPath, + targetPath, + ); + const disposition: LocalServiceManagerRollbackMutationDisposition = replay + ? 'replay-inspected' + : mutation.responseLost + ? 'response-loss-inspected' + : 'executed'; + const manualReason: LocalServiceManagerRollbackManualReason | null = proved + ? null + : immediateReason ?? + (descriptorDrifted + ? 'service_descriptor_drifted' + : mutation.failed + ? 'manager_command_failed' + : 'manager_state_unproved'); + const outcome = outcomeRecord( + authorization, + barrier, + proved ? 'legacy_running' : 'manual_required', + disposition, + manualReason, + legacyObservation, + targetObservation, + now, + ); + const status = publishOutcome( + actionDirectory, + ownerOutcomePath, + ownerUid, + ownerGid, + outcome, + ); + return result(command, status, outcome); +} + +export function runLocalServiceManagerLegacyRollbackBridgeCommandFile( + commandFile: string, + dependencies: Readonly = {}, +): Readonly { + return runLocalServiceManagerLegacyRollbackBridge( + readPrivateLocalCommandFile(commandFile), + dependencies, + ); +} diff --git a/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/consumer.ts b/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/consumer.ts new file mode 100644 index 00000000..16a435a5 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/consumer.ts @@ -0,0 +1,814 @@ +import path from 'node:path'; + +import { + MAX_PRIVATE_LOCAL_JSON_FILE_BYTES, + readPrivateLocalJsonFile, +} from '@qinglong/local-command-file'; + +import { + advanceLocalCutoverInstanceHead, + readLocalCutoverInstanceHead, +} from '../../cutover/instanceLineage'; +import { cutoverDigest } from '../../cutover/targetEvidence'; +import { currentIdentity } from '../../foundation/contract'; +import { LocalDeploymentConfigurationError } from '../../foundation/error'; +import { + preflightPublishedFile, + publishExactFile, +} from '../../foundation/files'; +import { + localServiceManagerLegacyCompletionPath, + localServiceManagerLegacyStartAuthorizationDigest, + localServiceManagerLegacyStartAuthorizationPath, + localServiceManagerLegacyStartOutcomePath, + normalizeLocalServiceManagerLegacyStartAuthorization, + normalizeLocalServiceManagerLegacyStartOutcome, + type LocalServiceManagerLegacyStartAuthorization, + type LocalServiceManagerLegacyStartOutcome, +} from './contract'; +import { + localServiceManagerLegacyRollbackPreparationPath, + normalizeLocalServiceManagerLegacyRollbackPreparation, + prepareLocalServiceManagerLegacyRollback, + type LocalServiceManagerLegacyRollbackPreparation, +} from './preparation'; + +const COMPLETION_SCHEMA = + 'qinglong3-local-service-manager-legacy-rollback-completion'; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const INSTANCE_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/; +const CUTOVER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const SAFE_PATH_PATTERN = /^\/[A-Za-z0-9._/@-]+$/; +const MAX_PATH_BYTES = 4_096; +const MAX_GENERATION = 15; + +interface RollbackIdentity { + readonly cutoverId: string; + readonly profile: 'edge' | 'standalone'; + readonly instanceId: string; + readonly generation: number; + readonly expectedActivationDigest: string; +} + +export interface LocalServiceManagerLegacyRollbackAuthorizeCommand { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.service-manager.legacy-rollback.authorize'; + readonly options: Readonly<{ + deploymentRoot: string; + allowRootService: boolean; + }>; + readonly request: Readonly< + RollbackIdentity & { + expectedPreparationDigest: string; + expectedInstanceHeadDigest: string; + requestedAtMs: number; + } + >; +} + +export interface LocalServiceManagerLegacyRollbackAuthorizeResult { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.service-manager.legacy-rollback.authorize'; + readonly status: 'prepared' | 'existing'; + readonly state: 'legacy_restart_requested'; + readonly cutoverId: string; + readonly generation: number; + readonly preparationDigest: string; + readonly authorizationDigest: string; + readonly instanceHeadDigest: string; +} + +export interface LocalServiceManagerLegacyRollbackConsumeCommand { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.service-manager.legacy-rollback.consume'; + readonly options: Readonly<{ + deploymentRoot: string; + allowRootService: boolean; + }>; + readonly request: Readonly< + RollbackIdentity & { + expectedPreparationDigest: string; + expectedAuthorizationDigest: string; + expectedAuthorizationHeadDigest: string; + requestedAtMs: number; + } + >; +} + +export interface LocalServiceManagerLegacyRollbackConsumeResult { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.service-manager.legacy-rollback.consume'; + readonly status: 'prepared' | 'existing'; + readonly state: 'legacy_running' | 'manual_required'; + readonly cutoverId: string; + readonly generation: number; + readonly outcomeDigest: string; + readonly completionDigest: string; + readonly instanceHeadDigest: string; +} + +export interface LocalServiceManagerLegacyRollbackCompletion { + readonly schema: typeof COMPLETION_SCHEMA; + readonly schemaVersion: 1; + readonly state: 'legacy_running' | 'manual_required'; + readonly cutoverId: string; + readonly profile: 'edge' | 'standalone'; + readonly instanceId: string; + readonly generation: number; + readonly activationDigest: string; + readonly preparationDigest: string; + readonly authorizationDigest: string; + readonly expectedAuthorizationHeadDigest: string; + readonly rootOutcomeDigest: string; + readonly rootBarrierDigest: string; + readonly legacyDescriptorDigest: string; + readonly targetDescriptorDigest: string; + readonly legacyObservationDigest: string; + readonly targetObservationDigest: string; + readonly manualReason: LocalServiceManagerLegacyStartOutcome['manualReason']; + readonly completedAtMs: number; + readonly completionDigest: string; +} + +function configurationError(message: string, cause?: unknown): never { + throw new LocalDeploymentConfigurationError(message, { cause }); +} + +function object(value: unknown, label: string): Record { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + (Object.getPrototypeOf(value) !== Object.prototype && + Object.getPrototypeOf(value) !== null) + ) { + configurationError(`${label} must be an object`); + } + return value as Record; +} + +function exact( + value: Record, + keys: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + configurationError(`${label} shape is invalid`); + } +} + +function safeAbsolutePath(value: unknown, label: string): string { + if ( + typeof value !== 'string' || + !path.isAbsolute(value) || + path.normalize(value) !== value || + path.parse(value).root === value || + value.includes('\0') || + value.includes('//') || + !SAFE_PATH_PATTERN.test(value) || + Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES + ) { + configurationError(`${label} must be a supervisor-safe absolute path`); + } + return value; +} + +function normalizeIdentity( + value: Record, +): Readonly { + if ( + typeof value.cutoverId !== 'string' || + !CUTOVER_ID_PATTERN.test(value.cutoverId) || + (value.profile !== 'edge' && value.profile !== 'standalone') || + typeof value.instanceId !== 'string' || + !INSTANCE_ID_PATTERN.test(value.instanceId) || + !Number.isSafeInteger(value.generation) || + (value.generation as number) < 1 || + (value.generation as number) > MAX_GENERATION || + typeof value.expectedActivationDigest !== 'string' || + !DIGEST_PATTERN.test(value.expectedActivationDigest) + ) { + configurationError('legacy rollback identity is invalid'); + } + return Object.freeze({ + cutoverId: value.cutoverId, + profile: value.profile, + instanceId: value.instanceId, + generation: value.generation as number, + expectedActivationDigest: value.expectedActivationDigest, + }); +} + +function normalizeAuthorizeCommand( + value: unknown, +): Readonly { + const command = object(value, 'legacy rollback authorize command'); + exact( + command, + ['operation', 'options', 'request', 'schemaVersion'], + 'command', + ); + const options = object(command.options, 'options'); + exact(options, ['allowRootService', 'deploymentRoot'], 'options'); + const request = object(command.request, 'request'); + exact( + request, + [ + 'cutoverId', + 'expectedActivationDigest', + 'expectedInstanceHeadDigest', + 'expectedPreparationDigest', + 'generation', + 'instanceId', + 'profile', + 'requestedAtMs', + ], + 'request', + ); + const identity = currentIdentity(); + const rollbackIdentity = normalizeIdentity(request); + if ( + command.schemaVersion !== 1 || + command.operation !== + 'local.deployment.service-manager.legacy-rollback.authorize' || + typeof options.allowRootService !== 'boolean' || + (identity.uid === 0) !== options.allowRootService || + typeof request.expectedPreparationDigest !== 'string' || + !DIGEST_PATTERN.test(request.expectedPreparationDigest) || + typeof request.expectedInstanceHeadDigest !== 'string' || + !DIGEST_PATTERN.test(request.expectedInstanceHeadDigest) || + !Number.isSafeInteger(request.requestedAtMs) || + (request.requestedAtMs as number) < 0 + ) { + configurationError('legacy rollback authorize command is invalid'); + } + return Object.freeze({ + schemaVersion: 1 as const, + operation: + 'local.deployment.service-manager.legacy-rollback.authorize' as const, + options: Object.freeze({ + deploymentRoot: safeAbsolutePath( + options.deploymentRoot, + 'deploymentRoot', + ), + allowRootService: options.allowRootService, + }), + request: Object.freeze({ + ...rollbackIdentity, + expectedPreparationDigest: request.expectedPreparationDigest, + expectedInstanceHeadDigest: request.expectedInstanceHeadDigest, + requestedAtMs: request.requestedAtMs as number, + }), + }); +} + +function normalizeConsumeCommand( + value: unknown, +): Readonly { + const command = object(value, 'legacy rollback consume command'); + exact( + command, + ['operation', 'options', 'request', 'schemaVersion'], + 'command', + ); + const options = object(command.options, 'options'); + exact(options, ['allowRootService', 'deploymentRoot'], 'options'); + const request = object(command.request, 'request'); + exact( + request, + [ + 'cutoverId', + 'expectedActivationDigest', + 'expectedAuthorizationDigest', + 'expectedAuthorizationHeadDigest', + 'expectedPreparationDigest', + 'generation', + 'instanceId', + 'profile', + 'requestedAtMs', + ], + 'request', + ); + const identity = currentIdentity(); + const rollbackIdentity = normalizeIdentity(request); + if ( + command.schemaVersion !== 1 || + command.operation !== + 'local.deployment.service-manager.legacy-rollback.consume' || + typeof options.allowRootService !== 'boolean' || + (identity.uid === 0) !== options.allowRootService || + typeof request.expectedPreparationDigest !== 'string' || + !DIGEST_PATTERN.test(request.expectedPreparationDigest) || + typeof request.expectedAuthorizationDigest !== 'string' || + !DIGEST_PATTERN.test(request.expectedAuthorizationDigest) || + typeof request.expectedAuthorizationHeadDigest !== 'string' || + !DIGEST_PATTERN.test(request.expectedAuthorizationHeadDigest) || + !Number.isSafeInteger(request.requestedAtMs) || + (request.requestedAtMs as number) < 0 + ) { + configurationError('legacy rollback consume command is invalid'); + } + return Object.freeze({ + schemaVersion: 1 as const, + operation: + 'local.deployment.service-manager.legacy-rollback.consume' as const, + options: Object.freeze({ + deploymentRoot: safeAbsolutePath( + options.deploymentRoot, + 'deploymentRoot', + ), + allowRootService: options.allowRootService, + }), + request: Object.freeze({ + ...rollbackIdentity, + expectedPreparationDigest: request.expectedPreparationDigest, + expectedAuthorizationDigest: request.expectedAuthorizationDigest, + expectedAuthorizationHeadDigest: request.expectedAuthorizationHeadDigest, + requestedAtMs: request.requestedAtMs as number, + }), + }); +} + +function readPreparation( + deploymentRoot: string, + identity: Readonly, +): Readonly { + return normalizeLocalServiceManagerLegacyRollbackPreparation( + readPrivateLocalJsonFile( + localServiceManagerLegacyRollbackPreparationPath( + deploymentRoot, + identity.cutoverId, + identity.generation, + ), + { maxBytes: MAX_PRIVATE_LOCAL_JSON_FILE_BYTES }, + ), + ); +} + +function assertPreparationBinding( + preparation: Readonly, + identity: Readonly, + expectedPreparationDigest: string, +): void { + if ( + preparation.cutoverId !== identity.cutoverId || + preparation.profile !== identity.profile || + preparation.instanceId !== identity.instanceId || + preparation.generation !== identity.generation || + preparation.activationDigest !== identity.expectedActivationDigest || + preparation.preparationDigest !== expectedPreparationDigest + ) { + configurationError('legacy rollback preparation binding drifted'); + } +} + +function authorizationRecord( + command: Readonly, + preparation: Readonly, +): Readonly { + const payload = Object.freeze({ + schema: + 'qinglong3-local-service-manager-legacy-start-authorization' as const, + schemaVersion: 1 as const, + state: 'legacy_restart_requested' as const, + cutoverId: command.request.cutoverId, + profile: command.request.profile, + instanceId: command.request.instanceId, + generation: command.request.generation, + activationDigest: command.request.expectedActivationDigest, + managerKind: preparation.managerKind, + expectedInstanceHeadDigest: command.request.expectedInstanceHeadDigest, + preparationDigest: preparation.preparationDigest, + reconciliationEvidenceDigest: preparation.reconciliation.evidenceDigest, + applicationConfigDigest: preparation.applicationConfigDigest, + commitmentFileDigest: preparation.commitmentFileDigest, + targetDescriptorDigest: preparation.targetDescriptorDigest, + requestedAtMs: command.request.requestedAtMs, + }); + return normalizeLocalServiceManagerLegacyStartAuthorization({ + ...payload, + authorizationDigest: + localServiceManagerLegacyStartAuthorizationDigest(payload), + }); +} + +function authorizeResult( + command: Readonly, + status: 'prepared' | 'existing', + preparationDigest: string, + authorizationDigest: string, + instanceHeadDigest: string, +): Readonly { + return Object.freeze({ + schemaVersion: 1 as const, + operation: command.operation, + status, + state: 'legacy_restart_requested' as const, + cutoverId: command.request.cutoverId, + generation: command.request.generation, + preparationDigest, + authorizationDigest, + instanceHeadDigest, + }); +} + +export function authorizeLocalServiceManagerLegacyRollback( + input: unknown, +): Readonly { + const command = normalizeAuthorizeCommand(input); + const identity = currentIdentity(); + const preparation = readPreparation( + command.options.deploymentRoot, + command.request, + ); + assertPreparationBinding( + preparation, + command.request, + command.request.expectedPreparationDigest, + ); + const authorization = authorizationRecord(command, preparation); + const authorizationPath = localServiceManagerLegacyStartAuthorizationPath( + command.options.deploymentRoot, + command.request.cutoverId, + command.request.generation, + ); + const head = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + command.request.instanceId, + identity.uid, + ); + if (head.state === 'legacy_restart_requested') { + const existing = normalizeLocalServiceManagerLegacyStartAuthorization( + readPrivateLocalJsonFile(authorizationPath, { + maxBytes: MAX_PRIVATE_LOCAL_JSON_FILE_BYTES, + }), + ); + if ( + existing.authorizationDigest !== authorization.authorizationDigest || + head.sourceRecordDigest !== existing.authorizationDigest || + head.previousHeadDigest !== command.request.expectedInstanceHeadDigest + ) { + configurationError('legacy rollback authorization replay drifted'); + } + return authorizeResult( + command, + 'existing', + preparation.preparationDigest, + existing.authorizationDigest, + head.headDigest, + ); + } + if ( + head.state !== 'rollback_prepared' || + head.headDigest !== command.request.expectedInstanceHeadDigest || + head.sourceRecordDigest !== preparation.preparationDigest || + head.profile !== command.request.profile || + head.cutoverId !== command.request.cutoverId || + head.activationDigest !== command.request.expectedActivationDigest || + head.generation !== command.request.generation || + preparation.requestedAtMs > command.request.requestedAtMs + ) { + configurationError('legacy rollback authorization lost the instance head'); + } + const revalidated = prepareLocalServiceManagerLegacyRollback({ + schemaVersion: 1, + operation: 'local.deployment.service-manager.legacy-rollback.prepare', + options: command.options, + request: { + cutoverId: preparation.cutoverId, + profile: preparation.profile, + instanceId: preparation.instanceId, + generation: preparation.generation, + expectedActivationDigest: preparation.activationDigest, + expectedStoppedRecordDigest: preparation.stoppedRecordDigest, + expectedInstanceHeadDigest: preparation.expectedInstanceHeadDigest, + requestedAtMs: preparation.requestedAtMs, + }, + }); + if ( + revalidated.status !== 'existing' || + revalidated.state !== 'rollback_prepared' || + revalidated.preparationDigest !== preparation.preparationDigest + ) { + configurationError('legacy rollback preparation cannot authorize start'); + } + const contents = `${JSON.stringify(authorization, null, 2)}\n`; + preflightPublishedFile( + authorizationPath, + contents, + 0o600, + identity.uid, + 'legacy start authorization', + ); + const status = publishExactFile( + authorizationPath, + contents, + 0o600, + identity.uid, + 'legacy start authorization', + ); + const next = advanceLocalCutoverInstanceHead( + { + options: { deploymentRoot: command.options.deploymentRoot }, + request: command.request, + }, + identity.uid, + 'legacy_restart_requested', + command.request.generation, + authorization.authorizationDigest, + ); + return authorizeResult( + command, + status, + preparation.preparationDigest, + authorization.authorizationDigest, + next.headDigest, + ); +} + +function completionRecord( + command: Readonly, + outcome: Readonly, +): Readonly { + const payload = Object.freeze({ + schema: COMPLETION_SCHEMA, + schemaVersion: 1 as const, + state: outcome.state, + cutoverId: command.request.cutoverId, + profile: command.request.profile, + instanceId: command.request.instanceId, + generation: command.request.generation, + activationDigest: command.request.expectedActivationDigest, + preparationDigest: command.request.expectedPreparationDigest, + authorizationDigest: command.request.expectedAuthorizationDigest, + expectedAuthorizationHeadDigest: + command.request.expectedAuthorizationHeadDigest, + rootOutcomeDigest: outcome.outcomeDigest, + rootBarrierDigest: outcome.barrierDigest, + legacyDescriptorDigest: outcome.legacyDescriptorDigest, + targetDescriptorDigest: outcome.targetDescriptorDigest, + legacyObservationDigest: outcome.legacyObservation.observationDigest, + targetObservationDigest: outcome.targetObservation.observationDigest, + manualReason: outcome.manualReason, + completedAtMs: Math.max( + command.request.requestedAtMs, + outcome.completedAtMs, + ), + }); + return Object.freeze({ + ...payload, + completionDigest: cutoverDigest(payload), + }); +} + +export function normalizeLocalServiceManagerLegacyRollbackCompletion( + value: unknown, +): Readonly { + const record = object(value, 'legacy rollback completion'); + exact( + record, + [ + 'activationDigest', + 'authorizationDigest', + 'completedAtMs', + 'completionDigest', + 'cutoverId', + 'expectedAuthorizationHeadDigest', + 'generation', + 'instanceId', + 'legacyDescriptorDigest', + 'legacyObservationDigest', + 'manualReason', + 'preparationDigest', + 'profile', + 'rootBarrierDigest', + 'rootOutcomeDigest', + 'schema', + 'schemaVersion', + 'state', + 'targetDescriptorDigest', + 'targetObservationDigest', + ], + 'legacy rollback completion', + ); + const { completionDigest, ...payload } = record; + const digests = [ + record.activationDigest, + record.preparationDigest, + record.authorizationDigest, + record.expectedAuthorizationHeadDigest, + record.rootOutcomeDigest, + record.rootBarrierDigest, + record.legacyDescriptorDigest, + record.targetDescriptorDigest, + record.legacyObservationDigest, + record.targetObservationDigest, + ]; + if ( + record.schema !== COMPLETION_SCHEMA || + record.schemaVersion !== 1 || + (record.state !== 'legacy_running' && record.state !== 'manual_required') || + typeof record.cutoverId !== 'string' || + !CUTOVER_ID_PATTERN.test(record.cutoverId) || + (record.profile !== 'edge' && record.profile !== 'standalone') || + typeof record.instanceId !== 'string' || + !INSTANCE_ID_PATTERN.test(record.instanceId) || + !Number.isSafeInteger(record.generation) || + (record.generation as number) < 1 || + (record.generation as number) > MAX_GENERATION || + digests.some( + (candidate) => + typeof candidate !== 'string' || !DIGEST_PATTERN.test(candidate), + ) || + (record.manualReason !== null && + record.manualReason !== 'start_precondition_unproved' && + record.manualReason !== 'authorization_material_drifted' && + record.manualReason !== 'manager_command_failed' && + record.manualReason !== 'manager_state_unproved' && + record.manualReason !== 'service_descriptor_drifted') || + (record.state === 'manual_required') !== (record.manualReason !== null) || + !Number.isSafeInteger(record.completedAtMs) || + (record.completedAtMs as number) < 0 || + typeof completionDigest !== 'string' || + !DIGEST_PATTERN.test(completionDigest) || + cutoverDigest(payload) !== completionDigest + ) { + configurationError('legacy rollback completion drifted'); + } + return record as unknown as Readonly; +} + +function consumeResult( + command: Readonly, + status: 'prepared' | 'existing', + completion: Readonly, + instanceHeadDigest: string, +): Readonly { + return Object.freeze({ + schemaVersion: 1 as const, + operation: command.operation, + status, + state: completion.state, + cutoverId: completion.cutoverId, + generation: completion.generation, + outcomeDigest: completion.rootOutcomeDigest, + completionDigest: completion.completionDigest, + instanceHeadDigest, + }); +} + +export function consumeLocalServiceManagerLegacyRollback( + input: unknown, +): Readonly { + const command = normalizeConsumeCommand(input); + const identity = currentIdentity(); + const preparation = readPreparation( + command.options.deploymentRoot, + command.request, + ); + assertPreparationBinding( + preparation, + command.request, + command.request.expectedPreparationDigest, + ); + const authorization = normalizeLocalServiceManagerLegacyStartAuthorization( + readPrivateLocalJsonFile( + localServiceManagerLegacyStartAuthorizationPath( + command.options.deploymentRoot, + command.request.cutoverId, + command.request.generation, + ), + { maxBytes: MAX_PRIVATE_LOCAL_JSON_FILE_BYTES }, + ), + ); + if ( + authorization.cutoverId !== command.request.cutoverId || + authorization.profile !== command.request.profile || + authorization.instanceId !== command.request.instanceId || + authorization.generation !== command.request.generation || + authorization.activationDigest !== + command.request.expectedActivationDigest || + authorization.preparationDigest !== preparation.preparationDigest || + authorization.authorizationDigest !== + command.request.expectedAuthorizationDigest + ) { + configurationError('legacy start authorization binding drifted'); + } + const head = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + command.request.instanceId, + identity.uid, + ); + const completionPath = localServiceManagerLegacyCompletionPath( + command.options.deploymentRoot, + command.request.cutoverId, + command.request.generation, + ); + if (head.state === 'legacy_running' || head.state === 'manual_required') { + const existing = normalizeLocalServiceManagerLegacyRollbackCompletion( + readPrivateLocalJsonFile(completionPath, { + maxBytes: MAX_PRIVATE_LOCAL_JSON_FILE_BYTES, + }), + ); + if ( + existing.state !== head.state || + existing.authorizationDigest !== authorization.authorizationDigest || + existing.expectedAuthorizationHeadDigest !== + command.request.expectedAuthorizationHeadDigest || + head.sourceRecordDigest !== existing.completionDigest || + head.previousHeadDigest !== + command.request.expectedAuthorizationHeadDigest + ) { + configurationError('legacy rollback completion replay drifted'); + } + return consumeResult(command, 'existing', existing, head.headDigest); + } + if ( + head.state !== 'legacy_restart_requested' || + head.headDigest !== command.request.expectedAuthorizationHeadDigest || + head.sourceRecordDigest !== authorization.authorizationDigest || + head.profile !== command.request.profile || + head.cutoverId !== command.request.cutoverId || + head.activationDigest !== command.request.expectedActivationDigest || + head.generation !== command.request.generation || + authorization.requestedAtMs > command.request.requestedAtMs + ) { + configurationError('legacy rollback consume lost the instance head'); + } + const outcome = normalizeLocalServiceManagerLegacyStartOutcome( + readPrivateLocalJsonFile( + localServiceManagerLegacyStartOutcomePath( + command.options.deploymentRoot, + command.request.cutoverId, + command.request.generation, + ), + { maxBytes: MAX_PRIVATE_LOCAL_JSON_FILE_BYTES }, + ), + ); + if ( + outcome.cutoverId !== command.request.cutoverId || + outcome.profile !== command.request.profile || + outcome.instanceId !== command.request.instanceId || + outcome.generation !== command.request.generation || + outcome.activationDigest !== command.request.expectedActivationDigest || + outcome.managerKind !== preparation.managerKind || + outcome.preparationDigest !== preparation.preparationDigest || + outcome.authorizationDigest !== authorization.authorizationDigest || + outcome.targetDescriptorDigest !== preparation.targetDescriptorDigest || + outcome.completedAtMs > command.request.requestedAtMs + ) { + configurationError('legacy start outcome binding drifted'); + } + const completion = completionRecord(command, outcome); + const contents = `${JSON.stringify(completion, null, 2)}\n`; + preflightPublishedFile( + completionPath, + contents, + 0o600, + identity.uid, + 'legacy rollback completion', + ); + const status = publishExactFile( + completionPath, + contents, + 0o600, + identity.uid, + 'legacy rollback completion', + ); + const next = advanceLocalCutoverInstanceHead( + { + options: { deploymentRoot: command.options.deploymentRoot }, + request: command.request, + }, + identity.uid, + completion.state, + command.request.generation, + completion.completionDigest, + ); + return consumeResult(command, status, completion, next.headDigest); +} + +export function authorizeLocalServiceManagerLegacyRollbackCommandFile( + filePath: string, +): Readonly { + return authorizeLocalServiceManagerLegacyRollback( + readPrivateLocalJsonFile(filePath, { + maxBytes: MAX_PRIVATE_LOCAL_JSON_FILE_BYTES, + }), + ); +} + +export function consumeLocalServiceManagerLegacyRollbackCommandFile( + filePath: string, +): Readonly { + return consumeLocalServiceManagerLegacyRollback( + readPrivateLocalJsonFile(filePath, { + maxBytes: MAX_PRIVATE_LOCAL_JSON_FILE_BYTES, + }), + ); +} diff --git a/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/contract.ts b/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/contract.ts new file mode 100644 index 00000000..d504da16 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/contract.ts @@ -0,0 +1,575 @@ +import crypto from 'node:crypto'; +import path from 'node:path'; + +import { LocalDeploymentConfigurationError } from '../../foundation/error'; +import type { + LocalServiceBridgeManager, + LocalServiceManagerKind, +} from '../serviceBridgeContract'; + +const AUTHORIZATION_SCHEMA = + 'qinglong3-local-service-manager-legacy-start-authorization'; +const OUTCOME_SCHEMA = 'qinglong3-local-service-manager-legacy-start-outcome'; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const INSTANCE_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/; +const CUTOVER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const SAFE_PATH_PATTERN = /^\/[A-Za-z0-9._/@-]+$/; +const MAX_PATH_BYTES = 4_096; +const MAX_GENERATION = 15; +const MAX_PID = 0x7fffffff; + +export type LocalServiceManagerRollbackMutationDisposition = + | 'executed' + | 'response-loss-inspected' + | 'replay-inspected'; + +export type LocalServiceManagerRollbackManualReason = + | 'start_precondition_unproved' + | 'authorization_material_drifted' + | 'manager_command_failed' + | 'manager_state_unproved' + | 'service_descriptor_drifted'; + +export interface LocalServiceManagerLegacyStartAuthorization { + readonly schema: typeof AUTHORIZATION_SCHEMA; + readonly schemaVersion: 1; + readonly state: 'legacy_restart_requested'; + readonly cutoverId: string; + readonly profile: 'edge' | 'standalone'; + readonly instanceId: string; + readonly generation: number; + readonly activationDigest: string; + readonly managerKind: LocalServiceManagerKind; + readonly expectedInstanceHeadDigest: string; + readonly preparationDigest: string; + readonly reconciliationEvidenceDigest: string; + readonly applicationConfigDigest: string; + readonly commitmentFileDigest: string; + readonly targetDescriptorDigest: string; + readonly requestedAtMs: number; + readonly authorizationDigest: string; +} + +export interface LocalServiceManagerRollbackObservation { + readonly managerKind: LocalServiceManagerKind; + readonly serviceName: 'qinglong' | 'qinglong3'; + readonly fragmentPath: string; + readonly loadState: 'loaded' | 'not-found' | 'unknown'; + readonly activeState: 'active' | 'inactive' | 'failed' | 'unknown'; + readonly subState: string; + readonly enabledState: 'enabled' | 'disabled' | 'static' | 'unknown'; + readonly mainPid: number; + readonly observedAtMs: number; + readonly observationDigest: string; +} + +export interface LocalServiceManagerLegacyStartOutcome { + readonly schema: typeof OUTCOME_SCHEMA; + readonly schemaVersion: 1; + readonly state: 'legacy_running' | 'manual_required'; + readonly cutoverId: string; + readonly profile: 'edge' | 'standalone'; + readonly instanceId: string; + readonly generation: number; + readonly activationDigest: string; + readonly managerKind: LocalServiceManagerKind; + readonly preparationDigest: string; + readonly authorizationDigest: string; + readonly barrierDigest: string; + readonly legacyDescriptorDigest: string; + readonly targetDescriptorDigest: string; + readonly mutationDisposition: LocalServiceManagerRollbackMutationDisposition; + readonly manualReason: LocalServiceManagerRollbackManualReason | null; + readonly legacyObservation: Readonly; + readonly targetObservation: Readonly; + readonly completedAtMs: number; + readonly outcomeDigest: string; +} + +export interface LocalServiceManagerLegacyRollbackBridgeCommand { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.service-manager.legacy-rollback.execute'; + readonly options: Readonly<{ + deploymentRoot: string; + controllerRoot: string; + allowRootController: true; + manager: LocalServiceBridgeManager; + }>; + readonly request: Readonly<{ + cutoverId: string; + generation: number; + expectedAuthorizationDigest: string; + }>; +} + +export interface LocalServiceManagerLegacyRollbackBridgeResult { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.service-manager.legacy-rollback.execute'; + readonly status: 'prepared' | 'existing'; + readonly state: LocalServiceManagerLegacyStartOutcome['state']; + readonly cutoverId: string; + readonly generation: number; + readonly outcomeDigest: string; +} + +function configurationError(message: string): never { + throw new LocalDeploymentConfigurationError(message); +} + +function object(value: unknown, label: string): Record { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + (Object.getPrototypeOf(value) !== Object.prototype && + Object.getPrototypeOf(value) !== null) + ) { + configurationError(`${label} must be an object`); + } + return value as Record; +} + +function exact( + value: Record, + keys: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + configurationError(`${label} shape is invalid`); + } +} + +function digest(value: unknown): string { + return crypto + .createHash('sha256') + .update(JSON.stringify(value)) + .digest('hex'); +} + +function safeAbsolutePath(value: unknown, label: string): string { + if ( + typeof value !== 'string' || + !path.isAbsolute(value) || + path.normalize(value) !== value || + path.parse(value).root === value || + value.includes('\0') || + value.includes('//') || + !SAFE_PATH_PATTERN.test(value) || + Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES + ) { + configurationError(`${label} must be a supervisor-safe absolute path`); + } + return value; +} + +function normalizedManager(value: unknown): LocalServiceBridgeManager { + const manager = object(value, 'manager'); + if (manager.kind === 'systemd') { + exact(manager, ['executable', 'kind'], 'manager'); + return Object.freeze({ + kind: 'systemd' as const, + executable: safeAbsolutePath(manager.executable, 'systemd executable'), + }); + } + exact(manager, ['kind', 'serviceExecutable', 'updateExecutable'], 'manager'); + if (manager.kind !== 'openrc') configurationError('manager kind is invalid'); + return Object.freeze({ + kind: 'openrc' as const, + serviceExecutable: safeAbsolutePath( + manager.serviceExecutable, + 'OpenRC service executable', + ), + updateExecutable: safeAbsolutePath( + manager.updateExecutable, + 'OpenRC update executable', + ), + }); +} + +export function localServiceManagerLegacyStartAuthorizationPath( + deploymentRoot: string, + cutoverId: string, + generation: number, +): string { + return path.join( + deploymentRoot, + 'service', + 'cutovers', + cutoverId, + `service-manager-g${String(generation).padStart( + 2, + '0', + )}-legacy-start-intent.json`, + ); +} + +export function localServiceManagerLegacyStartOutcomePath( + deploymentRoot: string, + cutoverId: string, + generation: number, +): string { + return path.join( + deploymentRoot, + 'service', + 'cutovers', + cutoverId, + `service-manager-g${String(generation).padStart( + 2, + '0', + )}-legacy-start-outcome.json`, + ); +} + +export function localServiceManagerLegacyCompletionPath( + deploymentRoot: string, + cutoverId: string, + generation: number, +): string { + return path.join( + deploymentRoot, + 'service', + 'cutovers', + cutoverId, + `service-manager-g${String(generation).padStart( + 2, + '0', + )}-legacy-completion.json`, + ); +} + +export function localServiceManagerLegacyDescriptorPath( + kind: LocalServiceManagerKind, +): string { + return kind === 'systemd' + ? '/etc/systemd/system/qinglong.service' + : '/etc/init.d/qinglong'; +} + +export function localServiceManagerTargetDescriptorPath( + kind: LocalServiceManagerKind, +): string { + return kind === 'systemd' + ? '/etc/systemd/system/qinglong3.service' + : '/etc/init.d/qinglong3'; +} + +export function localServiceManagerLegacyStartAuthorizationDigest( + value: Omit< + LocalServiceManagerLegacyStartAuthorization, + 'authorizationDigest' + >, +): string { + return digest(value); +} + +export function normalizeLocalServiceManagerLegacyStartAuthorization( + value: unknown, +): Readonly { + const record = object(value, 'legacy start authorization'); + exact( + record, + [ + 'activationDigest', + 'applicationConfigDigest', + 'authorizationDigest', + 'commitmentFileDigest', + 'cutoverId', + 'expectedInstanceHeadDigest', + 'generation', + 'instanceId', + 'managerKind', + 'preparationDigest', + 'profile', + 'reconciliationEvidenceDigest', + 'requestedAtMs', + 'schema', + 'schemaVersion', + 'state', + 'targetDescriptorDigest', + ], + 'legacy start authorization', + ); + const { authorizationDigest, ...payload } = record; + if ( + record.schema !== AUTHORIZATION_SCHEMA || + record.schemaVersion !== 1 || + record.state !== 'legacy_restart_requested' || + typeof record.cutoverId !== 'string' || + !CUTOVER_ID_PATTERN.test(record.cutoverId) || + (record.profile !== 'edge' && record.profile !== 'standalone') || + typeof record.instanceId !== 'string' || + !INSTANCE_ID_PATTERN.test(record.instanceId) || + !Number.isSafeInteger(record.generation) || + (record.generation as number) < 1 || + (record.generation as number) > MAX_GENERATION || + typeof record.activationDigest !== 'string' || + !DIGEST_PATTERN.test(record.activationDigest) || + (record.managerKind !== 'systemd' && record.managerKind !== 'openrc') || + typeof record.expectedInstanceHeadDigest !== 'string' || + !DIGEST_PATTERN.test(record.expectedInstanceHeadDigest) || + typeof record.preparationDigest !== 'string' || + !DIGEST_PATTERN.test(record.preparationDigest) || + typeof record.reconciliationEvidenceDigest !== 'string' || + !DIGEST_PATTERN.test(record.reconciliationEvidenceDigest) || + typeof record.applicationConfigDigest !== 'string' || + !DIGEST_PATTERN.test(record.applicationConfigDigest) || + typeof record.commitmentFileDigest !== 'string' || + !DIGEST_PATTERN.test(record.commitmentFileDigest) || + typeof record.targetDescriptorDigest !== 'string' || + !DIGEST_PATTERN.test(record.targetDescriptorDigest) || + !Number.isSafeInteger(record.requestedAtMs) || + (record.requestedAtMs as number) < 0 || + typeof authorizationDigest !== 'string' || + !DIGEST_PATTERN.test(authorizationDigest) || + digest(payload) !== authorizationDigest + ) { + configurationError('legacy start authorization drifted'); + } + return record as unknown as Readonly; +} + +export function localServiceManagerRollbackObservationDigest( + value: Omit, +): string { + return digest(value); +} + +export function normalizeLocalServiceManagerRollbackObservation( + value: unknown, +): Readonly { + const observation = object(value, 'legacy rollback observation'); + exact( + observation, + [ + 'activeState', + 'enabledState', + 'fragmentPath', + 'loadState', + 'mainPid', + 'managerKind', + 'observationDigest', + 'observedAtMs', + 'serviceName', + 'subState', + ], + 'legacy rollback observation', + ); + const payload = Object.freeze({ + managerKind: observation.managerKind, + serviceName: observation.serviceName, + fragmentPath: safeAbsolutePath(observation.fragmentPath, 'fragmentPath'), + loadState: observation.loadState, + activeState: observation.activeState, + subState: observation.subState, + enabledState: observation.enabledState, + mainPid: observation.mainPid, + observedAtMs: observation.observedAtMs, + }); + if ( + (observation.managerKind !== 'systemd' && + observation.managerKind !== 'openrc') || + (observation.serviceName !== 'qinglong' && + observation.serviceName !== 'qinglong3') || + (observation.loadState !== 'loaded' && + observation.loadState !== 'not-found' && + observation.loadState !== 'unknown') || + (observation.activeState !== 'active' && + observation.activeState !== 'inactive' && + observation.activeState !== 'failed' && + observation.activeState !== 'unknown') || + typeof observation.subState !== 'string' || + Buffer.byteLength(observation.subState, 'utf8') > 128 || + (observation.enabledState !== 'enabled' && + observation.enabledState !== 'disabled' && + observation.enabledState !== 'static' && + observation.enabledState !== 'unknown') || + !Number.isSafeInteger(observation.mainPid) || + (observation.mainPid as number) < 0 || + (observation.mainPid as number) > MAX_PID || + !Number.isSafeInteger(observation.observedAtMs) || + (observation.observedAtMs as number) < 0 || + typeof observation.observationDigest !== 'string' || + !DIGEST_PATTERN.test(observation.observationDigest) || + digest(payload) !== observation.observationDigest + ) { + configurationError('legacy rollback observation is invalid'); + } + return Object.freeze({ + ...(payload as Omit< + LocalServiceManagerRollbackObservation, + 'observationDigest' + >), + observationDigest: observation.observationDigest, + }); +} + +export function localServiceManagerLegacyStartOutcomeDigest( + value: Omit, +): string { + return digest(value); +} + +export function normalizeLocalServiceManagerLegacyStartOutcome( + value: unknown, +): Readonly { + const record = object(value, 'legacy start outcome'); + exact( + record, + [ + 'activationDigest', + 'authorizationDigest', + 'barrierDigest', + 'completedAtMs', + 'cutoverId', + 'generation', + 'instanceId', + 'legacyDescriptorDigest', + 'legacyObservation', + 'managerKind', + 'manualReason', + 'mutationDisposition', + 'outcomeDigest', + 'preparationDigest', + 'profile', + 'schema', + 'schemaVersion', + 'state', + 'targetDescriptorDigest', + 'targetObservation', + ], + 'legacy start outcome', + ); + const legacyObservation = normalizeLocalServiceManagerRollbackObservation( + record.legacyObservation, + ); + const targetObservation = normalizeLocalServiceManagerRollbackObservation( + record.targetObservation, + ); + const { outcomeDigest, ...rawPayload } = record; + const payload = { + ...rawPayload, + legacyObservation, + targetObservation, + }; + if ( + record.schema !== OUTCOME_SCHEMA || + record.schemaVersion !== 1 || + (record.state !== 'legacy_running' && record.state !== 'manual_required') || + typeof record.cutoverId !== 'string' || + !CUTOVER_ID_PATTERN.test(record.cutoverId) || + (record.profile !== 'edge' && record.profile !== 'standalone') || + typeof record.instanceId !== 'string' || + !INSTANCE_ID_PATTERN.test(record.instanceId) || + !Number.isSafeInteger(record.generation) || + (record.generation as number) < 1 || + (record.generation as number) > MAX_GENERATION || + typeof record.activationDigest !== 'string' || + !DIGEST_PATTERN.test(record.activationDigest) || + (record.managerKind !== 'systemd' && record.managerKind !== 'openrc') || + typeof record.preparationDigest !== 'string' || + !DIGEST_PATTERN.test(record.preparationDigest) || + typeof record.authorizationDigest !== 'string' || + !DIGEST_PATTERN.test(record.authorizationDigest) || + typeof record.barrierDigest !== 'string' || + !DIGEST_PATTERN.test(record.barrierDigest) || + typeof record.legacyDescriptorDigest !== 'string' || + !DIGEST_PATTERN.test(record.legacyDescriptorDigest) || + typeof record.targetDescriptorDigest !== 'string' || + !DIGEST_PATTERN.test(record.targetDescriptorDigest) || + (record.mutationDisposition !== 'executed' && + record.mutationDisposition !== 'response-loss-inspected' && + record.mutationDisposition !== 'replay-inspected') || + (record.manualReason !== null && + record.manualReason !== 'start_precondition_unproved' && + record.manualReason !== 'authorization_material_drifted' && + record.manualReason !== 'manager_command_failed' && + record.manualReason !== 'manager_state_unproved' && + record.manualReason !== 'service_descriptor_drifted') || + (record.state === 'manual_required') !== (record.manualReason !== null) || + legacyObservation.managerKind !== record.managerKind || + legacyObservation.serviceName !== 'qinglong' || + targetObservation.managerKind !== record.managerKind || + targetObservation.serviceName !== 'qinglong3' || + !Number.isSafeInteger(record.completedAtMs) || + (record.completedAtMs as number) < legacyObservation.observedAtMs || + (record.completedAtMs as number) < targetObservation.observedAtMs || + typeof outcomeDigest !== 'string' || + !DIGEST_PATTERN.test(outcomeDigest) || + digest(payload) !== outcomeDigest + ) { + configurationError('legacy start outcome drifted'); + } + return Object.freeze({ + ...(payload as Omit< + LocalServiceManagerLegacyStartOutcome, + 'outcomeDigest' + >), + outcomeDigest, + }); +} + +export function normalizeLocalServiceManagerLegacyRollbackBridgeCommand( + value: unknown, +): Readonly { + const command = object(value, 'legacy rollback bridge command'); + exact( + command, + ['operation', 'options', 'request', 'schemaVersion'], + 'legacy rollback bridge command', + ); + const options = object(command.options, 'options'); + exact( + options, + ['allowRootController', 'controllerRoot', 'deploymentRoot', 'manager'], + 'options', + ); + const request = object(command.request, 'request'); + exact( + request, + ['cutoverId', 'expectedAuthorizationDigest', 'generation'], + 'request', + ); + const manager = normalizedManager(options.manager); + if ( + command.schemaVersion !== 1 || + command.operation !== + 'local.deployment.service-manager.legacy-rollback.execute' || + options.allowRootController !== true || + typeof request.cutoverId !== 'string' || + !CUTOVER_ID_PATTERN.test(request.cutoverId) || + !Number.isSafeInteger(request.generation) || + (request.generation as number) < 1 || + (request.generation as number) > MAX_GENERATION || + typeof request.expectedAuthorizationDigest !== 'string' || + !DIGEST_PATTERN.test(request.expectedAuthorizationDigest) + ) { + configurationError('legacy rollback bridge command is invalid'); + } + return Object.freeze({ + schemaVersion: 1 as const, + operation: + 'local.deployment.service-manager.legacy-rollback.execute' as const, + options: Object.freeze({ + deploymentRoot: safeAbsolutePath( + options.deploymentRoot, + 'deploymentRoot', + ), + controllerRoot: safeAbsolutePath( + options.controllerRoot, + 'controllerRoot', + ), + allowRootController: true as const, + manager, + }), + request: Object.freeze({ + cutoverId: request.cutoverId, + generation: request.generation as number, + expectedAuthorizationDigest: request.expectedAuthorizationDigest, + }), + }); +} diff --git a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceLegacyRollback.ts b/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/preparation.ts similarity index 90% rename from packages/ql3-local-owner-cli/src/deployment/service-manager/serviceLegacyRollback.ts rename to packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/preparation.ts index 4da7731d..d1df62d8 100644 --- a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceLegacyRollback.ts +++ b/packages/ql3-local-owner-cli/src/deployment/service-manager/legacy-rollback/preparation.ts @@ -1,5 +1,4 @@ import crypto from 'node:crypto'; -import fs from 'node:fs'; import path from 'node:path'; import { @@ -11,21 +10,26 @@ import { import { advanceLocalCutoverInstanceHead, readLocalCutoverInstanceHead, -} from '../cutover/instanceLineage'; +} from '../../cutover/instanceLineage'; import { readTargetDataReconciliationEvidenceForPaths, verifyTargetDataReconciliationEvidence, type TargetDataReconciliationEvidence, -} from '../cutover/targetDataEvidence'; -import { cutoverDigest } from '../cutover/targetEvidence'; -import { currentIdentity } from '../foundation/contract'; -import { LocalDeploymentConfigurationError } from '../foundation/error'; -import { preflightPublishedFile, publishExactFile } from '../foundation/files'; +} from '../../cutover/targetDataEvidence'; +import { cutoverDigest } from '../../cutover/targetEvidence'; +import { currentIdentity } from '../../foundation/contract'; +import { LocalDeploymentConfigurationError } from '../../foundation/error'; +import { + preflightPublishedFile, + publishExactFile, +} from '../../foundation/files'; import { localServiceManagerIntentPath, normalizeLocalServiceManagerIntent, -} from './serviceBridgeContract'; -import { normalizeLocalServiceManagerCutoverRecord } from './serviceCutoverJournal'; + type LocalServiceManagerKind, +} from '../serviceBridgeContract'; +import { readServiceBridgeFile } from '../serviceBridgeFiles'; +import { normalizeLocalServiceManagerCutoverRecord } from '../serviceCutoverJournal'; const SCHEMA = 'qinglong3-local-service-manager-rollback-preparation'; const ZERO_DIGEST = '0'.repeat(64); @@ -78,11 +82,14 @@ export interface LocalServiceManagerLegacyRollbackPreparation { readonly instanceId: string; readonly generation: number; readonly activationDigest: string; + readonly managerKind: LocalServiceManagerKind; readonly expectedInstanceHeadDigest: string; readonly stoppedRecordDigest: string; readonly stoppedManagerOutcomeDigest: string; readonly applicationConfigDigest: string; readonly commitmentDigest: string; + readonly commitmentFileDigest: string; + readonly targetDescriptorDigest: string; readonly shutdownReceiptDigest: string; readonly reconciliation: Readonly; readonly requestedAtMs: number; @@ -95,6 +102,7 @@ interface AdoptedPaths { readonly targetDatabasePath: string; readonly expectedActivationDigest: string; readonly commitmentDigest: string; + readonly commitmentFileDigest: string; } function configurationError(message: string, cause?: unknown): never { @@ -253,17 +261,20 @@ export function localServiceManagerLegacyRollbackPreparationPath( ); } -function applicationSha256(filePath: string): string { +function privateFileSha256( + filePath: string, + uid: number, + gid: number, + label: string, +): string { + let bytes: Buffer | undefined; try { - return crypto - .createHash('sha256') - .update(fs.readFileSync(filePath)) - .digest('hex'); + bytes = readServiceBridgeFile(filePath, { uid, gid, mode: 0o600 }, label); + return crypto.createHash('sha256').update(bytes).digest('hex'); } catch (error) { - return configurationError( - 'application configuration cannot be hashed', - error, - ); + return configurationError(`${label} cannot be hashed`, error); + } finally { + bytes?.fill(0); } } @@ -272,8 +283,17 @@ function adoptedPaths( command: Readonly, expectedApplicationDigest: string, expectedCommitmentDigest: string, + uid: number, + gid: number, ): Readonly { - if (applicationSha256(applicationPath) !== expectedApplicationDigest) { + if ( + privateFileSha256( + applicationPath, + uid, + gid, + 'application configuration', + ) !== expectedApplicationDigest + ) { configurationError('adopted application configuration digest drifted'); } const application = object( @@ -329,6 +349,12 @@ function adoptedPaths( ), expectedActivationDigest: command.request.expectedActivationDigest, commitmentDigest: expectedCommitmentDigest, + commitmentFileDigest: privateFileSha256( + commitmentPath, + uid, + gid, + 'legacy silence commitment', + ), }); } @@ -336,6 +362,9 @@ function preparationRecord( command: Readonly, stopped: ReturnType, reconciliation: Readonly, + managerKind: LocalServiceManagerKind, + commitmentFileDigest: string, + targetDescriptorDigest: string, ): Readonly { if ( stopped.evidence.shutdownReceiptDigest === null || @@ -352,11 +381,14 @@ function preparationRecord( instanceId: command.request.instanceId, generation: command.request.generation, activationDigest: command.request.expectedActivationDigest, + managerKind, expectedInstanceHeadDigest: command.request.expectedInstanceHeadDigest, stoppedRecordDigest: stopped.recordDigest, stoppedManagerOutcomeDigest: stopped.evidence.managerOutcomeDigest, applicationConfigDigest: stopped.evidence.applicationConfigDigest, commitmentDigest: stopped.evidence.commitmentDigest, + commitmentFileDigest, + targetDescriptorDigest, shutdownReceiptDigest: stopped.evidence.shutdownReceiptDigest, reconciliation, requestedAtMs: command.request.requestedAtMs, @@ -377,10 +409,12 @@ export function normalizeLocalServiceManagerLegacyRollbackPreparation( 'activationDigest', 'applicationConfigDigest', 'commitmentDigest', + 'commitmentFileDigest', 'cutoverId', 'expectedInstanceHeadDigest', 'generation', 'instanceId', + 'managerKind', 'preparationDigest', 'profile', 'reconciliation', @@ -391,6 +425,7 @@ export function normalizeLocalServiceManagerLegacyRollbackPreparation( 'state', 'stoppedManagerOutcomeDigest', 'stoppedRecordDigest', + 'targetDescriptorDigest', ], 'service manager rollback preparation', ); @@ -413,6 +448,7 @@ export function normalizeLocalServiceManagerLegacyRollbackPreparation( (record.generation as number) > MAX_GENERATION || typeof record.activationDigest !== 'string' || !DIGEST_PATTERN.test(record.activationDigest) || + (record.managerKind !== 'systemd' && record.managerKind !== 'openrc') || typeof record.expectedInstanceHeadDigest !== 'string' || !DIGEST_PATTERN.test(record.expectedInstanceHeadDigest) || typeof record.stoppedRecordDigest !== 'string' || @@ -423,6 +459,10 @@ export function normalizeLocalServiceManagerLegacyRollbackPreparation( !DIGEST_PATTERN.test(record.applicationConfigDigest) || typeof record.commitmentDigest !== 'string' || !DIGEST_PATTERN.test(record.commitmentDigest) || + typeof record.commitmentFileDigest !== 'string' || + !DIGEST_PATTERN.test(record.commitmentFileDigest) || + typeof record.targetDescriptorDigest !== 'string' || + !DIGEST_PATTERN.test(record.targetDescriptorDigest) || typeof record.shutdownReceiptDigest !== 'string' || !DIGEST_PATTERN.test(record.shutdownReceiptDigest) || reconciliation.disposition !== 'rollback_candidate' || @@ -531,6 +571,8 @@ export function prepareLocalServiceManagerLegacyRollback( command, stopped.evidence.applicationConfigDigest, stopped.evidence.commitmentDigest, + identity.uid, + identity.gid, ); const reconciliation = readTargetDataReconciliationEvidenceForPaths( { @@ -555,7 +597,14 @@ export function prepareLocalServiceManagerLegacyRollback( head.headDigest, ); } - const preparation = preparationRecord(command, stopped, reconciliation); + const preparation = preparationRecord( + command, + stopped, + reconciliation, + intent.service.kind, + paths.commitmentFileDigest, + intent.descriptor.sha256, + ); const filePath = localServiceManagerLegacyRollbackPreparationPath( command.options.deploymentRoot, command.request.cutoverId, diff --git a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeCli.ts b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeCli.ts index 7375549e..02e7e5a9 100644 --- a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeCli.ts +++ b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeCli.ts @@ -1,6 +1,9 @@ #!/usr/bin/env node -import { runLocalServiceBridgeCommandFile } from './serviceBridge'; +import { readPrivateLocalCommandFile } from '@qinglong/local-command-file'; + +import { runLocalServiceBridge } from './serviceBridge'; +import { runLocalServiceManagerLegacyRollbackBridge } from './legacy-rollback/bridge'; const USAGE = 'Usage: ql3-service-bridge run --command-file /absolute/root-owned-command.json'; @@ -21,7 +24,15 @@ function main(argv: readonly string[]): void { return; } try { - const result = runLocalServiceBridgeCommandFile(argv[2]!); + const command = readPrivateLocalCommandFile(argv[2]!); + const operation = + command && typeof command === 'object' && !Array.isArray(command) + ? (command as Record).operation + : undefined; + const result = + operation === 'local.deployment.service-manager.legacy-rollback.execute' + ? runLocalServiceManagerLegacyRollbackBridge(command) + : runLocalServiceBridge(command); process.stdout.write(`${JSON.stringify(result)}\n`); if (result.state === 'manual_required') process.exitCode = 2; } catch (error) { diff --git a/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs b/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs index 8903faa5..01828bd7 100644 --- a/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs +++ b/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs @@ -11,7 +11,16 @@ const { const { localServiceManagerLegacyRollbackPreparationPath, prepareLocalServiceManagerLegacyRollback, -} = require('../dist/deployment/service-manager/serviceLegacyRollback.js'); +} = require('../dist/deployment/service-manager/legacy-rollback/preparation.js'); +const { + authorizeLocalServiceManagerLegacyRollback, + consumeLocalServiceManagerLegacyRollback, +} = require('../dist/deployment/service-manager/legacy-rollback/consumer.js'); +const { + localServiceManagerLegacyStartOutcomeDigest, + localServiceManagerLegacyStartOutcomePath, + localServiceManagerRollbackObservationDigest, +} = require('../dist/deployment/service-manager/legacy-rollback/contract.js'); const { prepareLocalServiceManagerIntent, } = require('../dist/deployment/service-manager/serviceManagerIntent.js'); @@ -384,6 +393,126 @@ function rollbackPrepareCommand(state, stoppedResult, head) { }; } +function rollbackAuthorizeCommand(state, prepared) { + return { + schemaVersion: 1, + operation: 'local.deployment.service-manager.legacy-rollback.authorize', + options: { + deploymentRoot: state.root, + allowRootService: process.getuid() === 0, + }, + request: { + cutoverId: state.cutoverId, + profile: 'edge', + instanceId: 'edge-router-1', + generation: 1, + expectedActivationDigest: state.activationDigest, + expectedPreparationDigest: prepared.preparationDigest, + expectedInstanceHeadDigest: prepared.instanceHeadDigest, + requestedAtMs: 1786416000500, + }, + }; +} + +function rollbackObservation(serviceName, active, observedAtMs) { + const payload = { + managerKind: 'systemd', + serviceName, + fragmentPath: `/etc/systemd/system/${serviceName}.service`, + loadState: 'loaded', + activeState: active ? 'active' : 'inactive', + subState: active ? 'running' : 'dead', + enabledState: 'enabled', + mainPid: active ? 5723 : 0, + observedAtMs, + }; + return { + ...payload, + observationDigest: localServiceManagerRollbackObservationDigest(payload), + }; +} + +function publishLegacyStartOutcome( + state, + prepared, + authorized, + outcomeState = 'legacy_running', +) { + const preparation = JSON.parse( + fs.readFileSync( + localServiceManagerLegacyRollbackPreparationPath( + state.root, + state.cutoverId, + 1, + ), + 'utf8', + ), + ); + const legacyObservation = rollbackObservation( + 'qinglong', + outcomeState === 'legacy_running', + 1786416000600, + ); + const targetObservation = rollbackObservation( + 'qinglong3', + false, + 1786416000601, + ); + const payload = { + schema: 'qinglong3-local-service-manager-legacy-start-outcome', + schemaVersion: 1, + state: outcomeState, + cutoverId: state.cutoverId, + profile: 'edge', + instanceId: 'edge-router-1', + generation: 1, + activationDigest: state.activationDigest, + managerKind: 'systemd', + preparationDigest: prepared.preparationDigest, + authorizationDigest: authorized.authorizationDigest, + barrierDigest: '9'.repeat(64), + legacyDescriptorDigest: 'a'.repeat(64), + targetDescriptorDigest: preparation.targetDescriptorDigest, + mutationDisposition: 'executed', + manualReason: + outcomeState === 'legacy_running' ? null : 'manager_state_unproved', + legacyObservation, + targetObservation, + completedAtMs: 1786416000610, + }; + const outcome = { + ...payload, + outcomeDigest: localServiceManagerLegacyStartOutcomeDigest(payload), + }; + writePrivate( + localServiceManagerLegacyStartOutcomePath(state.root, state.cutoverId, 1), + outcome, + ); + return outcome; +} + +function rollbackConsumeCommand(state, prepared, authorized) { + return { + schemaVersion: 1, + operation: 'local.deployment.service-manager.legacy-rollback.consume', + options: { + deploymentRoot: state.root, + allowRootService: process.getuid() === 0, + }, + request: { + cutoverId: state.cutoverId, + profile: 'edge', + instanceId: 'edge-router-1', + generation: 1, + expectedActivationDigest: state.activationDigest, + expectedPreparationDigest: prepared.preparationDigest, + expectedAuthorizationDigest: authorized.authorizationDigest, + expectedAuthorizationHeadDigest: authorized.instanceHeadDigest, + requestedAtMs: 1786416000700, + }, + }; +} + test('commits adopted service active evidence and replays from the instance head', async (t) => { const state = fixture(t); const prepared = prepare( @@ -574,6 +703,92 @@ test('prepares and exactly replays lossless service-manager legacy rollback evid assert.equal(replay.instanceHeadDigest, prepared.instanceHeadDigest); }); +test('authorizes, consumes and exactly replays service-manager legacy running evidence', async (t) => { + const state = fixture(t); + const { stoppedResult, head } = await stopAdoptedTarget(state, '039'); + const prepared = prepareLocalServiceManagerLegacyRollback( + rollbackPrepareCommand(state, stoppedResult, head), + ); + const authorizeCommand = rollbackAuthorizeCommand(state, prepared); + const authorized = + authorizeLocalServiceManagerLegacyRollback(authorizeCommand); + assert.equal(authorized.status, 'prepared'); + assert.equal(authorized.state, 'legacy_restart_requested'); + assert.equal( + authorizeLocalServiceManagerLegacyRollback(authorizeCommand).status, + 'existing', + ); + const requestedHead = readLocalCutoverInstanceHead( + state.root, + 'edge-router-1', + process.getuid(), + ); + assert.equal(requestedHead.state, 'legacy_restart_requested'); + assert.equal( + requestedHead.sourceRecordDigest, + authorized.authorizationDigest, + ); + const outcome = publishLegacyStartOutcome(state, prepared, authorized); + const consumeCommand = rollbackConsumeCommand(state, prepared, authorized); + const consumed = consumeLocalServiceManagerLegacyRollback(consumeCommand); + assert.equal(consumed.status, 'prepared'); + assert.equal(consumed.state, 'legacy_running'); + assert.equal(consumed.outcomeDigest, outcome.outcomeDigest); + assert.match(consumed.completionDigest, /^[0-9a-f]{64}$/); + const replay = consumeLocalServiceManagerLegacyRollback(consumeCommand); + assert.equal(replay.status, 'existing'); + assert.equal(replay.completionDigest, consumed.completionDigest); + assert.equal(replay.instanceHeadDigest, consumed.instanceHeadDigest); +}); + +test('requires current rollback evidence before publishing legacy start authorization', async (t) => { + const state = fixture(t); + const { stoppedResult, head } = await stopAdoptedTarget(state, '041'); + const prepared = prepareLocalServiceManagerLegacyRollback( + rollbackPrepareCommand(state, stoppedResult, head), + ); + fs.writeFileSync(state.targetPath, 'target-drift-before-authorization\n', { + mode: 0o600, + }); + assert.throws( + () => + authorizeLocalServiceManagerLegacyRollback( + rollbackAuthorizeCommand(state, prepared), + ), + /prepared rollback data drifted after authorization/, + ); + const unchanged = readLocalCutoverInstanceHead( + state.root, + 'edge-router-1', + process.getuid(), + ); + assert.equal(unchanged.state, 'rollback_prepared'); + assert.equal(unchanged.headDigest, prepared.instanceHeadDigest); +}); + +test('terminalizes an unproved root legacy start outcome', async (t) => { + const state = fixture(t); + const { stoppedResult, head } = await stopAdoptedTarget(state, '043'); + const prepared = prepareLocalServiceManagerLegacyRollback( + rollbackPrepareCommand(state, stoppedResult, head), + ); + const authorized = authorizeLocalServiceManagerLegacyRollback( + rollbackAuthorizeCommand(state, prepared), + ); + publishLegacyStartOutcome(state, prepared, authorized, 'manual_required'); + const consumed = consumeLocalServiceManagerLegacyRollback( + rollbackConsumeCommand(state, prepared, authorized), + ); + assert.equal(consumed.state, 'manual_required'); + const terminal = readLocalCutoverInstanceHead( + state.root, + 'edge-router-1', + process.getuid(), + ); + assert.equal(terminal.state, 'manual_required'); + assert.equal(terminal.sourceRecordDigest, consumed.completionDigest); +}); + test('keeps target_stopped when service-manager rollback would discard target writes', async (t) => { const state = fixture(t); const { stoppedResult, head } = await stopAdoptedTarget(state, '033'); diff --git a/packages/ql3-local-owner-cli/test/serviceLegacyRollbackContract.test.cjs b/packages/ql3-local-owner-cli/test/serviceLegacyRollbackContract.test.cjs new file mode 100644 index 00000000..6e159334 --- /dev/null +++ b/packages/ql3-local-owner-cli/test/serviceLegacyRollbackContract.test.cjs @@ -0,0 +1,144 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + localServiceManagerLegacyStartOutcomeDigest, + localServiceManagerRollbackObservationDigest, + normalizeLocalServiceManagerLegacyRollbackBridgeCommand, + normalizeLocalServiceManagerLegacyStartOutcome, +} = require('../dist/deployment/service-manager/legacy-rollback/contract.js'); + +function command(kind = 'systemd') { + return { + schemaVersion: 1, + operation: 'local.deployment.service-manager.legacy-rollback.execute', + options: { + deploymentRoot: '/var/lib/qinglong3', + controllerRoot: '/var/lib/qinglong3-service-bridge', + allowRootController: true, + manager: + kind === 'systemd' + ? { kind, executable: '/usr/bin/systemctl' } + : { + kind, + serviceExecutable: '/sbin/rc-service', + updateExecutable: '/sbin/rc-update', + }, + }, + request: { + cutoverId: 'edge-router-cutover', + generation: 1, + expectedAuthorizationDigest: '1'.repeat(64), + }, + }; +} + +function observation(serviceName, active, observedAtMs) { + const payload = { + managerKind: 'systemd', + serviceName, + fragmentPath: `/etc/systemd/system/${serviceName}.service`, + loadState: 'loaded', + activeState: active ? 'active' : 'inactive', + subState: active ? 'running' : 'dead', + enabledState: 'enabled', + mainPid: active ? 3101 : 0, + observedAtMs, + }; + return { + ...payload, + observationDigest: localServiceManagerRollbackObservationDigest(payload), + }; +} + +function outcome() { + const payload = { + schema: 'qinglong3-local-service-manager-legacy-start-outcome', + schemaVersion: 1, + state: 'legacy_running', + cutoverId: 'edge-router-cutover', + profile: 'edge', + instanceId: 'edge-router-1', + generation: 1, + activationDigest: '2'.repeat(64), + managerKind: 'systemd', + preparationDigest: '3'.repeat(64), + authorizationDigest: '4'.repeat(64), + barrierDigest: '5'.repeat(64), + legacyDescriptorDigest: '6'.repeat(64), + targetDescriptorDigest: '7'.repeat(64), + mutationDisposition: 'response-loss-inspected', + manualReason: null, + legacyObservation: observation('qinglong', true, 100), + targetObservation: observation('qinglong3', false, 101), + completedAtMs: 102, + }; + return { + ...payload, + outcomeDigest: localServiceManagerLegacyStartOutcomeDigest(payload), + }; +} + +test('normalizes exact systemd and OpenRC legacy rollback bridge commands', () => { + assert.equal( + normalizeLocalServiceManagerLegacyRollbackBridgeCommand(command()).options + .manager.kind, + 'systemd', + ); + assert.equal( + normalizeLocalServiceManagerLegacyRollbackBridgeCommand(command('openrc')) + .options.manager.kind, + 'openrc', + ); +}); + +test('rejects shell surface, path drift and unbound root acknowledgement', () => { + assert.throws( + () => + normalizeLocalServiceManagerLegacyRollbackBridgeCommand({ + ...command(), + shell: 'systemctl start qinglong', + }), + /shape is invalid/, + ); + assert.throws( + () => + normalizeLocalServiceManagerLegacyRollbackBridgeCommand({ + ...command(), + options: { ...command().options, deploymentRoot: '/var/lib/../tmp' }, + }), + /supervisor-safe absolute path/, + ); + assert.throws( + () => + normalizeLocalServiceManagerLegacyRollbackBridgeCommand({ + ...command(), + options: { ...command().options, allowRootController: false }, + }), + /command is invalid/, + ); +}); + +test('binds both service observations and all rollback outcome digests', () => { + const value = outcome(); + const normalized = normalizeLocalServiceManagerLegacyStartOutcome(value); + assert.equal(normalized.state, 'legacy_running'); + assert.equal(normalized.legacyObservation.serviceName, 'qinglong'); + assert.equal(normalized.targetObservation.serviceName, 'qinglong3'); + assert.throws( + () => + normalizeLocalServiceManagerLegacyStartOutcome({ + ...value, + targetDescriptorDigest: '8'.repeat(64), + }), + /outcome drifted/, + ); + assert.throws( + () => + normalizeLocalServiceManagerLegacyStartOutcome({ + ...value, + legacyObservation: value.targetObservation, + }), + /outcome drifted/, + ); +}); diff --git a/scripts/ql3-service-bridge-import-audit.cjs b/scripts/ql3-service-bridge-import-audit.cjs index e8039aa0..85a6ea03 100644 --- a/scripts/ql3-service-bridge-import-audit.cjs +++ b/scripts/ql3-service-bridge-import-audit.cjs @@ -24,16 +24,19 @@ function main() { 'ql3-service-bridge binary entry drifted', ); } - const entry = path.join( - packageRoot, + const entries = [ 'dist/deployment/service-manager/serviceBridge.js', - ); - if (!fs.existsSync(entry)) { - throw new QingLong3ServiceBridgeImportAuditError( - 'service bridge must be built before import audit', - ); + 'dist/deployment/service-manager/legacy-rollback/bridge.js', + ]; + for (const relativeEntry of entries) { + const entry = path.join(packageRoot, relativeEntry); + if (!fs.existsSync(entry)) { + throw new QingLong3ServiceBridgeImportAuditError( + `${relativeEntry} must be built before import audit`, + ); + } + require(entry); } - require(entry); const loaded = Object.keys(require.cache) .map((filePath) => path.resolve(filePath)) .filter( @@ -69,8 +72,14 @@ function main() { file: 'packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeCli.ts', }); } + if (!source.includes("from './legacy-rollback/bridge'")) { + findings.push({ + code: 'LEGACY_ROLLBACK_BRIDGE_CLI_ENTRY_DRIFT', + file: 'packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeCli.ts', + }); + } const report = { - schemaVersion: 1, + schemaVersion: 2, compatible: findings.length === 0, binary: expectedBin, loaded, diff --git a/scripts/ql3-service-manager-bridge-docker-gate.cjs b/scripts/ql3-service-manager-bridge-docker-gate.cjs index a5d61126..18549074 100644 --- a/scripts/ql3-service-manager-bridge-docker-gate.cjs +++ b/scripts/ql3-service-manager-bridge-docker-gate.cjs @@ -61,7 +61,7 @@ function waitReady(name, kind) { fail(`${kind} container did not become ready`); } -function actorReport(name, kind, identityMode) { +function actorReport(name, kind, identityMode, scenario = 'success') { const stdout = run([ 'exec', name, @@ -69,6 +69,7 @@ function actorReport(name, kind, identityMode) { '/workspace/scripts/ql3-service-manager-bridge-live-actor.cjs', kind, identityMode, + scenario, ]); const report = JSON.parse(stdout); const requiredGates = [ @@ -84,10 +85,28 @@ function actorReport(name, kind, identityMode) { 'adoptedRollbackPrepared', 'adoptedRollbackReplay', 'adoptedRollbackCandidate', + 'adoptedRollbackAuthorized', + 'adoptedRollbackAuthorizationReplay', + 'adoptedLegacyStartReplay', + 'adoptedLegacyConsumed', + 'adoptedTargetRemainedStopped', + ...(scenario === 'success' + ? [ + 'adoptedLegacyStarted', + 'adoptedLegacyProcessIdentity', + 'adoptedLegacyResponseLossInspected', + ] + : [ + 'adoptedLegacyBarrierCrash', + 'adoptedLegacyInspectOnlyConvergence', + 'adoptedLegacyRemainedStopped', + 'adoptedLegacyManualConsumed', + ]), ]; if ( report.managerKind !== kind || report.identityMode !== identityMode || + report.scenario !== scenario || report.serviceUid !== (identityMode === 'root' ? 0 : 10001) || !requiredGates.every((gate) => report.gates?.[gate] === true) || (report.gates?.restartRequired === true && @@ -152,10 +171,21 @@ function main() { ); waitReady(name, target.kind); reports.push(actorReport(name, target.kind, identityMode)); + if (identityMode === 'root') { + reports.push( + actorReport(name, target.kind, identityMode, 'barrier-crash'), + ); + } run(['rm', '--force', name]); containers.splice(containers.indexOf(name), 1); } } + const successReports = reports.filter( + (report) => report.scenario === 'success', + ); + const crashReports = reports.filter( + (report) => report.scenario === 'barrier-crash', + ); const payload = { schemaVersion: 1, evidenceClass: 'qinglong3_service_manager_bridge_docker_gate', @@ -179,31 +209,65 @@ function main() { report.managerKind === 'openrc' && report.identityMode === 'nonroot', ), - rootCommandFile: reports.every( + rootCommandFile: successReports.every( (report) => report.gates.rootCommandFile, ), - exactReplay: reports.every((report) => report.gates.exactReplay), - ownerOutcomeVerified: reports.every( + exactReplay: successReports.every((report) => report.gates.exactReplay), + ownerOutcomeVerified: successReports.every( (report) => report.gates.ownerOutcomeVerified, ), - serviceProcessIdentity: reports.every( + serviceProcessIdentity: successReports.every( (report) => report.gates.serviceProcessIdentity, ), - adoptedCutoverActive: reports.every( + adoptedCutoverActive: successReports.every( (report) => report.gates.adoptedCutoverActive, ), - adoptedCutoverStopped: reports.every( + adoptedCutoverStopped: successReports.every( (report) => report.gates.adoptedCutoverStopped, ), - adoptedRollbackPrepared: reports.every( + adoptedRollbackPrepared: successReports.every( (report) => report.gates.adoptedRollbackPrepared, ), - adoptedRollbackReplay: reports.every( + adoptedRollbackReplay: successReports.every( (report) => report.gates.adoptedRollbackReplay, ), - adoptedRollbackCandidate: reports.every( + adoptedRollbackCandidate: successReports.every( (report) => report.gates.adoptedRollbackCandidate, ), + adoptedRollbackAuthorized: successReports.every( + (report) => report.gates.adoptedRollbackAuthorized, + ), + adoptedRollbackAuthorizationReplay: successReports.every( + (report) => report.gates.adoptedRollbackAuthorizationReplay, + ), + adoptedLegacyStarted: successReports.every( + (report) => report.gates.adoptedLegacyStarted, + ), + adoptedLegacyStartReplay: successReports.every( + (report) => report.gates.adoptedLegacyStartReplay, + ), + adoptedLegacyConsumed: successReports.every( + (report) => report.gates.adoptedLegacyConsumed, + ), + adoptedTargetRemainedStopped: successReports.every( + (report) => report.gates.adoptedTargetRemainedStopped, + ), + adoptedLegacyProcessIdentity: successReports.every( + (report) => report.gates.adoptedLegacyProcessIdentity, + ), + adoptedLegacyResponseLossInspected: successReports.every( + (report) => report.gates.adoptedLegacyResponseLossInspected, + ), + adoptedLegacyBarrierCrash: crashReports.length === 2, + adoptedLegacyInspectOnlyConvergence: crashReports.every( + (report) => report.gates.adoptedLegacyInspectOnlyConvergence, + ), + adoptedLegacyRemainedStopped: crashReports.every( + (report) => report.gates.adoptedLegacyRemainedStopped, + ), + adoptedLegacyManualConsumed: crashReports.every( + (report) => report.gates.adoptedLegacyManualConsumed, + ), systemdRestart: reports.some( (report) => report.managerKind === 'systemd' && report.gates.restarted, diff --git a/scripts/ql3-service-manager-bridge-live-actor.cjs b/scripts/ql3-service-manager-bridge-live-actor.cjs index fc697634..c08c6c0d 100644 --- a/scripts/ql3-service-manager-bridge-live-actor.cjs +++ b/scripts/ql3-service-manager-bridge-live-actor.cjs @@ -20,6 +20,12 @@ const { const { runLocalServiceBridgeCommandFile, } = require('../packages/ql3-local-owner-cli/dist/deployment/service-manager/serviceBridge.js'); +const { + runLocalServiceManagerLegacyRollbackBridgeCommandFile, +} = require('../packages/ql3-local-owner-cli/dist/deployment/service-manager/legacy-rollback/bridge.js'); +const { + localServiceManagerLegacyStartOutcomePath, +} = require('../packages/ql3-local-owner-cli/dist/deployment/service-manager/legacy-rollback/contract.js'); const OWNER_DEPLOYMENT_CLI = path.resolve( __dirname, @@ -58,6 +64,30 @@ function run(executable, args, accepted = [0]) { return String(result.stdout ?? ''); } +function responseLossManager(request) { + const result = spawnSync(request.executable, [...request.args], { + encoding: 'utf8', + env: { + PATH: '/usr/sbin:/usr/bin:/sbin:/bin', + LANG: 'C', + LC_ALL: 'C', + }, + timeout: request.timeoutMs, + maxBuffer: 64 * 1024, + }); + const legacyStart = + (request.args[0] === 'start' && request.args[1] === 'qinglong.service') || + (request.args[0] === 'qinglong' && request.args[1] === 'start'); + return { + status: legacyStart ? null : result.status, + signal: result.signal, + stdout: String(result.stdout ?? ''), + stderr: String(result.stderr ?? ''), + responseLost: + legacyStart || result.error !== undefined || result.status === null, + }; +} + function executable(candidates, label) { for (const candidate of candidates) { if (fs.existsSync(candidate)) return fs.realpathSync(candidate); @@ -106,6 +136,37 @@ function descriptor(kind, uid, gid, root, adopted = false) { ].join('\n'); } +function legacyDescriptor(kind, uid, gid) { + if (kind === 'systemd') { + return [ + '[Unit]', + 'Description=QingLong 2 legacy rollback live gate', + '', + '[Service]', + 'Type=simple', + `User=${uid}`, + `Group=${gid}`, + 'ExecStart=/bin/sleep 300', + 'Restart=no', + '', + '[Install]', + 'WantedBy=multi-user.target', + '', + ].join('\n'); + } + return [ + '#!/sbin/openrc-run', + 'name="qinglong"', + 'description="QingLong 2 legacy rollback live gate"', + 'command="/bin/sleep"', + 'command_args="300"', + `command_user="${uid}:${gid}"`, + 'supervisor="supervise-daemon"', + 'retry="TERM/5/KILL/2"', + '', + ].join('\n'); +} + function manager(kind) { if (kind === 'systemd') { return { @@ -436,6 +497,222 @@ function prepareAdoptedRollback(root, adopted, stopped, uid, gid) { }; } +function authorizeAdoptedRollback(root, adopted, prepared, uid, gid) { + const command = { + schemaVersion: 1, + operation: 'local.deployment.service-manager.legacy-rollback.authorize', + options: { deploymentRoot: root, allowRootService: uid === 0 }, + request: { + cutoverId: adopted.cutoverId, + profile: 'edge', + instanceId: adopted.instanceId, + generation: 1, + expectedActivationDigest: adopted.activationDigest, + expectedPreparationDigest: prepared.preparationDigest, + expectedInstanceHeadDigest: prepared.instanceHeadDigest, + requestedAtMs: Date.now(), + }, + }; + const commandPath = path.join(root, 'owner-legacy-rollback-authorize.json'); + writePrivate( + commandPath, + `${JSON.stringify(command, null, 2)}\n`, + 0o600, + uid, + gid, + ); + return { + commandPath, + result: ownerCli( + uid, + gid, + 'service-legacy-rollback-authorize', + commandPath, + ), + }; +} + +function installLegacyService(kind, managerOptions, uid, gid) { + const descriptorPath = + kind === 'systemd' + ? '/etc/systemd/system/qinglong.service' + : '/etc/init.d/qinglong'; + fs.rmSync(descriptorPath, { force: true }); + writePrivate( + descriptorPath, + legacyDescriptor(kind, uid, gid), + kind === 'systemd' ? 0o644 : 0o755, + 0, + 0, + ); + if (kind === 'systemd') { + run(managerOptions.executable, ['daemon-reload']); + } +} + +function executeLegacyRollback( + root, + controllerRoot, + managerOptions, + adopted, + prepared, + authorized, + uid, + gid, +) { + const command = { + schemaVersion: 1, + operation: 'local.deployment.service-manager.legacy-rollback.execute', + options: { + deploymentRoot: root, + controllerRoot, + allowRootController: true, + manager: managerOptions, + }, + request: { + cutoverId: adopted.cutoverId, + generation: 1, + expectedAuthorizationDigest: authorized.authorizationDigest, + }, + }; + const commandPath = path.join(root, 'root-legacy-rollback-command.json'); + writePrivate(commandPath, `${JSON.stringify(command, null, 2)}\n`); + const result = runLocalServiceManagerLegacyRollbackBridgeCommandFile( + commandPath, + { runManager: responseLossManager }, + ); + const replay = + runLocalServiceManagerLegacyRollbackBridgeCommandFile(commandPath); + const consumeCommand = { + schemaVersion: 1, + operation: 'local.deployment.service-manager.legacy-rollback.consume', + options: { deploymentRoot: root, allowRootService: uid === 0 }, + request: { + cutoverId: adopted.cutoverId, + profile: 'edge', + instanceId: adopted.instanceId, + generation: 1, + expectedActivationDigest: adopted.activationDigest, + expectedPreparationDigest: prepared.preparationDigest, + expectedAuthorizationDigest: authorized.authorizationDigest, + expectedAuthorizationHeadDigest: authorized.instanceHeadDigest, + requestedAtMs: Date.now(), + }, + }; + const consumePath = path.join(root, 'owner-legacy-rollback-consume.json'); + writePrivate( + consumePath, + `${JSON.stringify(consumeCommand, null, 2)}\n`, + 0o600, + uid, + gid, + ); + const consumed = ownerCli( + uid, + gid, + 'service-legacy-rollback-consume', + consumePath, + ); + const outcome = JSON.parse( + fs.readFileSync( + localServiceManagerLegacyStartOutcomePath(root, adopted.cutoverId, 1), + 'utf8', + ), + ); + return { commandPath, result, replay, consumed, outcome }; +} + +function executeLegacyRollbackBarrierCrash( + root, + controllerRoot, + managerOptions, + adopted, + prepared, + authorized, + uid, + gid, +) { + const command = { + schemaVersion: 1, + operation: 'local.deployment.service-manager.legacy-rollback.execute', + options: { + deploymentRoot: root, + controllerRoot, + allowRootController: true, + manager: managerOptions, + }, + request: { + cutoverId: adopted.cutoverId, + generation: 1, + expectedAuthorizationDigest: authorized.authorizationDigest, + }, + }; + const commandPath = path.join( + root, + 'root-legacy-rollback-barrier-crash-command.json', + ); + writePrivate(commandPath, `${JSON.stringify(command, null, 2)}\n`); + let interrupted = false; + try { + runLocalServiceManagerLegacyRollbackBridgeCommandFile(commandPath, { + afterBarrier() { + throw new QingLong3ServiceManagerBridgeLiveActorError( + 'simulated crash after legacy start barrier', + ); + }, + }); + } catch (error) { + interrupted = + error instanceof QingLong3ServiceManagerBridgeLiveActorError && + error.message === 'simulated crash after legacy start barrier'; + } + if (!interrupted) fail('legacy start barrier crash was not observed'); + const result = + runLocalServiceManagerLegacyRollbackBridgeCommandFile(commandPath); + const replay = + runLocalServiceManagerLegacyRollbackBridgeCommandFile(commandPath); + const consumeCommand = { + schemaVersion: 1, + operation: 'local.deployment.service-manager.legacy-rollback.consume', + options: { deploymentRoot: root, allowRootService: uid === 0 }, + request: { + cutoverId: adopted.cutoverId, + profile: 'edge', + instanceId: adopted.instanceId, + generation: 1, + expectedActivationDigest: adopted.activationDigest, + expectedPreparationDigest: prepared.preparationDigest, + expectedAuthorizationDigest: authorized.authorizationDigest, + expectedAuthorizationHeadDigest: authorized.instanceHeadDigest, + requestedAtMs: Date.now(), + }, + }; + const consumePath = path.join( + root, + 'owner-legacy-rollback-barrier-crash-consume.json', + ); + writePrivate( + consumePath, + `${JSON.stringify(consumeCommand, null, 2)}\n`, + 0o600, + uid, + gid, + ); + const consumed = ownerCli( + uid, + gid, + 'service-legacy-rollback-consume', + consumePath, + ); + const outcome = JSON.parse( + fs.readFileSync( + localServiceManagerLegacyStartOutcomePath(root, adopted.cutoverId, 1), + 'utf8', + ), + ); + return { commandPath, result, replay, consumed, outcome }; +} + function execute(root, controllerRoot, managerOptions, prepared, uid, gid) { const command = { schemaVersion: 1, @@ -509,24 +786,30 @@ function serviceProcessUid(kind, outcome) { function cleanup(kind, managerOptions) { if (kind === 'systemd') { - spawnSync( - managerOptions.executable, - ['disable', '--now', 'qinglong3.service'], - { timeout: 10_000 }, - ); - fs.rmSync('/etc/systemd/system/qinglong3.service', { force: true }); + for (const serviceName of ['qinglong', 'qinglong3']) { + spawnSync( + managerOptions.executable, + ['disable', '--now', `${serviceName}.service`], + { timeout: 10_000 }, + ); + fs.rmSync(`/etc/systemd/system/${serviceName}.service`, { force: true }); + } spawnSync(managerOptions.executable, ['daemon-reload'], { timeout: 10_000, }); return; } - spawnSync(managerOptions.serviceExecutable, ['qinglong3', 'stop'], { - timeout: 10_000, - }); - spawnSync(managerOptions.updateExecutable, ['del', 'qinglong3', 'default'], { - timeout: 10_000, - }); - fs.rmSync('/etc/init.d/qinglong3', { force: true }); + for (const serviceName of ['qinglong', 'qinglong3']) { + spawnSync(managerOptions.serviceExecutable, [serviceName, 'stop'], { + timeout: 10_000, + }); + spawnSync( + managerOptions.updateExecutable, + ['del', serviceName, 'default'], + { timeout: 10_000 }, + ); + fs.rmSync(`/etc/init.d/${serviceName}`, { force: true }); + } } async function main(argv) { @@ -535,12 +818,14 @@ async function main(argv) { } const kind = argv[0]; const identityMode = argv[1]; + const scenario = argv[2] ?? 'success'; if ( (kind !== 'systemd' && kind !== 'openrc') || - (identityMode !== 'root' && identityMode !== 'nonroot') + (identityMode !== 'root' && identityMode !== 'nonroot') || + (scenario !== 'success' && scenario !== 'barrier-crash') ) { fail( - 'usage: ql3-service-manager-bridge-live-actor.cjs ', + 'usage: ql3-service-manager-bridge-live-actor.cjs [success|barrier-crash]', ); } const uid = identityMode === 'root' ? 0 : NON_ROOT_SERVICE_UID; @@ -728,11 +1013,91 @@ async function main(argv) { ) { fail('adopted rollback preparation did not converge exactly'); } + const adoptedAuthorization = authorizeAdoptedRollback( + root, + adopted, + adoptedRollback.result, + uid, + gid, + ); + const adoptedAuthorizationReplay = ownerCli( + uid, + gid, + 'service-legacy-rollback-authorize', + adoptedAuthorization.commandPath, + ); + if ( + adoptedAuthorization.result.status !== 'prepared' || + adoptedAuthorization.result.state !== 'legacy_restart_requested' || + adoptedAuthorizationReplay.status !== 'existing' || + adoptedAuthorizationReplay.authorizationDigest !== + adoptedAuthorization.result.authorizationDigest + ) { + fail('adopted rollback authorization did not converge exactly'); + } + installLegacyService(kind, managerOptions, uid, gid); + const adoptedLegacyStarted = + scenario === 'success' + ? executeLegacyRollback( + root, + controllerRoot, + managerOptions, + adopted, + adoptedRollback.result, + adoptedAuthorization.result, + uid, + gid, + ) + : executeLegacyRollbackBarrierCrash( + root, + controllerRoot, + managerOptions, + adopted, + adoptedRollback.result, + adoptedAuthorization.result, + uid, + gid, + ); + if (scenario === 'success') { + if ( + adoptedLegacyStarted.result.state !== 'legacy_running' || + adoptedLegacyStarted.replay.status !== 'existing' || + adoptedLegacyStarted.replay.state !== 'legacy_running' || + adoptedLegacyStarted.consumed.state !== 'legacy_running' || + adoptedLegacyStarted.outcome.mutationDisposition !== + 'response-loss-inspected' || + adoptedLegacyStarted.outcome.targetObservation.activeState !== + 'inactive' || + adoptedLegacyStarted.outcome.legacyObservation.activeState !== 'active' + ) { + fail('adopted legacy rollback did not converge to legacy_running'); + } + const legacyUid = serviceProcessUid(kind, { + observation: adoptedLegacyStarted.outcome.legacyObservation, + }); + if (legacyUid !== uid) { + fail(`legacy service process UID drifted: ${legacyUid} != ${uid}`); + } + } else if ( + adoptedLegacyStarted.result.state !== 'manual_required' || + adoptedLegacyStarted.result.status !== 'prepared' || + adoptedLegacyStarted.replay.status !== 'existing' || + adoptedLegacyStarted.replay.state !== 'manual_required' || + adoptedLegacyStarted.consumed.state !== 'manual_required' || + adoptedLegacyStarted.outcome.mutationDisposition !== 'replay-inspected' || + adoptedLegacyStarted.outcome.manualReason !== 'manager_state_unproved' || + adoptedLegacyStarted.outcome.targetObservation.activeState !== + 'inactive' || + adoptedLegacyStarted.outcome.legacyObservation.activeState !== 'inactive' + ) { + fail('legacy barrier replay did not fail closed without a second start'); + } const payload = { schemaVersion: 1, evidenceClass: 'qinglong3_service_manager_bridge_live_actor', managerKind: kind, identityMode, + scenario, serviceUid: uid, manager: managerOptions, actions: [ @@ -746,6 +1111,11 @@ async function main(argv) { stopped: adoptedStopped, rollbackPrepared: adoptedRollback.result, rollbackReplay: adoptedRollbackReplay, + rollbackAuthorized: adoptedAuthorization.result, + rollbackAuthorizationReplay: adoptedAuthorizationReplay, + legacyStarted: adoptedLegacyStarted.result, + legacyStartReplay: adoptedLegacyStarted.replay, + legacyConsumed: adoptedLegacyStarted.consumed, }, gates: { rootCommandFile: true, @@ -762,6 +1132,18 @@ async function main(argv) { adoptedRollbackPrepared: true, adoptedRollbackReplay: true, adoptedRollbackCandidate: true, + adoptedRollbackAuthorized: true, + adoptedRollbackAuthorizationReplay: true, + adoptedLegacyStarted: scenario === 'success', + adoptedLegacyStartReplay: true, + adoptedLegacyConsumed: true, + adoptedTargetRemainedStopped: true, + adoptedLegacyProcessIdentity: scenario === 'success', + adoptedLegacyResponseLossInspected: scenario === 'success', + adoptedLegacyBarrierCrash: scenario === 'barrier-crash', + adoptedLegacyInspectOnlyConvergence: scenario === 'barrier-crash', + adoptedLegacyRemainedStopped: scenario === 'barrier-crash', + adoptedLegacyManualConsumed: scenario === 'barrier-crash', }, }; process.stdout.write( diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index 2462098c..928ab5f3 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -207,10 +207,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: localOwnerCli.rootSourceFileRoles, }, { - sourceFiles: 108, + sourceFiles: 111, rootSourceFiles: 1, rootSourceLines: 50, - nestedSourceFiles: 107, + nestedSourceFiles: 110, rootSourceFileRoles: { 'cli.ts': 'binary_entry' }, }, );