From 5f62f2f72414c5bed85c266bfc76717e7107b915 Mon Sep 17 00:00:00 2001 From: whyour Date: Sun, 23 Aug 2026 00:34:46 +0800 Subject: [PATCH] feat(local): bind service completion restart --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 15 + ...-domain-reconciliation-completion-fence.md | 4 +- ...vice-manager-completion-restart-lineage.md | 61 ++++ .../src/deployment/cutover/instanceLineage.ts | 1 + .../service-manager/serviceBridgeContract.ts | 70 ++++- .../service-manager/serviceCutoverConsumer.ts | 75 ++++- .../service-manager/serviceCutoverJournal.ts | 55 +++- .../service-manager/serviceManagerIntent.ts | 42 ++- .../test/serviceBridgeContract.test.cjs | 50 ++++ .../test/serviceCutoverConsumer.test.cjs | 263 ++++++++++++++++++ .../workerPosixExecutionExecutor.test.cjs | 74 +++-- 11 files changed, 659 insertions(+), 51 deletions(-) create mode 100644 docs/adr/ADR-0489-service-manager-completion-restart-lineage.md diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index e661d789..54d7f50e 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -109,6 +109,21 @@ `268 total / 261 pass / 7 conditional skip / 0 fail`、18-package clean build/逐包测试、dependency/package boundary `70/70` 和 122-module Edge import audit 已通过;其余六个领域的 terminal adapter、Service Manager v2 binding、完整 target readiness/restart 演练和 Cluster 自有 completion authority 仍是后续门禁。 +- D-395/ADR-0489(已接受):systemd/OpenRC 不再因 v1 `previousRecordDigest` 同时承担历史 service record 与 current head source 两种含义而永久拒绝 + completed restart。Service Manager intent 新增严格版本化 schema v2,且只允许 adopted generation ≥ 2 的 `restart`:保留 + `previousRecordDigest` 精确绑定上一代 active record,并以 `completionFence.expectedInstanceHeadDigest|expectedCompletionDigest` 同时绑定当前 + `reconciliation_completed` head 和 completion receipt digest。Owner 在 intent publish 前及 root outcome consume 后都重验 head,并打开上一代 active + record 复验 Profile、instance、activation、generation 与 record digest;v1 completed restart、缺字段、额外字段、旧 record、stale head、时间倒序全部失败关闭。 + + v2 产生的 service cutover record 使用 schema v3 显式保存双谱系证据;发布仍为 record-first、head-second。record publish 后响应丢失时,重放只有在 + v3 record、intent、manager outcome、completed head 和 completion fence 全部一致时才 CAS 到下一代 `target_active`,不重复 root manager mutation,也不接受 + 上一代 startup receipt。新 receipt 或进程 identity 无法证明时,从 completed head 窄化为 `manual_required` 并可 exact replay。旧 v1 intent、普通 active + restart 和 v1/v2 journal 保持原语义,没有静默升级。实现全部留在既有 Local Owner Service Manager 子域,没有新增 package、`src/` 根平铺、dependency、 + SQL、daemon、timer、watcher、listener、Pool 或 Cluster workload;Cluster restart/completion 仍需独立事务与 HA evidence。focused 组合门为 + `75 total / 73 pass / 2 conditional Docker skip / 0 fail`,完整 Local Owner 为 `271 total / 264 pass / 7 conditional Docker skip / 0 fail`,18-package + clean build/逐包测试为 `2917 total / 2895 pass / 22 conditional integration skip / 0 fail`;package/dependency boundary `70/70`、122-module Edge import audit + 与 18-package non-shallow 边界保持通过。本机 root service bridge Docker 门因 Docker Desktop `ENOSPC` 条件跳过且未清理用户匿名卷,必须由同一提交的远程 + fresh runner root/non-root 门补齐。 - D-392/ADR-0485(已接受):D-391 的 signed review 不能直接获得通用 DML authority;表级 `adopt_legacy/retain_both` 也不能证明 Automation 行级 command/trigger 兼容,更不能覆盖 Secret custody、append-only history、Plugin/AI 外部资产与 Identity/Policy 语义。 因此既有 Local Owner 新增 `reconciliation.application.prepare|commit|verify`,以 diff --git a/docs/adr/ADR-0488-cross-domain-reconciliation-completion-fence.md b/docs/adr/ADR-0488-cross-domain-reconciliation-completion-fence.md index 0964d1a4..59505048 100644 --- a/docs/adr/ADR-0488-cross-domain-reconciliation-completion-fence.md +++ b/docs/adr/ADR-0488-cross-domain-reconciliation-completion-fence.md @@ -51,7 +51,7 @@ receipt publish、seal、head advance 或 backup collection 任一响应丢失 Docker target authority 继续拒绝 `reconciliation_application_planned`、`reconciliation_automation_applied` 和所有 manual/rolled-back 中间态;只有 `reconciliation_completed` 能重新进入 `target_active`,且调用方必须使用下一 generation。 -Service Manager 虽然复用同一个 head assertion,但既有 v1 intent 只绑定 prior service journal `previousRecordDigest`,没有独立的 expected completion-head digest;其后续 compare-and-swap 因而继续拒绝 completed head。D-394 不通过忽略该比较来伪造兼容性。systemd/OpenRC 的 restart-ready 必须由后续 v2 intent 同时绑定 prior service record 与 exact completion head 后才能开放。 +Service Manager 虽然复用同一个 head assertion,但既有 v1 intent 只绑定 prior service journal `previousRecordDigest`,没有独立的 expected completion-head digest;其后续 compare-and-swap 因而继续拒绝 completed head。D-394 不通过忽略该比较来伪造兼容性。该后续边界现由 D-395/ADR-0489 的 v2 intent 完成:systemd/OpenRC 必须同时绑定 prior service record、exact completion head 与 completion digest 才能获得 restart-ready authority;v1 仍失败关闭。 ### 5. 部署规模与代码边界 @@ -84,4 +84,4 @@ Service Manager 虽然复用同一个 head assertion,但既有 v1 intent 只 - 完整 Local Owner `268 total / 261 pass / 7 conditional skip / 0 fail`;18-package clean build 与逐包测试 exit 0。 - dependency/package boundary 组合门 `70/70`,Edge import audit 为 122 modules、0 forbidden;workspace 保持 18 packages、无 single-source/shallow package;Local Owner 为 `172 source / 171 nested / 1 root binary entry`。 - 不新增 production dependency、SQL migration、daemon、timer、watcher、listener、Pool、PostgreSQL role/ACL 或 cluster workload。 -- Secret/Config、Run History、Plugin Package、AI/Tool、Identity/Policy/Audit、Unknown 的 terminal adapter,以及 Service Manager v2 completion-head binding/真实 restart 演练仍是后续工作;D-394 不把尚未实现的领域或部署方式宣称为完成。 +- Secret/Config、Run History、Plugin Package、AI/Tool、Identity/Policy/Audit、Unknown 的 terminal adapter,以及固定真实 systemd/OpenRC completed restart 演练仍是后续工作;Service Manager v2 completion-head binding 已由 D-395/ADR-0489 完成,D-394 本身不倒填该结论。 diff --git a/docs/adr/ADR-0489-service-manager-completion-restart-lineage.md b/docs/adr/ADR-0489-service-manager-completion-restart-lineage.md new file mode 100644 index 00000000..524a898e --- /dev/null +++ b/docs/adr/ADR-0489-service-manager-completion-restart-lineage.md @@ -0,0 +1,61 @@ +# ADR-0489:Service Manager 完成围栏重启谱系 + +- 状态:Accepted +- 日期:2026-08-22 +- 决策:D-395 +- 关联:ADR-0487、ADR-0488 + +## 背景 + +ADR-0488 建立 `reconciliation_completed`,并允许直接 Docker target 在下一 generation 重启,但 systemd/OpenRC 仍保持失败关闭。既有 Service Manager v1 intent 的 `previousRecordDigest` 同时承担“上一代 service active record”和“当前 instance head source record”两种含义;在 reconciliation 开始前二者相等,在 completed head 上却必然分离:历史 active record 仍用于排除旧 startup receipt,当前 head source 已变成 completion receipt digest。 + +忽略其中任意一条比较都会形成越权:只认历史 active record 会跳过八领域 completion,只认 completed head 则无法证明新 startup receipt 相对上一代发生变化。 + +## 决策 + +### 1. v2 intent 只表示 completed restart + +`qinglong3-local-service-manager-intent` 接受 schema v2,但 v2 只允许 adopted、generation ≥ 2 的 `restart`。其 lineage 保留 `previousRecordDigest` 作为上一代 Service Manager active record,并新增一个内聚 `completionFence`: + +- `expectedInstanceHeadDigest`:精确绑定当前 `reconciliation_completed` head; +- `expectedCompletionDigest`:必须等于该 head 的 `sourceRecordDigest`。 + +Owner 在 intent publish 前和 root outcome consume 后都重新读取 head;同时按 generation 打开上一代 active record,复验 record digest、Profile、instance 与 activation。intent 时间不得早于 completed head。调用方不能用 v2 表示 fresh install、普通 active restart、stop 或第一代 start。 + +### 2. v1 不被静默升级 + +v1 intent、普通 `target_active → restart` 和既有 v1/v2 service journal 继续按原契约解析。v1 intent 即使携带额外 completion 字段也因 exact-shape 失败;v2 缺少 completion fence 同样失败。completed head 不对 v1 放宽,因此旧 command、stale intent 或 root bridge replay 不能意外获得新 authority。 + +### 3. v3 journal 保存双谱系证据 + +由 v2 intent 产生的 Service Manager cutover record 使用 schema v3,在既有 manager outcome、application/activation/commitment、target identity、legacy data application 与 startup receipt 证据外,显式保存 completion fence。record 自身 digest 和 intent digest 共同覆盖上一代 service record 与 completed head;普通路径仍写既有 schema。 + +发布顺序保持 record-first、head-second。若 record 已发布而进程丢失响应,重放只在 v3 record、intent、outcome、completed head 和 completion fence 全部一致时把 head CAS 到 generation N 的 `target_active`;不会再次执行 root manager mutation 或接受旧 startup receipt。若 manager active 但新 receipt 无法证明,则允许从 completed head 窄化到 `manual_required`,该终态也能精确重放。 + +### 4. 部署与代码边界 + +实现留在既有 `@qinglong/local-owner-cli/deployment/service-manager` 内,没有新增 workspace package、常驻进程、timer、watcher、listener、Pool、SQL migration 或 production dependency。Edge/Standalone 只在显式管理命令中读取一个 current head 和一个上一代 service record;systemd/OpenRC root bridge 仍只消费 content-bounded intent,不读取 SQLite 或 reconciliation 大对象。 + +Cluster 不复用本机文件 journal。Kubernetes rollout、PostgreSQL completion 和多副本 restart authority 仍需各自的事务/CAS 与 HA evidence。 + +## 被拒绝的替代方案 + +### 把 `previousRecordDigest` 改成 completion digest + +拒绝。它会失去上一代 startup receipt 和 Service Manager journal 的连续性,无法证明 restart 产生了新进程证据。 + +### completed head 对所有 v1 restart 自动放行 + +拒绝。旧 intent 没有声明或 digest-bind completion head,放行会把协议升级变成隐式权限扩大。 + +### 先推进 head、再补 service record + +拒绝。响应丢失会留下 restart-ready head,却没有 manager outcome、startup receipt 或 process identity 的 durable record。 + +## 验收证据 + +- Service Manager contract/cutover、legacy rollback 与 reconciliation focused 组合门为 `75 total / 73 pass / 2 conditional Docker skip / 0 fail`,覆盖 v1/v2 exact shape、systemd/OpenRC contract、上一代 record 与 completed head 双重漂移拒绝、新 startup receipt、record-first response-loss replay,以及缺失新 receipt 时的 `manual_required` 收敛。 +- HIGH/CRITICAL 影响范围通过完整 Local Owner `271 total / 264 pass / 7 conditional Docker skip / 0 fail`;18-package clean build/逐包测试为 `2917 total / 2895 pass / 22 conditional integration skip / 0 fail`。Worker receipt 测试在完整矩阵中复现约 `1.004 s` 的合法完成,轮询预算由 1 秒修正为 5 秒后 Worker Runtime 为 `134/134`,生产执行语义不变。 +- dependency/package boundary 组合门 `70/70`,Edge import audit 为 122 modules、0 forbidden;workspace 保持 18 packages、无 single-source/shallow package,Local Owner 为 `172 source / 171 nested / 1 root binary entry`。 +- 本机 root systemd/OpenRC Docker 门因 Docker Desktop 存储 `ENOSPC` 条件跳过,未清理用户匿名卷;同一提交仍必须通过远程全新 runner 的 root/non-root service bridge 和多架构门,不能用聚焦测试替代。 +- 不新增 production dependency、SQL migration、daemon、timer、watcher、listener、Pool、PostgreSQL role/ACL、cluster workload、package 或 `src/` 根平铺文件。 diff --git a/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts b/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts index ccdd5706..2618885c 100644 --- a/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts +++ b/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts @@ -474,6 +474,7 @@ export function advanceLocalCutoverInstanceHead( (state === 'manual_required' && (current.state === 'legacy_stopped' || current.state === 'target_active' || + current.state === 'reconciliation_completed' || current.state === 'rollback_prepared' || current.state === 'legacy_restart_requested')); if (!allowed) diff --git a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeContract.ts b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeContract.ts index 64c40fb4..2852b6dc 100644 --- a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeContract.ts +++ b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeContract.ts @@ -27,10 +27,14 @@ export type LocalServiceManagerIntentLineage = generation: number; expectedActivationDigest: string; previousRecordDigest: string; + completionFence?: Readonly<{ + expectedInstanceHeadDigest: string; + expectedCompletionDigest: string; + }>; }>; export interface LocalServiceManagerIntent { - readonly schemaVersion: 1; + readonly schemaVersion: 1 | 2; readonly kind: 'qinglong3-local-service-manager-intent'; readonly actionId: string; readonly action: LocalServiceManagerAction; @@ -169,7 +173,10 @@ function safeInteger(value: unknown, label: string, maximum: number): number { return value as number; } -function normalizedLineage(value: unknown): LocalServiceManagerIntentLineage { +function normalizedLineage( + value: unknown, + schemaVersion: 1 | 2, +): LocalServiceManagerIntentLineage { const lineage = object(value, 'lineage'); if (lineage.mode === 'fresh') { exact(lineage, ['mode'], 'lineage'); @@ -179,6 +186,7 @@ function normalizedLineage(value: unknown): LocalServiceManagerIntentLineage { lineage, [ 'cutoverId', + ...(schemaVersion === 2 ? ['completionFence'] : []), 'expectedActivationDigest', 'generation', 'mode', @@ -186,6 +194,17 @@ function normalizedLineage(value: unknown): LocalServiceManagerIntentLineage { ], 'lineage', ); + const completionFence = + schemaVersion === 2 + ? object(lineage.completionFence, 'completion fence') + : undefined; + if (completionFence !== undefined) { + exact( + completionFence, + ['expectedCompletionDigest', 'expectedInstanceHeadDigest'], + 'completion fence', + ); + } if ( lineage.mode !== 'adopted' || typeof lineage.cutoverId !== 'string' || @@ -196,7 +215,12 @@ function normalizedLineage(value: unknown): LocalServiceManagerIntentLineage { typeof lineage.expectedActivationDigest !== 'string' || !DIGEST_PATTERN.test(lineage.expectedActivationDigest) || typeof lineage.previousRecordDigest !== 'string' || - !DIGEST_PATTERN.test(lineage.previousRecordDigest) + !DIGEST_PATTERN.test(lineage.previousRecordDigest) || + (completionFence !== undefined && + (typeof completionFence.expectedInstanceHeadDigest !== 'string' || + !DIGEST_PATTERN.test(completionFence.expectedInstanceHeadDigest) || + typeof completionFence.expectedCompletionDigest !== 'string' || + !DIGEST_PATTERN.test(completionFence.expectedCompletionDigest))) ) { configurationError('adopted lineage is invalid'); } @@ -206,6 +230,16 @@ function normalizedLineage(value: unknown): LocalServiceManagerIntentLineage { generation: lineage.generation as number, expectedActivationDigest: lineage.expectedActivationDigest, previousRecordDigest: lineage.previousRecordDigest, + ...(completionFence === undefined + ? {} + : { + completionFence: Object.freeze({ + expectedInstanceHeadDigest: + completionFence.expectedInstanceHeadDigest as string, + expectedCompletionDigest: + completionFence.expectedCompletionDigest as string, + }), + }), }); } @@ -297,9 +331,12 @@ export function normalizeLocalServiceManagerIntent( : '/etc/init.d/qinglong3'; const expectedSourceMode = service.kind === 'systemd' ? 0o600 : 0o700; const expectedDestinationMode = service.kind === 'systemd' ? 0o644 : 0o755; - const lineage = normalizedLineage(intent.lineage); + if (intent.schemaVersion !== 1 && intent.schemaVersion !== 2) { + configurationError('service manager intent schema version is invalid'); + } + const schemaVersion = intent.schemaVersion; + const lineage = normalizedLineage(intent.lineage, schemaVersion); if ( - intent.schemaVersion !== 1 || intent.kind !== 'qinglong3-local-service-manager-intent' || typeof intent.actionId !== 'string' || !UUID_V4_PATTERN.test(intent.actionId) || @@ -336,20 +373,27 @@ export function normalizeLocalServiceManagerIntent( } if ( lineage.mode === 'adopted' && - ((lineage.generation === 1 && - intent.action !== 'install-enable-start' && - intent.action !== 'start' && - intent.action !== 'stop') || - (lineage.generation >= 2 && - intent.action !== 'restart' && - intent.action !== 'stop')) + (schemaVersion === 2 + ? lineage.generation < 2 || intent.action !== 'restart' + : (lineage.generation === 1 && + intent.action !== 'install-enable-start' && + intent.action !== 'start' && + intent.action !== 'stop') || + (lineage.generation >= 2 && + intent.action !== 'restart' && + intent.action !== 'stop')) ) { configurationError( 'service manager action does not match the adopted generation', ); } + if (schemaVersion === 2 && lineage.mode !== 'adopted') { + configurationError( + 'service manager v2 intent requires adopted completion lineage', + ); + } const payload = Object.freeze({ - schemaVersion: 1 as const, + schemaVersion, kind: 'qinglong3-local-service-manager-intent' as const, actionId: intent.actionId, action: intent.action, diff --git a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverConsumer.ts b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverConsumer.ts index ee3dfa2a..dbd227e1 100644 --- a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverConsumer.ts +++ b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverConsumer.ts @@ -84,6 +84,7 @@ export interface LocalServiceManagerCutoverDependencies { readonly procRoot?: string; readonly now?: () => number; readonly wait?: (milliseconds: number) => Promise; + readonly afterRecordPublished?: () => void; } interface AdoptedBinding { @@ -736,16 +737,28 @@ function replayResult( ): Readonly | undefined { if (intent.lineage.mode !== 'adopted') return undefined; const state = desiredState(intent, outcome); - const recordPath = localServiceManagerCutoverRecordPath(intent, state); - if (!fs.existsSync(recordPath)) return undefined; + const candidatePaths = [ + localServiceManagerCutoverRecordPath(intent, state), + ...(state === 'manual_required' + ? [] + : [localServiceManagerCutoverRecordPath(intent, 'manual_required')]), + ].filter((candidate) => fs.existsSync(candidate)); + if (candidatePaths.length === 0) return undefined; + if (candidatePaths.length !== 1) { + configurationError('service manager cutover replay is ambiguous'); + } const record = normalizeLocalServiceManagerCutoverRecord( - readPrivateLocalCommandFile(recordPath), + readPrivateLocalCommandFile(candidatePaths[0]!), ); - const head = readLocalCutoverInstanceHead( + let head = readLocalCutoverInstanceHead( intent.deployment.root, intent.instanceId, currentIdentity().uid, ); + const completionFence = + intent.schemaVersion === 2 && intent.lineage.mode === 'adopted' + ? intent.lineage.completionFence + : undefined; const stoppedCaptureProgress = record.state === 'target_stopped' && (head.state === 'reconciliation_capture_prepared' || @@ -759,17 +772,58 @@ function replayResult( head.state === 'reconciliation_automation_planned' || head.state === 'reconciliation_automation_decision_prepared' || head.state === 'reconciliation_automation_reviewed'); + const completionRestartPendingHead = + (record.state === 'target_active' || record.state === 'manual_required') && + completionFence !== undefined && + record.schemaVersion === 3 && + record.evidence.completionFence?.expectedInstanceHeadDigest === + completionFence.expectedInstanceHeadDigest && + record.evidence.completionFence.expectedCompletionDigest === + completionFence.expectedCompletionDigest && + head.state === 'reconciliation_completed' && + head.headDigest === completionFence.expectedInstanceHeadDigest && + head.sourceRecordDigest === completionFence.expectedCompletionDigest; if ( record.actionId !== intent.actionId || record.intentDigest !== intent.intentDigest || record.evidence.managerOutcomeDigest !== outcome.outcomeDigest || + record.profile !== intent.profile || + record.instanceId !== intent.instanceId || + record.activationDigest !== intent.lineage.expectedActivationDigest || + record.generation !== intent.lineage.generation || + record.previousRecordDigest !== intent.lineage.previousRecordDigest || head.cutoverId !== intent.lineage.cutoverId || - head.generation !== intent.lineage.generation || - (!stoppedCaptureProgress && head.state !== record.state) || - (!stoppedCaptureProgress && head.sourceRecordDigest !== record.recordDigest) + head.generation !== + (completionRestartPendingHead + ? intent.lineage.generation - 1 + : intent.lineage.generation) || + (!stoppedCaptureProgress && + !completionRestartPendingHead && + head.state !== record.state) || + (!stoppedCaptureProgress && + !completionRestartPendingHead && + head.sourceRecordDigest !== record.recordDigest) ) { configurationError('service manager cutover replay drifted'); } + if (completionRestartPendingHead) { + head = advanceLocalCutoverInstanceHead( + { + options: { deploymentRoot: intent.deployment.root }, + request: { + cutoverId: intent.lineage.cutoverId, + profile: intent.profile, + instanceId: intent.instanceId, + expectedActivationDigest: intent.lineage.expectedActivationDigest, + requestedAtMs: record.completedAtMs, + }, + }, + currentIdentity().uid, + record.state, + intent.lineage.generation, + record.recordDigest, + ); + } return Object.freeze({ schemaVersion: 1 as const, operation: command.operation, @@ -895,6 +949,12 @@ export async function consumeLocalServiceManagerCutoverOutcome( shutdownReceiptDigest, processIdentityDigest, manualReason, + ...(intent.schemaVersion === 2 && + intent.lineage.completionFence !== undefined + ? { + completionFence: intent.lineage.completionFence, + } + : {}), }); const record = localServiceManagerCutoverRecord( intent, @@ -907,6 +967,7 @@ export async function consumeLocalServiceManagerCutoverOutcome( record, identity.uid, ); + dependencies.afterRecordPublished?.(); const head = advanceLocalCutoverInstanceHead( { options: { deploymentRoot: intent.deployment.root }, diff --git a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverJournal.ts b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverJournal.ts index e53ecc7b..c3dd1aa2 100644 --- a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverJournal.ts +++ b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverJournal.ts @@ -33,11 +33,15 @@ export interface LocalServiceManagerCutoverEvidence { readonly shutdownReceiptDigest: string | null; readonly processIdentityDigest: string | null; readonly manualReason: string | null; + readonly completionFence?: Readonly<{ + expectedInstanceHeadDigest: string; + expectedCompletionDigest: string; + }>; } export interface LocalServiceManagerCutoverRecord { readonly schema: typeof SCHEMA; - readonly schemaVersion: 1 | 2; + readonly schemaVersion: 1 | 2 | 3; readonly actionId: string; readonly action: LocalServiceManagerAction; readonly state: LocalServiceManagerCutoverState; @@ -145,6 +149,7 @@ export function localServiceManagerCutoverRecord( evidence, 'legacyDataApplicationReceiptDigest', ); + const hasCompletionFence = Object.hasOwn(evidence, 'completionFence'); if ( hasCommitDigest !== hasReceiptDigest || (hasCommitDigest && @@ -153,9 +158,25 @@ export function localServiceManagerCutoverRecord( ) { configurationError('legacy data application evidence is incomplete'); } + if ( + hasCompletionFence !== (intent.schemaVersion === 2) || + (hasCompletionFence && + (intent.lineage.mode !== 'adopted' || + intent.lineage.completionFence === undefined || + evidence.completionFence?.expectedInstanceHeadDigest !== + intent.lineage.completionFence.expectedInstanceHeadDigest || + evidence.completionFence?.expectedCompletionDigest !== + intent.lineage.completionFence.expectedCompletionDigest)) + ) { + configurationError('service manager completion evidence is incomplete'); + } const payload = Object.freeze({ schema: SCHEMA, - schemaVersion: hasCommitDigest ? (2 as const) : (1 as const), + schemaVersion: hasCompletionFence + ? (3 as const) + : hasCommitDigest + ? (2 as const) + : (1 as const), actionId: intent.actionId, action: intent.action, state, @@ -213,6 +234,12 @@ export function normalizeLocalServiceManagerCutoverRecord( 'legacyDataApplicationCommitDigest', 'legacyDataApplicationReceiptDigest', ] + : record.schemaVersion === 3 + ? [ + 'completionFence', + 'legacyDataApplicationCommitDigest', + 'legacyDataApplicationReceiptDigest', + ] : []), 'manualReason', 'processIdentityDigest', @@ -226,9 +253,22 @@ export function normalizeLocalServiceManagerCutoverRecord( candidate === null || (typeof candidate === 'string' && DIGEST_PATTERN.test(candidate)); const { recordDigest, ...payload } = record; + const completionFence = + record.schemaVersion === 3 + ? object(evidence.completionFence, 'service manager completion evidence') + : undefined; + if (completionFence !== undefined) { + exact( + completionFence, + ['expectedCompletionDigest', 'expectedInstanceHeadDigest'], + 'service manager completion evidence', + ); + } if ( record.schema !== SCHEMA || - (record.schemaVersion !== 1 && record.schemaVersion !== 2) || + (record.schemaVersion !== 1 && + record.schemaVersion !== 2 && + record.schemaVersion !== 3) || typeof record.actionId !== 'string' || (record.action !== 'install-enable-start' && record.action !== 'start' && @@ -244,6 +284,8 @@ export function normalizeLocalServiceManagerCutoverRecord( !DIGEST_PATTERN.test(record.activationDigest) || !Number.isSafeInteger(record.generation) || (record.generation as number) < 1 || + (record.schemaVersion === 3 && + (record.action !== 'restart' || (record.generation as number) < 2)) || typeof record.previousRecordDigest !== 'string' || !DIGEST_PATTERN.test(record.previousRecordDigest) || typeof record.intentDigest !== 'string' || @@ -264,11 +306,16 @@ export function normalizeLocalServiceManagerCutoverRecord( !DIGEST_PATTERN.test(evidence.commitmentDigest) || typeof evidence.targetDataIdentityDigest !== 'string' || !DIGEST_PATTERN.test(evidence.targetDataIdentityDigest) || - (record.schemaVersion === 2 && + ((record.schemaVersion === 2 || record.schemaVersion === 3) && (!nullableDigest(evidence.legacyDataApplicationCommitDigest) || !nullableDigest(evidence.legacyDataApplicationReceiptDigest) || (evidence.legacyDataApplicationCommitDigest === null) !== (evidence.legacyDataApplicationReceiptDigest === null))) || + (completionFence !== undefined && + (typeof completionFence.expectedInstanceHeadDigest !== 'string' || + !DIGEST_PATTERN.test(completionFence.expectedInstanceHeadDigest) || + typeof completionFence.expectedCompletionDigest !== 'string' || + !DIGEST_PATTERN.test(completionFence.expectedCompletionDigest))) || !nullableDigest(evidence.startupReceiptDigest) || !nullableDigest(evidence.shutdownReceiptDigest) || !nullableDigest(evidence.processIdentityDigest) || diff --git a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceManagerIntent.ts b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceManagerIntent.ts index a28096c6..38fd76e7 100644 --- a/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceManagerIntent.ts +++ b/packages/ql3-local-owner-cli/src/deployment/service-manager/serviceManagerIntent.ts @@ -34,6 +34,7 @@ import { normalizeLocalServiceManagerOutcome, type LocalServiceManagerOutcome, } from './serviceOutcomeContract'; +import { readLocalServiceManagerActiveRecord } from './serviceCutoverJournal'; const MAX_PATH_BYTES = 4_096; const MAX_DESCRIPTOR_BYTES = 64 * 1024; @@ -43,7 +44,7 @@ const UUID_V4_PATTERN = const DIGEST_PATTERN = /^[0-9a-f]{64}$/; export interface LocalServiceManagerIntentPrepareCommand { - readonly schemaVersion: 1; + readonly schemaVersion: 1 | 2; readonly operation: 'local.deployment.service-manager.intent.prepare'; readonly options: Readonly<{ deploymentRoot: string; @@ -454,6 +455,39 @@ function assertIntentLineageHead( }), uid, ); + if (intent.schemaVersion === 2) { + if ( + intent.action !== 'restart' || + intent.lineage.completionFence === undefined || + head.state !== 'reconciliation_completed' || + head.generation !== intent.lineage.generation - 1 || + head.headDigest !== + intent.lineage.completionFence.expectedInstanceHeadDigest || + head.sourceRecordDigest !== + intent.lineage.completionFence.expectedCompletionDigest || + intent.requestedAtMs < head.updatedAtMs + ) { + configurationError( + 'service manager completion restart lost the instance head compare-and-swap', + ); + } + const previous = readLocalServiceManagerActiveRecord( + intent.deployment.root, + intent.lineage.cutoverId, + intent.lineage.generation - 1, + ); + if ( + previous.recordDigest !== intent.lineage.previousRecordDigest || + previous.profile !== intent.profile || + previous.instanceId !== intent.instanceId || + previous.activationDigest !== intent.lineage.expectedActivationDigest + ) { + configurationError( + 'service manager completion restart lost the previous active record', + ); + } + return; + } const expected = intent.action === 'restart' ? Object.freeze({ @@ -495,7 +529,7 @@ function normalizePrepareCommand( 'request', ); if ( - command.schemaVersion !== 1 || + (command.schemaVersion !== 1 && command.schemaVersion !== 2) || command.operation !== 'local.deployment.service-manager.intent.prepare' || typeof request.actionId !== 'string' || !UUID_V4_PATTERN.test(request.actionId) || @@ -510,7 +544,7 @@ function normalizePrepareCommand( configurationError('service manager intent command is invalid'); } return Object.freeze({ - schemaVersion: 1 as const, + schemaVersion: command.schemaVersion, operation: 'local.deployment.service-manager.intent.prepare' as const, options: Object.freeze({ deploymentRoot: safeAbsolutePath( @@ -576,7 +610,7 @@ export function prepareLocalServiceManagerIntent( const application = parseApplicationIdentity(applicationBytes); const payload: Omit = Object.freeze({ - schemaVersion: 1 as const, + schemaVersion: command.schemaVersion, kind: 'qinglong3-local-service-manager-intent' as const, actionId: command.request.actionId, action: command.request.action, diff --git a/packages/ql3-local-owner-cli/test/serviceBridgeContract.test.cjs b/packages/ql3-local-owner-cli/test/serviceBridgeContract.test.cjs index 877e5a5f..d8fff9a2 100644 --- a/packages/ql3-local-owner-cli/test/serviceBridgeContract.test.cjs +++ b/packages/ql3-local-owner-cli/test/serviceBridgeContract.test.cjs @@ -92,6 +92,32 @@ test('normalizes exact systemd fresh and OpenRC adopted intents', () => { ), }); assert.deepEqual(normalizeLocalServiceManagerIntent(openrc), openrc); + + const completedRestart = intent({ + schemaVersion: 2, + actionId: '123e4567-e89b-42d3-a456-426614174003', + action: 'restart', + lineage: { + mode: 'adopted', + cutoverId: 'router-edge-1-cutover', + generation: 2, + expectedActivationDigest: 'c'.repeat(64), + previousRecordDigest: 'd'.repeat(64), + completionFence: { + expectedInstanceHeadDigest: '1'.repeat(64), + expectedCompletionDigest: '2'.repeat(64), + }, + }, + outcomePath: path.join( + '/opt/qinglong3', + 'service/service-manager-outcomes', + '123e4567-e89b-42d3-a456-426614174003.json', + ), + }); + assert.deepEqual( + normalizeLocalServiceManagerIntent(completedRestart), + completedRestart, + ); }); test('rejects arbitrary destinations, root drift, digest drift and unknown fields', () => { @@ -127,6 +153,30 @@ test('rejects arbitrary destinations, root drift, digest drift and unknown field previousRecordDigest: 'd'.repeat(64), }, }), + intent({ + schemaVersion: 2, + action: 'restart', + lineage: { + mode: 'adopted', + cutoverId: 'router-edge-1-cutover', + generation: 2, + expectedActivationDigest: 'c'.repeat(64), + previousRecordDigest: 'd'.repeat(64), + }, + }), + intent({ + lineage: { + mode: 'adopted', + cutoverId: 'router-edge-1-cutover', + generation: 2, + expectedActivationDigest: 'c'.repeat(64), + previousRecordDigest: 'd'.repeat(64), + completionFence: { + expectedInstanceHeadDigest: '1'.repeat(64), + expectedCompletionDigest: '2'.repeat(64), + }, + }, + }), ]) { assert.throws( () => normalizeLocalServiceManagerIntent(candidate), diff --git a/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs b/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs index e06b654b..0a8850cf 100644 --- a/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs +++ b/packages/ql3-local-owner-cli/test/serviceCutoverConsumer.test.cjs @@ -403,6 +403,110 @@ async function stopAdoptedTarget(state, suffix = '031') { return { stoppedResult, head }; } +async function completeStoppedServiceTarget(state, suffix = '061') { + const active = prepare( + state, + 1, + 'install-enable-start', + state.commitmentDigest, + `123e4567-e89b-42d3-a456-426614174${suffix}`, + ); + publishOutcome(active, 'install-enable-start', 'active', 6123, 1786416000200); + const startupReceiptDigest = publishReceipt(state, 6123, '100061'); + const activeResult = await consumeLocalServiceManagerCutoverOutcome( + consumeCommand(state, active), + { procRoot: state.procRoot }, + ); + const stopped = prepare( + state, + 1, + 'stop', + activeResult.recordDigest, + `123e4567-e89b-42d3-a456-426614174${String(Number(suffix) + 1).padStart( + 3, + '0', + )}`, + ); + publishOutcome(stopped, 'stop', 'stopped', 0, 1786416000300); + publishShutdownReceipt(state, 6123, '100061', startupReceiptDigest); + fs.rmSync(path.join(state.procRoot, '6123'), { + recursive: true, + force: true, + }); + const stoppedResult = await consumeLocalServiceManagerCutoverOutcome( + consumeCommand(state, stopped), + { procRoot: state.procRoot }, + ); + const states = [ + 'reconciliation_capture_prepared', + 'reconciliation_captured', + 'reconciliation_plan_prepared', + 'reconciliation_planned', + 'reconciliation_review_prepared', + 'reconciliation_reviewed', + 'reconciliation_application_prepared', + 'reconciliation_application_planned', + 'reconciliation_completed', + ]; + let completedHead; + for (const [index, nextState] of states.entries()) { + completedHead = advanceLocalCutoverInstanceHead( + { + options: { deploymentRoot: state.root }, + request: { + cutoverId: state.cutoverId, + profile: 'edge', + instanceId: 'edge-router-1', + expectedActivationDigest: state.activationDigest, + requestedAtMs: 1786416000400 + index, + }, + }, + process.getuid(), + nextState, + 1, + index === states.length - 1 + ? sha256('service-manager-completion-receipt') + : sha256(`service-manager-${nextState}`), + ); + } + return { activeResult, stoppedResult, completedHead }; +} + +function prepareCompletionRestart( + state, + previousRecordDigest, + completedHead, + actionId, + lineageOverrides = {}, +) { + return prepareLocalServiceManagerIntent({ + schemaVersion: 2, + operation: 'local.deployment.service-manager.intent.prepare', + options: { + deploymentRoot: state.root, + allowRootService: process.getuid() === 0, + }, + request: { + actionId, + action: 'restart', + serviceKind: 'systemd', + lineage: { + mode: 'adopted', + cutoverId: state.cutoverId, + generation: 2, + expectedActivationDigest: state.activationDigest, + previousRecordDigest, + completionFence: { + expectedInstanceHeadDigest: completedHead.headDigest, + expectedCompletionDigest: completedHead.sourceRecordDigest, + }, + ...lineageOverrides, + }, + requestedAtMs: 1786416000600, + }, + }); +} + function rollbackPrepareCommand(state, stoppedResult, head) { return { schemaVersion: 1, @@ -646,6 +750,165 @@ test('restart cannot reuse the previous generation startup receipt', async (t) = assert.equal(head.state, 'manual_required'); }); +test('restarts a completed service lineage with dual evidence and replays record-first response loss', async (t) => { + const state = fixture(t); + const { activeResult, completedHead } = await completeStoppedServiceTarget( + state, + ); + const restart = prepareCompletionRestart( + state, + activeResult.recordDigest, + completedHead, + '123e4567-e89b-42d3-a456-426614174063', + ); + publishOutcome(restart, 'restart', 'active', 7123, 1786416000700); + const startupReceiptDigest = publishReceipt(state, 7123, '100063'); + const command = consumeCommand(state, restart); + let injected = false; + await assert.rejects( + () => + consumeLocalServiceManagerCutoverOutcome(command, { + procRoot: state.procRoot, + afterRecordPublished() { + if (!injected) { + injected = true; + throw new Error('simulated service record response loss'); + } + }, + }), + /simulated service record response loss/, + ); + const pendingHead = readLocalCutoverInstanceHead( + state.root, + 'edge-router-1', + process.getuid(), + ); + assert.equal(pendingHead.state, 'reconciliation_completed'); + assert.equal(pendingHead.headDigest, completedHead.headDigest); + + const record = JSON.parse( + fs.readFileSync( + path.join( + state.root, + 'service', + 'cutovers', + state.cutoverId, + 'service-manager-g02-active.json', + ), + 'utf8', + ), + ); + assert.equal(record.schemaVersion, 3); + assert.equal(record.evidence.startupReceiptDigest, startupReceiptDigest); + assert.deepEqual(record.evidence.completionFence, { + expectedInstanceHeadDigest: completedHead.headDigest, + expectedCompletionDigest: completedHead.sourceRecordDigest, + }); + + const replay = await consumeLocalServiceManagerCutoverOutcome(command, { + procRoot: state.procRoot, + }); + assert.equal(replay.status, 'existing'); + assert.equal(replay.state, 'target_active'); + const activeHead = readLocalCutoverInstanceHead( + state.root, + 'edge-router-1', + process.getuid(), + ); + assert.equal(activeHead.state, 'target_active'); + assert.equal(activeHead.generation, 2); + assert.equal(activeHead.sourceRecordDigest, replay.recordDigest); +}); + +test('rejects completed service restart without both the prior record and exact completion head', async (t) => { + const state = fixture(t); + const { activeResult, completedHead } = await completeStoppedServiceTarget( + state, + '071', + ); + assert.throws( + () => + prepare( + state, + 2, + 'restart', + activeResult.recordDigest, + '123e4567-e89b-42d3-a456-426614174073', + ), + /lost the instance lineage compare-and-swap/, + ); + assert.throws( + () => + prepareCompletionRestart( + state, + activeResult.recordDigest, + completedHead, + '123e4567-e89b-42d3-a456-426614174074', + { + completionFence: { + expectedInstanceHeadDigest: completedHead.headDigest, + expectedCompletionDigest: '0'.repeat(64), + }, + }, + ), + /completion restart lost the instance head compare-and-swap/, + ); + assert.throws( + () => + prepareCompletionRestart( + state, + '0'.repeat(64), + completedHead, + '123e4567-e89b-42d3-a456-426614174075', + ), + /completion restart lost the previous active record/, + ); + const unchanged = readLocalCutoverInstanceHead( + state.root, + 'edge-router-1', + process.getuid(), + ); + assert.equal(unchanged.headDigest, completedHead.headDigest); + assert.equal(unchanged.state, 'reconciliation_completed'); +}); + +test('terminalizes and replays a completed service restart without a new startup receipt', async (t) => { + const state = fixture(t); + const { activeResult, completedHead } = await completeStoppedServiceTarget( + state, + '081', + ); + const restart = prepareCompletionRestart( + state, + activeResult.recordDigest, + completedHead, + '123e4567-e89b-42d3-a456-426614174083', + ); + publishOutcome(restart, 'restart', 'active', 8123, 1786416000700); + let clock = 0; + const command = consumeCommand(state, restart); + const result = await consumeLocalServiceManagerCutoverOutcome(command, { + procRoot: state.procRoot, + now: () => clock, + wait: async (milliseconds) => { + clock += milliseconds; + }, + }); + assert.equal(result.state, 'manual_required'); + const terminal = readLocalCutoverInstanceHead( + state.root, + 'edge-router-1', + process.getuid(), + ); + assert.equal(terminal.state, 'manual_required'); + assert.equal(terminal.sourceRecordDigest, result.recordDigest); + const replay = await consumeLocalServiceManagerCutoverOutcome(command, { + procRoot: state.procRoot, + }); + assert.equal(replay.status, 'existing'); + assert.equal(replay.state, 'manual_required'); +}); + test('stop advances only after the exact receipted process identity disappears', async (t) => { const state = fixture(t); const first = prepare( diff --git a/packages/ql3-worker-runtime/test/workerPosixExecutionExecutor.test.cjs b/packages/ql3-worker-runtime/test/workerPosixExecutionExecutor.test.cjs index f9a04d49..e796a30d 100644 --- a/packages/ql3-worker-runtime/test/workerPosixExecutionExecutor.test.cjs +++ b/packages/ql3-worker-runtime/test/workerPosixExecutionExecutor.test.cjs @@ -49,7 +49,11 @@ async function preparedOutput(artifactRoot, offerId = 'offer-posix-1') { const allocator = new WorkerFileLogArtifactAllocator({ root: artifactRoot, policy: workerRemoteLogArtifactPolicy('edge'), - capacity: { async availableBytes() { return 1024n ** 4n; } }, + capacity: { + async availableBytes() { + return 1024n ** 4n; + }, + }, }); const prepared = await allocator.prepare({ projectId: 'project-1', @@ -84,7 +88,7 @@ function launch(prepared, output, overrides = {}) { async function waitForReceipt(root) { const store = new CompletionReceiptFileStore(root); - for (let attempt = 0; attempt < 100; attempt += 1) { + for (let attempt = 0; attempt < 500; attempt += 1) { const receipt = await store.read(ATTEMPT_ID); if (receipt) return receipt; await new Promise((resolve) => setTimeout(resolve, 10)); @@ -97,7 +101,11 @@ test('verifies the Worker barrier, launches through the reviewed fd and writes a const { prepared, output } = await preparedOutput(roots.artifactRoot); let barrier; const executor = new WorkerPosixExecutionExecutor({ - barrier: { async verify(input) { barrier = input; } }, + barrier: { + async verify(input) { + barrier = input; + }, + }, receiptRoot: roots.receiptRoot, identityProvider: identityProvider(), clock: { now: () => 100 }, @@ -128,13 +136,19 @@ test('does not spawn when the durable Worker barrier rejects authority', async ( const marker = path.join(roots.root, 'spawned'); const { prepared, output } = await preparedOutput(roots.artifactRoot); const executor = new WorkerPosixExecutionExecutor({ - barrier: { async verify() { throw new Error('stale inbox'); } }, + barrier: { + async verify() { + throw new Error('stale inbox'); + }, + }, receiptRoot: roots.receiptRoot, identityProvider: identityProvider(), }); - const result = await executor.start(launch(prepared, output, { - command: { kind: 'argv', file: '/usr/bin/touch', args: [marker] }, - })); + const result = await executor.start( + launch(prepared, output, { + command: { kind: 'argv', file: '/usr/bin/touch', args: [marker] }, + }), + ); assert.deepEqual(result, { status: 'rejected' }); await assert.rejects(fs.stat(marker), { code: 'ENOENT' }); }); @@ -144,13 +158,19 @@ test('rejects timeout without durable control-plane deadline before spawn', asyn const { prepared, output } = await preparedOutput(roots.artifactRoot); let barriers = 0; const executor = new WorkerPosixExecutionExecutor({ - barrier: { async verify() { barriers += 1; } }, + barrier: { + async verify() { + barriers += 1; + }, + }, receiptRoot: roots.receiptRoot, identityProvider: identityProvider(), }); - const result = await executor.start(launch(prepared, output, { - timeoutMs: 1_000, - })); + const result = await executor.start( + launch(prepared, output, { + timeoutMs: 1_000, + }), + ); assert.deepEqual(result, { status: 'rejected' }); assert.equal(barriers, 0); }); @@ -160,14 +180,20 @@ test('accepts timeout only when starting ACK supplied a durable deadline', async const { prepared, output } = await preparedOutput(roots.artifactRoot); let barriers = 0; const executor = new WorkerPosixExecutionExecutor({ - barrier: { async verify() { barriers += 1; } }, + barrier: { + async verify() { + barriers += 1; + }, + }, receiptRoot: roots.receiptRoot, identityProvider: identityProvider(), }); - const result = await executor.start(launch(prepared, output, { - timeoutMs: 1_000, - executionDeadlineAtMs: 2_000, - })); + const result = await executor.start( + launch(prepared, output, { + timeoutMs: 1_000, + executionDeadlineAtMs: 2_000, + }), + ); assert.equal(result.status, 'started'); assert.equal(barriers, 1); await waitForReceipt(roots.receiptRoot); @@ -180,14 +206,20 @@ test('propagates unknown outcome when durable identity capture fails after spawn barrier: { async verify() {} }, receiptRoot: roots.receiptRoot, identityProvider: { - async capture() { throw new Error('procfs unavailable'); }, - async inspect() { return { status: 'unknown' }; }, + async capture() { + throw new Error('procfs unavailable'); + }, + async inspect() { + return { status: 'unknown' }; + }, }, }); await assert.rejects( - executor.start(launch(prepared, output, { - command: { kind: 'shell', command: 'sleep 5', shell: '/bin/sh' }, - })), + executor.start( + launch(prepared, output, { + command: { kind: 'shell', command: 'sleep 5', shell: '/bin/sh' }, + }), + ), (error) => error?.spawnOutcome === 'unknown', ); });