From 8496b56bed7cbc2227329329fa9cc50ccc1dab46 Mon Sep 17 00:00:00 2001 From: whyour Date: Mon, 17 Aug 2026 23:45:34 +0800 Subject: [PATCH] feat(ql3): fence Kubernetes resource retirement --- .../ql3-kubernetes-deployment-live.yml | 90 + docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 26 +- ...n-fenced-kubernetes-resource-retirement.md | 96 ++ docs/adr/README.md | 1 + docs/operations/ql3-release-set-deployment.md | 56 + .../ql3-kubernetes-deployment-ceremony.cjs | 1469 ++++++++++++++++- ...l3-kubernetes-deployment-live-contract.cjs | 160 +- .../ql3KubernetesDeploymentCeremony.test.cjs | 378 +++++ 8 files changed, 2242 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/ql3-kubernetes-deployment-live.yml create mode 100644 docs/adr/ADR-0435-uid-resource-version-fenced-kubernetes-resource-retirement.md diff --git a/.github/workflows/ql3-kubernetes-deployment-live.yml b/.github/workflows/ql3-kubernetes-deployment-live.yml new file mode 100644 index 00000000..90910696 --- /dev/null +++ b/.github/workflows/ql3-kubernetes-deployment-live.yml @@ -0,0 +1,90 @@ +name: QingLong 3.0 Kubernetes deployment live contract + +on: + push: + branches: + - next + pull_request: + branches: + - next + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ql3-kubernetes-deployment-live-${{ github.ref }} + cancel-in-progress: true + +jobs: + kubernetes-deployment-live: + name: Deployment Head and fenced resource retirement + runs-on: ubuntu-24.04 + timeout-minutes: 25 + steps: + - uses: actions/checkout@v6 + - uses: pnpm/action-setup@v6 + with: + version: '8.3.1' + - uses: actions/setup-node@v6 + with: + node-version: '24.18.0' + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + + - name: Install workspace dependencies without lifecycle scripts + run: pnpm install --frozen-lockfile --ignore-scripts + + - name: Install verified kubectl v1.34.3 + shell: bash + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + --output "${RUNNER_TEMP}/kubectl" \ + https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl + curl --fail --location --silent --show-error \ + --output "${RUNNER_TEMP}/kubectl.sha256" \ + https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl.sha256 + test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \ + "$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)" + chmod 0755 "${RUNNER_TEMP}/kubectl" + + - name: Preload the digest-reviewed K3s distribution + run: docker pull rancher/k3s:v1.34.3-k3s1 + + - name: Recheck the closed deployment ceremony contract + run: node --test test/back/ql3KubernetesDeploymentCeremony.test.cjs + + - name: Prove install, UID/resourceVersion retirement and Head convergence + env: + QL3_KUBECTL_BIN: ${{ runner.temp }}/kubectl + QL3_DEPLOYMENT_LIVE_REPORT: ${{ runner.temp }}/ql3-kubernetes-deployment/report.json + shell: bash + run: | + set -euo pipefail + umask 077 + install -d -m 0700 "$(dirname "${QL3_DEPLOYMENT_LIVE_REPORT}")" + node scripts/ql3-kubernetes-deployment-live-contract.cjs > \ + "${QL3_DEPLOYMENT_LIVE_REPORT}" + node -e \ + "const fs=require('node:fs');const report=JSON.parse(fs.readFileSync(process.env.QL3_DEPLOYMENT_LIVE_REPORT,'utf8'));if(report.cleanupComplete!==true||report.receiptAuditCompatible!==true||report.retirement?.receiptAuditCompatible!==true||report.retirement?.targetAbsent!==true||report.retirement?.uidResourceVersionDeletePreconditions!==true||report.retirement?.deploymentHeadCas!==true||report.retirement?.unixSocketProxy!==true)process.exit(1)" + + - name: Verify isolated cleanup + if: always() + shell: bash + run: | + set -euo pipefail + test -z "$(docker ps -aq --filter name=ql3-deploy-live-)" + test -z "$(docker network ls -q --filter name=ql3-deploy-live-)" + + - name: Upload content-free deployment evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ql3-kubernetes-deployment-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/ql3-kubernetes-deployment/report.json + if-no-files-found: warn + retention-days: 14 + compression-level: 9 + overwrite: false + include-hidden-files: false diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 87c30f7a..4b9fc996 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -6,10 +6,32 @@ - 目标版本：QingLong 3.x - 作者：QingLong Maintainers - 创建日期：2026-07-17 -- 最后更新：2026-08-16 +- 最后更新：2026-08-17 - 讨论范围：架构与演进路线，不包含最终 UI 视觉方案 -最新增量证据（2026-08-16）： +最新增量证据（2026-08-17）： + +- D-343/ADR-0435（已接受；真实公开 catalog 运行仍待 release Gate）：在 D-342 的 resource-inventory-closed Head 上增加独立 + `cluster.deployment.retirement.preflight|apply|receipt.audit`。退役目标只能来自 current locked manifest 与 committed active inventory， + 最多 64 个显式 namespaced resource；固定 Head、cluster-scoped resource 以及 Secret/PVC/ServiceAccount 均拒绝进入通用删除面，survivor + inventory 必须非空并继续覆盖 `control|control-ai|admin|worker` 四类 authority。preflight 通过 API discovery 绑定每个对象的 exact + UID/resourceVersion、desired subset 与 `qinglong3-catalog-lock` Apply ownership，再发送带同一 Preconditions 的服务端 DeleteOptions + dry-run；apply 在删除前复验 observation，以 Head resourceVersion CAS 取得唯一 `applying` 意图，逐对象使用 Background DELETE 并确认 + old UID absent，之后才提交缩小 inventory 的下一代 committed Head 与 no-replace receipt。替换 UID、resourceVersion/字段漂移、 + terminating/finalizer、Head 冲突均失败关闭；DELETE 已生效但响应丢失时，只允许相同 command/mutation/preflight 从 durable applying + Head 和 absent target 恢复。同版本 rollback 可精确恢复 retirement 前一个 inventory，仍走完整 server-side apply/convergence。 + pinned kubectl 只在 ceremony 期间启动 owner-private Unix socket proxy，由 pinned curl 发送带 body 的 DELETE；不开放 TCP，不读取 + ambient HOME，完成即清理。没有新增 package、第三方依赖、controller、CRD、webhook、ServiceAccount、Pod、数据库、migration、Pool、 + watcher、timer 或常驻进程，Local/Edge/Standalone 零导入、零稳态成本。独立 GitHub Actions job 固定 kubectl/K3s 版本并验证真实 + install、退役、Head 收敛、content-free evidence 与隔离清理；定向 deployment ceremony 22/22，完整 backend 1,339 项为 1,337 pass/ + 2 条件 skip/0 fail，18-package clean build/test 退出 0。架构/部署审计保持 compatible，package boundary 为 18 packages 且 + `singleSourcePackages=[]`、`shallowSourcePackages=[]`；14 档 Local artifact 全部 compatible 且字节数与 D-342 完全一致。三节点 K3s + live 使用 digest-reviewed `rancher/k3s:v1.34.3-k3s1`，在 linux/arm64 上得到 3 个 Ready node，真实 install 后 Head generation + `1→2`、inventory `7→6`、目标 absent，server-side dry-run/apply、UID/resourceVersion DELETE preconditions、Unix socket proxy、Head + CAS 与两个 receipt audit 均通过，`cleanupComplete=true` 且 Docker container/network 残留为 0；最终 retirement receipt digest 为 + `sha256:d3145b3fee7a1191458b5c3a930c0063a0e1e446d3a196565bcdd20d9da06ec5`。PostgreSQL HA 在 18.6/arm64 上完成 + 142/142 gates、timeline `1→2` 与独立 evidence audit，mode-0600 报告 SHA-256 为 + `89fd4fb47f82f35d3819d1fd9540cf3764f65fa68ac6561effd16778bee4ab8a`，container/network/volume 残留为 0。 - D-342/ADR-0434（已接受；资源退休 ceremony 与真实公开 catalog 运行仍待后续 Gate）：Cluster deployment command、preflight 与 receipt 在尚未发布前直接升为 v2，并以目标 namespace 中唯一 `qinglong3-deployment-head` ConfigMap 建立部署顺序 authority。每次 diff --git a/docs/adr/ADR-0435-uid-resource-version-fenced-kubernetes-resource-retirement.md b/docs/adr/ADR-0435-uid-resource-version-fenced-kubernetes-resource-retirement.md new file mode 100644 index 00000000..86ee9a6e --- /dev/null +++ b/docs/adr/ADR-0435-uid-resource-version-fenced-kubernetes-resource-retirement.md @@ -0,0 +1,96 @@ +# ADR-0435：UID/resourceVersion 围栏的 Kubernetes 资源退役 Ceremony + +- 状态：Accepted +- 日期：2026-08-17 +- 关联 RFC：QL-RFC-0001 D-03、D-14、D-342、D-343 +- 关联 ADR：ADR-0431、ADR-0433、ADR-0434 +- 修正：ADR-0434 第 7 项中“资源退休仍待后续 ceremony”的未完成边界 + +## 上下文 + +ADR-0434 已让 Deployment Head 保存完整有序 inventory，并拒绝 upgrade 隐式遗漏资源，但仍没有安全缩小 inventory 的路径。普通 +`kubectl delete` 不执行 `resourceVersion` 比较；只按 kind/namespace/name 删除会在旧对象已被替换时误删新 UID。`kubectl apply +--prune` 也无法把每个删除绑定到 preflight 所观察的 UID/resourceVersion、当前 Head 与独立 receipt。 + +删除还是 Kubernetes 多对象非事务操作。请求在 API Server 已接受后丢失响应、对象被 finalizer 留在 terminating、并发 operator +推进 Head，都会产生不能靠“再跑一次 delete”猜测的窗口。QingLong 3.0 因而需要显式、短生命周期、可恢复的 retirement ceremony， +同时不能给低配 Local/Edge 设备增加 Kubernetes 依赖或常驻成本。 + +## 决策 + +1. 在既有 `qinglong/kubernetes-deployment-command@v2` 中增加 + `cluster.deployment.retirement.preflight|apply|receipt.audit`，不新增 package、controller、CRD、数据库或常驻进程。退役和普通 + install/upgrade/rollback 使用同一个 Deployment Head CAS authority。 +2. 调用方只能提交 current locked manifest 中、current committed inventory 内的显式有序 target identity。每次最多 64 个且只允许 + namespaced resource；Head ConfigMap及 `v1 Secret|PersistentVolumeClaim|ServiceAccount` 禁止进入此通用 ceremony。集群级对象、数据与 + credential authority 必须使用各自专用 ceremony。 +3. preflight 重新验证 catalog-bound lock、pinned kubectl/curl、owner-private kubeconfig、cluster UID 与 exact expected Head，从 API + discovery 得到唯一 plural/scope 映射，逐个 GET 并记录 exact UID/resourceVersion、期望字段与 + `qinglong3-catalog-lock` Apply ownership，再以 `DeleteOptions.dryRun=[All]`、Background propagation 和 UID/resourceVersion + preconditions 执行服务端 dry-run。preflight 不改变 Kubernetes 状态。 +4. survivor inventory 必须非空并继续包含 `control|control-ai|admin|worker` 四类 immutable image authority。退役集合必须精确等于 + `active - survivor`，不能通过手写 receipt 或隐式 prune 改写 inventory。 +5. apply 在任何删除前重读目标对象并要求 UID/resourceVersion 仍与 preflight 完全相同，然后用 Head ConfigMap 的 opaque + resourceVersion 把唯一意图 CAS 为 `phase=applying`。之后逐对象发送带相同 UID/resourceVersion preconditions 的 DELETE；仅在每个 + old UID 已确认 404 absent 后，才把缩小后的 inventory 作为下一 generation 的 `committed` Head 发布，并 no-replace 写 receipt。 +6. DELETE 不使用 force、grace-period=0 或 foreground 级联；固定 Background propagation。对象处于 terminating、precondition 409、 + UID 替换、resourceVersion 漂移、字段漂移、API discovery 歧义、Head 冲突或输入替换均失败关闭，不发布成功 receipt。 +7. 若 DELETE 已完成但响应丢失，Head 仍保存 exact applying intent。相同 command/mutation/preflight 重放时，只有目标已经 absent 才可把 + 该次删除记为 recovered 并继续提交；不同 UID、仍 terminating 或任何非同意图都不能自动接管。若 Head 已 committed 而本地 receipt + 丢失，相同 command 可从 Head 确定性重建。 +8. 普通 kubectl 缺少带 body 的安全 DELETE 接口，因此一次性启动 pinned kubectl 的 owner-private Unix socket proxy，并用 pinned + curl 发送 JSON DeleteOptions。proxy 只接受 `/api|/apis`，拒绝 POST/PUT/PATCH 等无关方法，使用独立 HOME/XDG/TMP，完成即关闭并删除 + socket；不监听 TCP，不读取 ambient HOME。 +9. retirement receipt 显式声明 `crossResourceAtomicity=false`、 + `recovery=resume_exact_retirement_from_target_head`、UID/resourceVersion preconditions、absence、survivor role closure 与 Head CAS。 + offline audit 重建 command/receipt digest 闭包，但保持 `externalResultsReplayed=false`、`kubernetesMutation=false`。 +10. Head schema 在 3.0 尚未正式发布时原地升为 v2；普通 rollback 可在 current transition 为 retire 时，以同一 release version 精确恢复 + Head.previous 的 lock 和完整 inventory。恢复对象仍走原 server-side apply/convergence ceremony，不把 DELETE 本身假装成可逆事务。 + +## 部署与资源影响 + +- Local/Edge/Standalone 不导入该脚本和 Kubernetes/YAML/semver 依赖，产物与稳态进程不变。 +- Cluster 每次 retirement 临时产生一个私有目录、一个 Unix socket proxy 与顺序有界 API 请求；完成后全部清理。没有 listener 常驻、 + watcher、timer、ServiceAccount、Pod、sidecar、Pool、migration 或第三方依赖。 +- 最多 64 个 target，按顺序执行 discovery/read/dry-run/delete/absence；这是明确的运维时成本，不进入 runtime hot path。 +- 多资源删除仍非事务。Head 提供的是唯一意图、精确恢复与可审计 inventory closure，不宣称 API Server 跨对象原子性。 + +## 被拒绝的替代方案 + +### `kubectl delete -f` 或按名称删除 + +拒绝。普通命令不会绑定 preflight 观察到的 `resourceVersion`，旧操作可能删除已经替换的新 UID。 + +### `kubectl apply --prune` + +拒绝。隐式集合差不能形成逐对象 UID/resourceVersion 删除证明，也不能表达响应丢失恢复、finalizer 和 Head inventory commit 顺序。 + +### 常驻 controller/finalizer + +拒绝。QingLong 不是通用 Kubernetes desired-state controller；常驻 reconciler 会扩大凭据、升级、资源占用和低配部署边界。 + +### force delete 或 `grace-period=0` + +拒绝。它会绕过正常终止/级联语义，扩大数据丢失和依赖对象孤儿风险，也不能弥补多对象非事务事实。 + +## 验证 + +- 定向 Node 契约 22/22：原 14 项部署 Head 契约保持通过，新增 exact UID/resourceVersion dry-run、preconditioned delete、inventory + closure、offline audit、替换 UID fail-closed、DELETE 响应丢失恢复、同版本精确 rollback、独立 CI live workflow、curl/kubectl 工具解析隔离 + 与锚定 proxy method 正则闭包； +- 隔离三节点 K3s v1.34.3+k3s1 live 门在 linux/arm64 同时执行 install 与一个 namespaced ConfigMap 退役，验证真实 DeleteOptions + preconditions、对象 absent、Head generation 1→2、inventory 7→6、两个 receipt audit 与临时 Docker/Unix socket 清理；最终 retirement + receipt digest 为 `sha256:d3145b3fee7a1191458b5c3a930c0063a0e1e446d3a196565bcdd20d9da06ec5`，门后 container/network + 残留均为 0。该门固定进入独立 GitHub Actions job，并上传 content-free evidence； +- 完整 backend 为 1,339 total/1,337 pass/2 条件 skip/0 fail；18-package clean build/test 退出 0。架构/边界审计保持 + compatible，14 档 Local artifact 字节数与 D-342 一致； +- PostgreSQL 18.6/arm64 HA 门为 142/142 gates、timeline 1→2，独立 evidence audit compatible；mode-0600 报告 SHA-256 为 + `89fd4fb47f82f35d3819d1fd9540cf3764f65fa68ac6561effd16778bee4ab8a`，门后 container/network/volume 残留均为 0。完整证据记录在 + QL-RFC-0001 D-343。 + +## 规范依据 + +- [Kubernetes API concepts：deletion、resourceVersion 与条件操作](https://kubernetes.io/docs/reference/using-api/api-concepts/) +- [Kubernetes DeleteOptions：UID/resourceVersion Preconditions](https://kubernetes.io/docs/reference/kubernetes-api/definitions/delete-options-v1-meta/) +- [kubectl delete：默认不执行 resourceVersion 检查](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_delete/) +- [Kubernetes cascading deletion：Background propagation](https://kubernetes.io/docs/tasks/administer-cluster/use-cascading-deletion/) diff --git a/docs/adr/README.md b/docs/adr/README.md index 8e7447c0..2759120d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -438,6 +438,7 @@ | [ADR-0432](./ADR-0432-target-side-catalog-bound-local-compose-revisions.md) | 目标侧 Catalog-bound Local Compose 修订 | Accepted | | [ADR-0433](./ADR-0433-catalog-bound-kubernetes-target-deployment-ceremony.md) | Catalog-bound Kubernetes 目标部署 Ceremony | Accepted | | [ADR-0434](./ADR-0434-resource-inventory-closed-kubernetes-deployment-head.md) | Resource-inventory-closed Kubernetes Deployment Head | Accepted | +| [ADR-0435](./ADR-0435-uid-resource-version-fenced-kubernetes-resource-retirement.md) | UID/resourceVersion 围栏的 Kubernetes 资源退役 Ceremony | Accepted | ## 规则 diff --git a/docs/operations/ql3-release-set-deployment.md b/docs/operations/ql3-release-set-deployment.md index 8deb1ee6..d0d04d16 100644 --- a/docs/operations/ql3-release-set-deployment.md +++ b/docs/operations/ql3-release-set-deployment.md @@ -299,6 +299,62 @@ cluster UID，最后以同一 ConfigMap 的新 resourceVersion 提交 `committed mutation 前失败关闭。`rollback` 只接受 Head 中精确的上一部署 lock，且当前/目标 inventory 必须完全相同；涉及资源退休时须等待 独立的 UID/resourceVersion-precondition retirement ceremony，不能把遗留对象伪装成回滚成功。 +### Kubernetes 资源退役 + +缩小 active inventory 必须显式使用 `cluster.deployment.retirement.preflight`，不能先手工删除，也不能使用 `kubectl delete -f`、 +`kubectl apply --prune`、force delete 或 `--grace-period=0`。只允许退役 current locked manifest 与 committed Head inventory 中的 +namespaced 非 credential/data 对象；`Secret`、`PersistentVolumeClaim`、`ServiceAccount`、cluster-scoped resource 和固定 Head +ConfigMap 均须走专用运维流程。survivor inventory 必须仍包含 control、control-ai、admin、worker 四类 authority。 + +除上文相同的 locked manifest、lock report、kubectl、kubeconfig、context、cluster UID 与 expected Head 外，retirement command +还必须固定 realpath 后的 curl executable digest，并提交至多 64 个按 `apiVersion/kind/namespace/name` 排序的 target： + +```json +{ + "schemaVersion": 1, + "schema": "qinglong/kubernetes-deployment-command@v2", + "operation": "cluster.deployment.retirement.preflight", + "request": { + "preflightId": "", + "lockedManifest": "", + "lockReport": "", + "kubectl": "", + "curl": { + "path": "", + "expectedDigest": "" + }, + "kubeconfig": "", + "context": "", + "expectedClusterUid": "", + "expectedHead": "", + "targets": [ + { + "apiVersion": "v1", + "kind": "ConfigMap", + "namespace": "qinglong-system", + "name": "obsolete-release-resource" + } + ], + "output": "" + } +} +``` + +示例中的 `""` 与 `""` 是文档占位符，实际 command 必须展开为上文 +展示的 JSON object，且仍须是单行 canonical JSON。运行同一个 `pnpm cluster-deployment:ql3 -- --command-file=...` 入口。preflight +逐对象读取 exact UID/resourceVersion 与 Apply ownership，并向 API Server 发送带 Preconditions 的 `DeleteOptions` dry-run；成功报告 +保持 `kubernetesMutation:false`。人工核对 active/survivor inventory、target observation 和两个 tool digest 后，生成 +`cluster.deployment.retirement.apply` command：把 `preflightId` 改为新 `mutationId`，增加 +`preflight={path,expectedDigest}`，其余 authority 与 targets 必须逐字一致，output 改为新的 retirement receipt 路径。 + +apply 在删除前再次读取对象并 CAS Head 为 `applying`，随后通过 owner-private Unix socket proxy 发送带同一 UID/resourceVersion +preconditions 和 Background propagation 的 DELETE。只有 old UID 已确认 absent 才会提交缩小 inventory 的下一代 Head。若 DELETE +响应丢失，保留 command 与 preflight 原文件并原样重跑；只有相同 applying intent 且对象已经 absent 才可恢复。对象被新 UID 替换、 +resourceVersion 漂移、处于 terminating 或 Head 已被其他意图推进时必须人工处理，不能改 command 绕过围栏。 + +退役收据使用 `cluster.deployment.retirement.receipt.audit` 离线审计，结构与普通 audit 相同，但 applyCommand 必须指向 retirement +apply command。成功 audit 只证明 command/receipt digest 闭包，不重放外部删除结果。 + 离线审计使用 `cluster.deployment.receipt.audit` command，其中 `applyCommand.expectedDigest` 是 apply command 文件完整字节的 SHA-256，`receipt.expectedDigest` 是 receipt 内的 `receiptDigest`。审计不会访问 Kubernetes API，结果必须保持 `externalResultsReplayed:false`、`kubernetesMutation:false`。 diff --git a/scripts/lib/ql3-kubernetes-deployment-ceremony.cjs b/scripts/lib/ql3-kubernetes-deployment-ceremony.cjs index e0bb5d71..ea4faef3 100644 --- a/scripts/lib/ql3-kubernetes-deployment-ceremony.cjs +++ b/scripts/lib/ql3-kubernetes-deployment-ceremony.cjs @@ -4,7 +4,7 @@ const crypto = require('node:crypto'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); -const { spawnSync } = require('node:child_process'); +const { spawn, spawnSync } = require('node:child_process'); const yaml = require('js-yaml'); const semver = require('semver'); @@ -13,7 +13,11 @@ const LOCK_SCHEMA = 'qinglong/kubernetes-deployment-lock@v2'; const CATALOG_SCHEMA = 'qinglong/release-catalog-consumption-ceremony@v1'; const PREFLIGHT_SCHEMA = 'qinglong/kubernetes-deployment-preflight@v2'; const RECEIPT_SCHEMA = 'qinglong/kubernetes-deployment-receipt@v2'; -const HEAD_SCHEMA = 'qinglong/kubernetes-deployment-head@v1'; +const RETIREMENT_PREFLIGHT_SCHEMA = + 'qinglong/kubernetes-resource-retirement-preflight@v1'; +const RETIREMENT_RECEIPT_SCHEMA = + 'qinglong/kubernetes-resource-retirement-receipt@v1'; +const HEAD_SCHEMA = 'qinglong/kubernetes-deployment-head@v2'; const HEAD_NAME = 'qinglong3-deployment-head'; const HEAD_DATA_KEY = 'head.json'; const FIELD_MANAGER = 'qinglong3-catalog-lock'; @@ -24,6 +28,7 @@ const MAX_KUBECONFIG_BYTES = 1024 * 1024; const MAX_EXECUTABLE_BYTES = 256 * 1024 * 1024; const MAX_PROCESS_OUTPUT_BYTES = 4 * 1024 * 1024; const MAX_RESOURCE_COUNT = 2048; +const MAX_RETIREMENT_COUNT = 64; const MAX_STRUCTURE_DEPTH = 64; const PROCESS_TIMEOUT_MS = 60_000; const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u; @@ -36,6 +41,12 @@ const REPOSITORY_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?\/[a-z0-9._-]{1,100}$/u; const ROLE_ORDER = Object.freeze(['control', 'control-ai', 'admin', 'worker']); const TRANSITION_KINDS = Object.freeze(['install', 'upgrade', 'rollback']); +const HEAD_TRANSITION_KINDS = Object.freeze([...TRANSITION_KINDS, 'retire']); +const FORBIDDEN_RETIREMENT_IDENTITIES = Object.freeze([ + 'v1\u0000PersistentVolumeClaim', + 'v1\u0000Secret', + 'v1\u0000ServiceAccount', +]); const IMAGE_NAMES = Object.freeze({ control: 'qinglong3-cluster-control', 'control-ai': 'qinglong3-cluster-control-ai', @@ -320,6 +331,45 @@ function transitionRequest(request) { return Object.freeze({ transitionKind: request.transitionKind, head }); } +function retirementTargets(value) { + validateResourceInventory(value); + if (value.length > MAX_RETIREMENT_COUNT) { + fail('deployment retirement target count is invalid'); + } + for (const target of value) { + if ( + target.namespace === null || + FORBIDDEN_RETIREMENT_IDENTITIES.includes( + `${target.apiVersion}\u0000${target.kind}`, + ) || + (target.apiVersion === 'v1' && + target.kind === 'ConfigMap' && + target.name === HEAD_NAME) + ) { + fail('deployment retirement target requires a dedicated ceremony'); + } + } + return Object.freeze(value.map((target) => Object.freeze({ ...target }))); +} + +function retirementRequest(request) { + const head = expectedHead(request.expectedHead); + if (head.generation === 0) { + fail('deployment retirement requires a committed head'); + } + return Object.freeze({ + lockedManifest: artifact(request.lockedManifest, 'locked manifest'), + lockReport: artifact(request.lockReport, 'lock report'), + kubectl: artifact(request.kubectl, 'kubectl'), + curl: artifact(request.curl, 'curl'), + kubeconfig: artifact(request.kubeconfig, 'kubeconfig'), + context: request.context, + expectedClusterUid: request.expectedClusterUid, + expectedHead: head, + targets: retirementTargets(request.targets), + }); +} + function parseCommand(value) { if ( !exactKeys(value, ['schemaVersion', 'schema', 'operation', 'request']) || @@ -414,6 +464,97 @@ function parseCommand(value) { }), }); } + if (value.operation === 'cluster.deployment.retirement.preflight') { + if ( + !exactKeys(request, [ + 'preflightId', + 'lockedManifest', + 'lockReport', + 'kubectl', + 'curl', + 'kubeconfig', + 'context', + 'expectedClusterUid', + 'expectedHead', + 'targets', + 'output', + ]) || + !UUID_PATTERN.test(request.preflightId || '') || + !CONTEXT_PATTERN.test(request.context || '') || + !UUID_PATTERN.test(request.expectedClusterUid || '') || + typeof request.output !== 'string' + ) { + fail('retirement preflight command is invalid'); + } + return Object.freeze({ + schemaVersion: 1, + schema: COMMAND_SCHEMA, + operation: value.operation, + request: Object.freeze({ + preflightId: request.preflightId, + ...retirementRequest(request), + output: resolveCanonicalAbsolute( + request.output, + 'retirement preflight output', + ), + }), + }); + } + if (value.operation === 'cluster.deployment.retirement.apply') { + if ( + !exactKeys(request, [ + 'mutationId', + 'preflight', + 'lockedManifest', + 'lockReport', + 'kubectl', + 'curl', + 'kubeconfig', + 'context', + 'expectedClusterUid', + 'expectedHead', + 'targets', + 'output', + ]) || + !UUID_PATTERN.test(request.mutationId || '') || + !CONTEXT_PATTERN.test(request.context || '') || + !UUID_PATTERN.test(request.expectedClusterUid || '') || + typeof request.output !== 'string' + ) { + fail('retirement apply command is invalid'); + } + return Object.freeze({ + schemaVersion: 1, + schema: COMMAND_SCHEMA, + operation: value.operation, + request: Object.freeze({ + mutationId: request.mutationId, + preflight: artifact(request.preflight, 'retirement preflight report'), + ...retirementRequest(request), + output: resolveCanonicalAbsolute( + request.output, + 'retirement receipt output', + ), + }), + }); + } + if (value.operation === 'cluster.deployment.retirement.receipt.audit') { + if (!exactKeys(request, ['applyCommand', 'receipt'])) { + fail('retirement receipt audit command is invalid'); + } + return Object.freeze({ + schemaVersion: 1, + schema: COMMAND_SCHEMA, + operation: value.operation, + request: Object.freeze({ + applyCommand: artifact( + request.applyCommand, + 'retirement apply command', + ), + receipt: artifact(request.receipt, 'retirement receipt'), + }), + }); + } if (value.operation === 'cluster.deployment.receipt.audit') { if (!exactKeys(request, ['applyCommand', 'receipt'])) { fail('receipt audit command is invalid'); @@ -1119,6 +1260,19 @@ function inspectInputs(request) { }); } +function inspectRetirementInputs(request) { + const inputs = inspectInputs(request); + const curl = verifyExpected( + readStableFile(request.curl.path, 'curl', MAX_EXECUTABLE_BYTES, { + privateFile: false, + executable: true, + }), + request.curl.expectedDigest, + 'curl', + ); + return Object.freeze({ ...inputs, curl }); +} + function defaultRunProcess(executable, args, input) { const privateHome = fs.mkdtempSync( path.join(os.tmpdir(), 'ql3-kubectl-home-'), @@ -1146,6 +1300,198 @@ function defaultRunProcess(executable, args, input) { } } +function waitForProxySocket(socketPath, child) { + const deadline = Date.now() + 5_000; + const sleeper = new Int32Array(new SharedArrayBuffer(4)); + while (Date.now() < deadline) { + try { + if (fs.lstatSync(socketPath).isSocket()) return; + } catch {} + if (child.exitCode !== null) break; + Atomics.wait(sleeper, 0, 0, 25); + } + fail('private Kubernetes API proxy did not become ready'); +} + +function curlUnixRequest(inputs, socketPath, method, requestPath, body) { + const args = [ + '--silent', + '--show-error', + '--unix-socket', + socketPath, + '--request', + method, + '--header', + 'Accept: application/json', + '--max-time', + '60', + '--output', + '-', + '--write-out', + '\n%{http_code}', + ]; + let input = ''; + if (body !== null) { + args.push( + '--header', + 'Content-Type: application/json', + '--data-binary', + '@-', + ); + input = JSON.stringify(body); + } + args.push(`http://localhost${requestPath}`); + const result = spawnSync(inputs.curl.path, args, { + input, + encoding: 'utf8', + timeout: PROCESS_TIMEOUT_MS, + maxBuffer: MAX_PROCESS_OUTPUT_BYTES, + shell: false, + windowsHide: true, + }); + if ( + result.error || + result.status !== 0 || + typeof result.stdout !== 'string' + ) { + fail('private Kubernetes API request could not complete safely'); + } + const boundary = result.stdout.lastIndexOf('\n'); + const statusText = result.stdout.slice(boundary + 1); + const responseBody = result.stdout.slice(0, boundary); + if ( + boundary < 0 || + !/^\d{3}$/u.test(statusText) || + Buffer.byteLength(responseBody, 'utf8') > MAX_PROCESS_OUTPUT_BYTES + ) { + fail('private Kubernetes API response is invalid'); + } + return Object.freeze({ status: Number(statusText), body: responseBody }); +} + +function defaultRetirementApi(inputs, request, callback) { + const privateHome = fs.mkdtempSync( + path.join(os.tmpdir(), 'ql3-kubectl-retirement-'), + ); + fs.chmodSync(privateHome, 0o700); + const socketPath = path.join(privateHome, 'api.sock'); + const args = [ + `--kubeconfig=${inputs.kubeconfig.path}`, + `--context=${request.context}`, + '--request-timeout=60s', + 'proxy', + `--unix-socket=${socketPath}`, + '--api-prefix=/', + '--www=', + '--accept-paths=^/(api|apis)/.*$', + '--reject-methods=^(POST|PUT|PATCH|CONNECT|OPTIONS|TRACE)$', + '--keepalive=0s', + ]; + let child; + try { + child = spawn(inputs.executable.path, args, { + stdio: 'ignore', + shell: false, + windowsHide: true, + env: Object.freeze({ + HOME: privateHome, + XDG_CACHE_HOME: privateHome, + TMPDIR: privateHome, + LANG: 'C', + LC_ALL: 'C', + NO_COLOR: '1', + }), + }); + waitForProxySocket(socketPath, child); + return callback( + Object.freeze({ + request(method, requestPath, body = null) { + return curlUnixRequest(inputs, socketPath, method, requestPath, body); + }, + }), + ); + } finally { + if (child?.pid) child.kill('SIGTERM'); + try { + fs.rmSync(privateHome, { recursive: true, force: true }); + } catch {} + } +} + +function withRetirementApi(inputs, request, dependencies, callback) { + if (dependencies.retirementApi !== undefined) { + if (typeof dependencies.retirementApi?.request !== 'function') { + fail('retirement API dependency is invalid'); + } + return callback(dependencies.retirementApi); + } + return defaultRetirementApi(inputs, request, callback); +} + +function recordedApiRequest(records, api, name, method, requestPath, body) { + const response = api.request(method, requestPath, body); + if ( + !Number.isInteger(response?.status) || + response.status < 100 || + response.status > 599 || + typeof response.body !== 'string' || + Buffer.byteLength(response.body, 'utf8') > MAX_PROCESS_OUTPUT_BYTES + ) { + fail('retirement API dependency response is invalid'); + } + records.push( + Object.freeze({ + name, + method, + requestPathDigest: sha256(Buffer.from(requestPath, 'utf8')), + requestDigest: sha256( + Buffer.from(body === null ? '' : JSON.stringify(body), 'utf8'), + ), + status: response.status, + responseDigest: sha256(Buffer.from(response.body, 'utf8')), + responseBytes: Buffer.byteLength(response.body, 'utf8'), + }), + ); + return response; +} + +function aggregateRequestStep(name, records) { + const stdoutBytes = records.reduce( + (total, record) => total + record.responseBytes, + 0, + ); + if (records.length < 1 || stdoutBytes > MAX_PROCESS_OUTPUT_BYTES) { + fail('retirement API transcript exceeds its bound'); + } + return Object.freeze({ + name, + status: 0, + argvDigest: sha256( + Buffer.from( + JSON.stringify( + records.map((record) => ({ + name: record.name, + method: record.method, + requestPathDigest: record.requestPathDigest, + requestDigest: record.requestDigest, + status: record.status, + })), + ), + 'utf8', + ), + ), + stdoutDigest: sha256( + Buffer.from( + JSON.stringify(records.map((record) => record.responseDigest)), + 'utf8', + ), + ), + stdoutBytes, + stderrDigest: sha256(Buffer.alloc(0)), + stderrBytes: 0, + }); +} + function runStep( inputs, request, @@ -1263,6 +1609,23 @@ function revalidateStableInputs(inputs, request) { } } +function revalidateRetirementInputs(inputs, request) { + revalidateStableInputs(inputs, request); + const current = readStableFile( + inputs.curl.path, + 'curl', + MAX_EXECUTABLE_BYTES, + { privateFile: false, executable: true }, + ); + if ( + current.digest !== inputs.curl.digest || + !sameIdentity(current.identity, inputs.curl.identity) || + current.digest !== request.curl.expectedDigest + ) { + fail('curl changed during the retirement ceremony'); + } +} + function lockSummary(inputs) { const report = inputs.authority.report; return Object.freeze({ @@ -1294,6 +1657,241 @@ function toolSummary(inputs) { }); } +function retirementToolSummary(inputs) { + return Object.freeze({ + kubectl: toolSummary(inputs), + curl: Object.freeze({ + name: 'curl', + executableDigest: inputs.curl.digest, + }), + transport: 'owner_private_unix_socket_proxy', + }); +} + +function validateRetirementToolSummary(value) { + if ( + !exactKeys(value, ['kubectl', 'curl', 'transport']) || + value.transport !== 'owner_private_unix_socket_proxy' || + !exactKeys(value.curl, ['name', 'executableDigest']) || + value.curl.name !== 'curl' || + !DIGEST_PATTERN.test(value.curl.executableDigest || '') + ) { + fail('retirement tool summary is invalid'); + } + validateToolSummary(value.kubectl); +} + +function inventoryKey(value) { + return JSON.stringify({ + apiVersion: value.apiVersion, + kind: value.kind, + namespace: value.namespace ?? value.metadata?.namespace ?? null, + name: value.name ?? value.metadata?.name, + }); +} + +function retirementPlan(inputs, current, targets) { + if (current.state === null || current.state.phase !== 'committed') { + fail('deployment retirement requires a committed head'); + } + const currentDeployment = current.state.deployment; + if ( + JSON.stringify(currentDeployment.lock) !== + JSON.stringify(lockSummary(inputs)) + ) { + fail('deployment retirement lock is not the active lock'); + } + const manifestByIdentity = new Map( + inputs.resources.map((resource) => [inventoryKey(resource), resource]), + ); + const activeKeys = new Set( + currentDeployment.resources.map((resource) => inventoryKey(resource)), + ); + if ( + activeKeys.size !== currentDeployment.resources.length || + [...activeKeys].some((identity) => !manifestByIdentity.has(identity)) + ) { + fail('active deployment inventory is not backed by its lock'); + } + const retiredKeys = new Set(targets.map((target) => inventoryKey(target))); + if ([...retiredKeys].some((identity) => !activeKeys.has(identity))) { + fail('deployment retirement target is not active'); + } + const survivors = Object.freeze( + currentDeployment.resources.filter( + (resource) => !retiredKeys.has(inventoryKey(resource)), + ), + ); + if (survivors.length < 1) { + fail('deployment retirement cannot empty the active inventory'); + } + const survivorResources = survivors.map((item) => + manifestByIdentity.get(inventoryKey(item)), + ); + for (const role of ROLE_ORDER) { + if ( + !survivorResources.some((resource) => + containsExactString(resource, inputs.authority.references[role]), + ) + ) { + fail(`deployment retirement would remove the last ${role} authority`); + } + } + return Object.freeze({ + active: currentDeployment.resources, + survivors, + resources: Object.freeze( + targets.map((target) => manifestByIdentity.get(inventoryKey(target))), + ), + }); +} + +function apiVersionBase(apiVersion) { + if (apiVersion === 'v1') return '/api/v1'; + const segments = apiVersion.split('/'); + if ( + segments.length !== 2 || + segments.some( + (segment) => !segment || /[\u0000-\u001f\u007f]/u.test(segment), + ) + ) { + fail('retirement API version is invalid'); + } + return `/apis/${encodeURIComponent(segments[0])}/${encodeURIComponent( + segments[1], + )}`; +} + +function apiResourcePath(api, item, cache, records) { + const base = apiVersionBase(item.apiVersion); + let resources = cache.get(base); + if (resources === undefined) { + const response = recordedApiRequest( + records, + api, + 'api_discovery', + 'GET', + base, + null, + ); + if (response.status !== 200) fail('Kubernetes API discovery failed'); + let discovery; + try { + discovery = JSON.parse(response.body); + } catch { + fail('Kubernetes API discovery response is invalid'); + } + if (!Array.isArray(discovery?.resources)) { + fail('Kubernetes API discovery response is invalid'); + } + resources = discovery.resources; + cache.set(base, resources); + } + const candidates = resources.filter( + (resource) => + resource?.kind === item.kind && + typeof resource?.name === 'string' && + !resource.name.includes('/') && + Array.isArray(resource.verbs) && + resource.verbs.includes('get') && + resource.verbs.includes('delete') && + resource.namespaced === (item.namespace !== null), + ); + if ( + candidates.length !== 1 || + !/^[a-z0-9](?:[a-z0-9.-]{0,252})$/u.test(candidates[0].name) + ) { + fail('retirement API resource mapping is ambiguous'); + } + const collection = + item.namespace === null + ? `${base}/${candidates[0].name}` + : `${base}/namespaces/${encodeURIComponent(item.namespace)}/${ + candidates[0].name + }`; + return `${collection}/${encodeURIComponent(item.name)}`; +} + +function parseRetirementObject(response, desired, expected = null) { + let live; + try { + live = JSON.parse(response.body); + } catch { + fail('retirement target response is invalid'); + } + if ( + response.status !== 200 || + inventoryKey(live) !== inventoryKey(desired) || + !UUID_PATTERN.test(live.metadata?.uid || '') || + typeof live.metadata?.resourceVersion !== 'string' || + !live.metadata.resourceVersion || + live.metadata.deletionTimestamp !== undefined || + !Array.isArray(live.metadata.managedFields) || + !live.metadata.managedFields.some( + (entry) => + entry?.manager === FIELD_MANAGER && entry?.operation === 'Apply', + ) + ) { + fail('retirement target authority is invalid'); + } + assertDesiredSubset(desired, live); + const observation = Object.freeze({ + apiVersion: desired.apiVersion, + kind: desired.kind, + namespace: desired.metadata.namespace, + name: desired.metadata.name, + uid: live.metadata.uid, + resourceVersion: live.metadata.resourceVersion, + }); + if ( + expected !== null && + JSON.stringify(observation) !== JSON.stringify(expected) + ) { + fail('retirement target changed after preflight'); + } + return observation; +} + +function readRetirementTargets(api, resources, records) { + const cache = new Map(); + return Object.freeze( + resources.map((desired) => { + const item = { + apiVersion: desired.apiVersion, + kind: desired.kind, + namespace: desired.metadata?.namespace ?? null, + name: desired.metadata?.name, + }; + const requestPath = apiResourcePath(api, item, cache, records); + const response = recordedApiRequest( + records, + api, + 'retirement_target_get', + 'GET', + requestPath, + null, + ); + return Object.freeze({ + observation: parseRetirementObject(response, desired), + requestPath, + }); + }), + ); +} + +function deleteOptions(observation, dryRun) { + return Object.freeze({ + apiVersion: 'v1', + kind: 'DeleteOptions', + ...(dryRun ? { dryRun: Object.freeze(['All']) } : {}), + propagationPolicy: 'Background', + preconditions: Object.freeze({ + uid: observation.uid, + resourceVersion: observation.resourceVersion, + }), + }); +} + function validateResourceInventory(value) { if ( !Array.isArray(value) || @@ -1327,6 +1925,50 @@ function validateResourceInventory(value) { } } +function validateRetirementObservations(value) { + if ( + !Array.isArray(value) || + value.length < 1 || + value.length > MAX_RETIREMENT_COUNT + ) { + fail('deployment retirement observations are invalid'); + } + const identities = []; + for (const item of value) { + if ( + !exactKeys(item, [ + 'apiVersion', + 'kind', + 'namespace', + 'name', + 'uid', + 'resourceVersion', + ]) || + item.namespace === null || + !UUID_PATTERN.test(item.uid || '') || + typeof item.resourceVersion !== 'string' || + !item.resourceVersion + ) { + fail('deployment retirement observation is invalid'); + } + retirementTargets([ + { + apiVersion: item.apiVersion, + kind: item.kind, + namespace: item.namespace, + name: item.name, + }, + ]); + identities.push(inventoryKey(item)); + } + if ( + new Set(identities).size !== identities.length || + JSON.stringify([...identities].sort()) !== JSON.stringify(identities) + ) { + fail('deployment retirement observation ordering is invalid'); + } +} + function deploymentPlan(inputs) { return Object.freeze({ lock: lockSummary(inputs), @@ -1440,20 +2082,25 @@ function validateTransition(current, request, planned) { } return; } + const previous = current.state.previous; + const versionRollback = semver.lt(plannedVersion, activeVersion); + const retirementRestore = + current.state.transition.kind === 'retire' && + plannedVersion === activeVersion && + previous !== null && + previous.lock.release.version === activeVersion; if ( - current.state.previous === null || - !semver.lt(plannedVersion, activeVersion) || - JSON.stringify(planned.lock) !== - JSON.stringify(current.state.previous.lock) || - JSON.stringify(planned.resources) !== - JSON.stringify(current.state.previous.resources) || - JSON.stringify(planned.resources) !== JSON.stringify(active.resources) + previous === null || + (!versionRollback && !retirementRestore) || + JSON.stringify(planned.lock) !== JSON.stringify(previous.lock) || + JSON.stringify(planned.resources) !== JSON.stringify(previous.resources) ) { fail('deployment rollback must restore the exact previous resource set'); } } function validateHeadTransition(value) { + const retirement = value?.kind === 'retire'; if ( !exactKeys(value, [ 'kind', @@ -1461,15 +2108,18 @@ function validateHeadTransition(value) { 'commandDigest', 'preflightDigest', 'expectedHead', + ...(retirement ? ['retirements', 'curlExecutableDigest'] : []), ]) || - !TRANSITION_KINDS.includes(value.kind) || + !HEAD_TRANSITION_KINDS.includes(value.kind) || !UUID_PATTERN.test(value.mutationId || '') || !DIGEST_PATTERN.test(value.commandDigest || '') || - !DIGEST_PATTERN.test(value.preflightDigest || '') + !DIGEST_PATTERN.test(value.preflightDigest || '') || + (retirement && !DIGEST_PATTERN.test(value.curlExecutableDigest || '')) ) { fail('deployment head transition is invalid'); } validateHeadSnapshot(value.expectedHead); + if (retirement) validateRetirementObservations(value.retirements); } function deploymentEvidence(state) { @@ -1526,16 +2176,25 @@ function validateHeadState(value) { fail('applying deployment head is invalid'); } } else { + const expectedStepNames = + value.transition.kind === 'retire' + ? [ + 'cluster_identity_before', + 'resource_precondition_delete', + 'retirement_absence_read', + 'cluster_identity_after', + ] + : [ + 'cluster_identity_before', + 'server_side_dry_run', + 'server_side_apply', + 'server_side_convergence_read', + 'cluster_identity_after', + ]; if ( - value.steps.length !== 5 || + value.steps.length !== expectedStepNames.length || JSON.stringify(value.steps.map((step) => step.name)) !== - JSON.stringify([ - 'cluster_identity_before', - 'server_side_dry_run', - 'server_side_apply', - 'server_side_convergence_read', - 'cluster_identity_after', - ]) || + JSON.stringify(expectedStepNames) || value.deployment === null || JSON.stringify(value.deployment.lock) !== JSON.stringify(value.planned.lock) || @@ -1951,7 +2610,7 @@ function readAndValidatePreflight(request, inputs) { } function receiptFromCommitted(state, namespace) { - if (state.phase !== 'committed') { + if (state.phase !== 'committed' || state.transition.kind === 'retire') { fail('deployment receipt requires a committed head'); } const unsigned = { @@ -2076,6 +2735,705 @@ function validateReceipt(value, expectedCommandDigest) { return value; } +function retirementDeleteDryRun(api, targets) { + const dryRunRecords = []; + for (const target of targets) { + const response = recordedApiRequest( + dryRunRecords, + api, + 'retirement_delete_dry_run', + 'DELETE', + target.requestPath, + deleteOptions(target.observation, true), + ); + if (![200, 201, 202].includes(response.status)) { + fail('Kubernetes retirement dry-run was rejected'); + } + } + return aggregateRequestStep('server_side_delete_dry_run', dryRunRecords); +} + +function retirementRecheck(api, resources, targets) { + const records = []; + const cache = new Map(); + for (let index = 0; index < resources.length; index += 1) { + const desired = resources[index]; + const observation = targets[index].observation; + const requestPath = apiResourcePath(api, observation, cache, records); + const response = recordedApiRequest( + records, + api, + 'retirement_target_recheck', + 'GET', + requestPath, + null, + ); + parseRetirementObject(response, desired, observation); + } + return aggregateRequestStep('retirement_target_recheck', records); +} + +function validateRetirementPreflight(value) { + if ( + !exactKeys(value, [ + 'schemaVersion', + 'schema', + 'preflightId', + 'commandDigest', + 'lock', + 'activeResourceInventory', + 'survivorResourceInventory', + 'retirementTargets', + 'deploymentHead', + 'target', + 'tools', + 'steps', + 'verification', + 'preflightDigest', + ]) || + value.schemaVersion !== 1 || + value.schema !== RETIREMENT_PREFLIGHT_SCHEMA || + !UUID_PATTERN.test(value.preflightId || '') || + !DIGEST_PATTERN.test(value.commandDigest || '') || + !DIGEST_PATTERN.test(value.preflightDigest || '') || + !Array.isArray(value.steps) || + JSON.stringify(value.steps.map((step) => step.name)) !== + JSON.stringify([ + 'cluster_identity_before', + 'deployment_head_read', + 'retirement_target_read', + 'server_side_delete_dry_run', + 'retirement_target_recheck', + ]) || + !exactKeys(value.verification, [ + 'catalogBoundLock', + 'clusterIdentityBound', + 'deploymentHeadCompared', + 'uidResourceVersionObserved', + 'serverSideDeleteDryRun', + 'survivorRoleClosure', + 'resourceInventoryClosed', + 'networkAccess', + 'kubernetesMutation', + 'unixSocketProxy', + 'ambientHome', + 'ephemeralCache', + ]) || + JSON.stringify(value.verification) !== + JSON.stringify({ + catalogBoundLock: true, + clusterIdentityBound: true, + deploymentHeadCompared: true, + uidResourceVersionObserved: true, + serverSideDeleteDryRun: true, + survivorRoleClosure: true, + resourceInventoryClosed: true, + networkAccess: true, + kubernetesMutation: false, + unixSocketProxy: true, + ambientHome: false, + ephemeralCache: true, + }) + ) { + fail('retirement preflight report is invalid'); + } + validateLockSummary(value.lock); + validateResourceInventory(value.activeResourceInventory); + validateResourceInventory(value.survivorResourceInventory); + validateRetirementObservations(value.retirementTargets); + validateHeadSnapshot(value.deploymentHead); + validateTargetSummary(value.target); + validateRetirementToolSummary(value.tools); + for (const step of value.steps) validateStepRecord(step); + if (value.steps.some((step) => step.status !== 0)) { + fail('retirement preflight contains a failed step'); + } + const retired = new Set( + value.retirementTargets.map((target) => inventoryKey(target)), + ); + if ( + JSON.stringify( + value.activeResourceInventory.filter( + (resource) => !retired.has(inventoryKey(resource)), + ), + ) !== JSON.stringify(value.survivorResourceInventory) + ) { + fail('retirement preflight inventory closure is invalid'); + } + const { preflightDigest, ...unsigned } = value; + if ( + preflightDigest !== sha256(Buffer.from(JSON.stringify(unsigned), 'utf8')) + ) { + fail('retirement preflight self digest is invalid'); + } + return value; +} + +function createRetirementPreflight(commandState, dependencies = {}) { + const request = commandState.command.request; + const output = outputTarget(request.output, 'retirement preflight output'); + const inputs = inspectRetirementInputs(request); + const runner = dependencies.runProcess ?? defaultRunProcess; + const identity = identityStep( + inputs, + request, + runner, + 'cluster_identity_before', + ); + const current = headReadStep(inputs, request, runner); + if ( + current.state === null || + current.state.phase !== 'committed' || + JSON.stringify(current.snapshot) !== + JSON.stringify(expectedSnapshot(request, inputs.headNamespace)) + ) { + fail('deployment retirement head authority changed'); + } + const plan = retirementPlan(inputs, current, request.targets); + const apiEvidence = withRetirementApi( + inputs, + request, + dependencies, + (api) => { + const readRecords = []; + const targets = readRetirementTargets(api, plan.resources, readRecords); + return Object.freeze({ + targets, + read: aggregateRequestStep('retirement_target_read', readRecords), + dryRun: retirementDeleteDryRun(api, targets), + recheck: retirementRecheck(api, plan.resources, targets), + }); + }, + ); + revalidateRetirementInputs(inputs, request); + const unsigned = { + schemaVersion: 1, + schema: RETIREMENT_PREFLIGHT_SCHEMA, + preflightId: request.preflightId, + commandDigest: commandState.commandDigest, + lock: lockSummary(inputs), + activeResourceInventory: plan.active, + survivorResourceInventory: plan.survivors, + retirementTargets: Object.freeze( + apiEvidence.targets.map((target) => target.observation), + ), + deploymentHead: current.snapshot, + target: targetSummary(inputs, request), + tools: retirementToolSummary(inputs), + steps: Object.freeze([ + identity, + current.record, + apiEvidence.read, + apiEvidence.dryRun, + apiEvidence.recheck, + ]), + verification: { + catalogBoundLock: true, + clusterIdentityBound: true, + deploymentHeadCompared: true, + uidResourceVersionObserved: true, + serverSideDeleteDryRun: true, + survivorRoleClosure: true, + resourceInventoryClosed: true, + networkAccess: true, + kubernetesMutation: false, + unixSocketProxy: true, + ambientHome: false, + ephemeralCache: true, + }, + }; + const result = Object.freeze({ + ...unsigned, + preflightDigest: sha256(Buffer.from(JSON.stringify(unsigned), 'utf8')), + }); + writeNoReplace(output, result); + return result; +} + +function readAndValidateRetirementPreflight(request, inputs) { + const file = readCanonicalJson( + request.preflight.path, + 'retirement preflight report', + MAX_REPORT_BYTES, + ); + const report = validateRetirementPreflight(file.value); + if ( + report.preflightDigest !== request.preflight.expectedDigest || + JSON.stringify(report.lock) !== JSON.stringify(lockSummary(inputs)) || + JSON.stringify(report.deploymentHead) !== + JSON.stringify(expectedSnapshot(request, inputs.headNamespace)) || + JSON.stringify( + report.retirementTargets.map( + ({ uid: _uid, resourceVersion: _resourceVersion, ...item }) => item, + ), + ) !== JSON.stringify(request.targets) || + JSON.stringify(report.target) !== + JSON.stringify(targetSummary(inputs, request)) || + JSON.stringify(report.tools) !== + JSON.stringify(retirementToolSummary(inputs)) + ) { + fail('retirement preflight binding changed'); + } + return Object.freeze({ file, report }); +} + +function retirementApplyingHead(commandState, preflight, inputs, current) { + const unsigned = { + schemaVersion: 1, + schema: HEAD_SCHEMA, + phase: 'applying', + generation: current.state.generation + 1, + transition: { + kind: 'retire', + mutationId: commandState.command.request.mutationId, + commandDigest: commandState.commandDigest, + preflightDigest: preflight.report.preflightDigest, + expectedHead: current.snapshot, + retirements: preflight.report.retirementTargets, + curlExecutableDigest: inputs.curl.digest, + }, + previous: Object.freeze({ ...current.state.deployment }), + planned: Object.freeze({ + lock: current.state.deployment.lock, + resources: preflight.report.survivorResourceInventory, + }), + target: targetSummary(inputs, commandState.command.request), + tool: toolSummary(inputs), + steps: [], + deployment: null, + }; + return Object.freeze({ + ...unsigned, + stateDigest: sha256(Buffer.from(JSON.stringify(unsigned), 'utf8')), + }); +} + +function assertRetirementBinding(state, commandState, preflight, inputs) { + const request = commandState.command.request; + if ( + state.transition.kind !== 'retire' || + state.transition.mutationId !== request.mutationId || + state.transition.commandDigest !== commandState.commandDigest || + state.transition.preflightDigest !== preflight.report.preflightDigest || + state.transition.curlExecutableDigest !== inputs.curl.digest || + JSON.stringify(state.transition.expectedHead) !== + JSON.stringify(preflight.report.deploymentHead) || + JSON.stringify(state.transition.retirements) !== + JSON.stringify(preflight.report.retirementTargets) || + JSON.stringify(state.planned) !== + JSON.stringify({ + lock: preflight.report.lock, + resources: preflight.report.survivorResourceInventory, + }) || + JSON.stringify(state.target) !== + JSON.stringify(targetSummary(inputs, request)) || + JSON.stringify(state.tool) !== JSON.stringify(toolSummary(inputs)) + ) { + fail('retirement head transition binding changed'); + } +} + +function retirementObjectState(response, desired, expected) { + if (response.status === 404) return Object.freeze({ kind: 'absent' }); + if (response.status !== 200) { + fail('retirement target inspection failed'); + } + let live; + try { + live = JSON.parse(response.body); + } catch { + fail('retirement target response is invalid'); + } + if ( + inventoryKey(live) !== inventoryKey(expected) || + typeof live.metadata?.uid !== 'string' || + typeof live.metadata?.resourceVersion !== 'string' + ) { + fail('retirement target identity changed'); + } + if (live.metadata.uid !== expected.uid) { + fail('retirement target was replaced by a different UID'); + } + assertDesiredSubset(desired, live); + if (live.metadata.deletionTimestamp !== undefined) { + return Object.freeze({ kind: 'terminating' }); + } + if (live.metadata.resourceVersion !== expected.resourceVersion) { + fail('retirement target resourceVersion changed'); + } + return Object.freeze({ kind: 'current' }); +} + +function recoveredDeleteRecord(observation, requestPath) { + const body = deleteOptions(observation, false); + return Object.freeze({ + name: 'retirement_delete_recovered', + method: 'DELETE', + requestPathDigest: sha256(Buffer.from(requestPath, 'utf8')), + requestDigest: sha256(Buffer.from(JSON.stringify(body), 'utf8')), + status: 404, + responseDigest: sha256(Buffer.from('recovered_absent', 'utf8')), + responseBytes: 0, + }); +} + +function executeRetirementDeletes(api, inputs, preflight, resumed) { + const desiredByIdentity = new Map( + inputs.resources.map((resource) => [inventoryKey(resource), resource]), + ); + const cache = new Map(); + const deleteRecords = []; + const absenceRecords = []; + for (const observation of preflight.retirementTargets) { + const desired = desiredByIdentity.get(inventoryKey(observation)); + if (desired === undefined) { + fail('retirement target is no longer in the locked manifest'); + } + const requestPath = apiResourcePath(api, observation, cache, deleteRecords); + const before = recordedApiRequest( + deleteRecords, + api, + 'retirement_target_before_delete', + 'GET', + requestPath, + null, + ); + const beforeState = retirementObjectState(before, desired, observation); + if (beforeState.kind === 'absent') { + if (!resumed) fail('retirement target disappeared before deletion'); + deleteRecords.push(recoveredDeleteRecord(observation, requestPath)); + } else if (beforeState.kind === 'terminating') { + fail('retirement target deletion is not complete'); + } else { + const response = recordedApiRequest( + deleteRecords, + api, + 'retirement_target_delete', + 'DELETE', + requestPath, + deleteOptions(observation, false), + ); + if (![200, 201, 202].includes(response.status)) { + fail('Kubernetes retirement delete was rejected'); + } + } + const after = recordedApiRequest( + absenceRecords, + api, + 'retirement_target_absence', + 'GET', + requestPath, + null, + ); + const afterState = retirementObjectState(after, desired, observation); + if (afterState.kind === 'terminating') { + fail('retirement target deletion is not complete'); + } + if (afterState.kind !== 'absent') { + fail('retirement target remains after deletion'); + } + } + return Object.freeze({ + deletion: aggregateRequestStep( + 'resource_precondition_delete', + deleteRecords, + ), + absence: aggregateRequestStep('retirement_absence_read', absenceRecords), + }); +} + +function retirementReceiptFromCommitted(state, namespace, inputs) { + if (state.phase !== 'committed' || state.transition.kind !== 'retire') { + fail('retirement receipt requires a committed retirement head'); + } + const unsigned = { + schemaVersion: 1, + schema: RETIREMENT_RECEIPT_SCHEMA, + mutationId: state.transition.mutationId, + commandDigest: state.transition.commandDigest, + preflightDigest: state.transition.preflightDigest, + lock: state.deployment.lock, + retiredResources: state.transition.retirements, + resourceInventory: state.deployment.resources, + deploymentHead: headSnapshot(namespace, state), + target: state.target, + tools: retirementToolSummary(inputs), + steps: state.steps, + verification: { + catalogBoundLock: true, + clusterIdentityBound: true, + uidResourceVersionDeletePreconditions: true, + deploymentHeadCas: true, + resourceInventoryClosed: true, + survivorRoleClosure: true, + deletionAbsenceConfirmed: true, + networkAccess: true, + kubernetesMutation: true, + crossResourceAtomicity: false, + externalResultsReplayed: false, + recovery: 'resume_exact_retirement_from_target_head', + unixSocketProxy: true, + ambientHome: false, + ephemeralCache: true, + }, + }; + return Object.freeze({ + ...unsigned, + receiptDigest: sha256(Buffer.from(JSON.stringify(unsigned), 'utf8')), + }); +} + +function validateRetirementReceipt(value, expectedCommandDigest) { + if ( + !exactKeys(value, [ + 'schemaVersion', + 'schema', + 'mutationId', + 'commandDigest', + 'preflightDigest', + 'lock', + 'retiredResources', + 'resourceInventory', + 'deploymentHead', + 'target', + 'tools', + 'steps', + 'verification', + 'receiptDigest', + ]) || + value.schemaVersion !== 1 || + value.schema !== RETIREMENT_RECEIPT_SCHEMA || + !UUID_PATTERN.test(value.mutationId || '') || + value.commandDigest !== expectedCommandDigest || + !DIGEST_PATTERN.test(value.preflightDigest || '') || + !DIGEST_PATTERN.test(value.receiptDigest || '') || + !Array.isArray(value.steps) || + JSON.stringify(value.steps.map((step) => step.name)) !== + JSON.stringify([ + 'cluster_identity_before', + 'resource_precondition_delete', + 'retirement_absence_read', + 'cluster_identity_after', + ]) || + !exactKeys(value.verification, [ + 'catalogBoundLock', + 'clusterIdentityBound', + 'uidResourceVersionDeletePreconditions', + 'deploymentHeadCas', + 'resourceInventoryClosed', + 'survivorRoleClosure', + 'deletionAbsenceConfirmed', + 'networkAccess', + 'kubernetesMutation', + 'crossResourceAtomicity', + 'externalResultsReplayed', + 'recovery', + 'unixSocketProxy', + 'ambientHome', + 'ephemeralCache', + ]) || + JSON.stringify(value.verification) !== + JSON.stringify({ + catalogBoundLock: true, + clusterIdentityBound: true, + uidResourceVersionDeletePreconditions: true, + deploymentHeadCas: true, + resourceInventoryClosed: true, + survivorRoleClosure: true, + deletionAbsenceConfirmed: true, + networkAccess: true, + kubernetesMutation: true, + crossResourceAtomicity: false, + externalResultsReplayed: false, + recovery: 'resume_exact_retirement_from_target_head', + unixSocketProxy: true, + ambientHome: false, + ephemeralCache: true, + }) + ) { + fail('retirement receipt is invalid'); + } + validateLockSummary(value.lock); + validateRetirementObservations(value.retiredResources); + validateResourceInventory(value.resourceInventory); + validateHeadSnapshot(value.deploymentHead); + validateTargetSummary(value.target); + validateRetirementToolSummary(value.tools); + for (const step of value.steps) validateStepRecord(step); + if (value.steps.some((step) => step.status !== 0)) { + fail('retirement receipt contains a failed step'); + } + const { receiptDigest, ...unsigned } = value; + if (receiptDigest !== sha256(Buffer.from(JSON.stringify(unsigned), 'utf8'))) { + fail('retirement receipt self digest is invalid'); + } + return value; +} + +function existingRetirementReceipt(request, commandDigest) { + if (!fs.existsSync(request.output)) return null; + const file = readCanonicalJson( + request.output, + 'existing retirement receipt', + MAX_REPORT_BYTES, + ); + const receipt = validateRetirementReceipt(file.value, commandDigest); + if ( + receipt.mutationId !== request.mutationId || + receipt.preflightDigest !== request.preflight.expectedDigest + ) { + fail('existing retirement receipt binding changed'); + } + return receipt; +} + +function applyRetirement(commandState, dependencies = {}) { + const request = commandState.command.request; + const resumedReceipt = existingRetirementReceipt( + request, + commandState.commandDigest, + ); + const output = + resumedReceipt === null + ? outputTarget(request.output, 'retirement receipt output') + : request.output; + const inputs = inspectRetirementInputs(request); + const preflight = readAndValidateRetirementPreflight(request, inputs); + const runner = dependencies.runProcess ?? defaultRunProcess; + const current = headReadStep(inputs, request, runner); + + if (resumedReceipt !== null) { + if ( + current.state === null || + current.state.phase !== 'committed' || + resumedReceipt.preflightDigest !== preflight.report.preflightDigest || + JSON.stringify(resumedReceipt.lock) !== + JSON.stringify(preflight.report.lock) || + JSON.stringify(resumedReceipt.resourceInventory) !== + JSON.stringify(preflight.report.survivorResourceInventory) || + JSON.stringify(resumedReceipt.deploymentHead) !== + JSON.stringify(headSnapshot(inputs.headNamespace, current.state)) || + JSON.stringify(resumedReceipt.target) !== + JSON.stringify(targetSummary(inputs, request)) || + JSON.stringify(resumedReceipt.tools) !== + JSON.stringify(retirementToolSummary(inputs)) + ) { + fail('existing retirement receipt authority changed'); + } + assertRetirementBinding(current.state, commandState, preflight, inputs); + revalidateRetirementInputs(inputs, request); + return resumedReceipt; + } + + if ( + current.state !== null && + current.state.phase === 'committed' && + current.state.transition.kind === 'retire' && + current.state.transition.commandDigest === commandState.commandDigest + ) { + assertRetirementBinding(current.state, commandState, preflight, inputs); + const recovered = retirementReceiptFromCommitted( + current.state, + inputs.headNamespace, + inputs, + ); + writeNoReplace(output, recovered); + revalidateRetirementInputs(inputs, request); + return recovered; + } + + let applying; + let headResourceVersion; + let resumedHead = false; + if (current.state !== null && current.state.phase === 'applying') { + assertRetirementBinding(current.state, commandState, preflight, inputs); + applying = current.state; + headResourceVersion = current.resourceVersion; + resumedHead = true; + } else { + if ( + JSON.stringify(current.snapshot) !== + JSON.stringify(preflight.report.deploymentHead) + ) { + fail('deployment retirement head changed after preflight'); + } + const plan = retirementPlan(inputs, current, request.targets); + if ( + JSON.stringify(plan.active) !== + JSON.stringify(preflight.report.activeResourceInventory) || + JSON.stringify(plan.survivors) !== + JSON.stringify(preflight.report.survivorResourceInventory) + ) { + fail('deployment retirement inventory changed after preflight'); + } + withRetirementApi(inputs, request, dependencies, (api) => { + const records = []; + const observations = readRetirementTargets( + api, + plan.resources, + records, + ).map((target) => target.observation); + if ( + JSON.stringify(observations) !== + JSON.stringify(preflight.report.retirementTargets) + ) { + fail('deployment retirement target changed after preflight'); + } + }); + applying = retirementApplyingHead(commandState, preflight, inputs, current); + const acquired = writeHead( + inputs, + request, + runner, + applying, + current.resourceVersion, + 'deployment_head_acquire', + ); + headResourceVersion = acquired.resourceVersion; + } + + const beforeIdentity = identityStep( + inputs, + request, + runner, + 'cluster_identity_before', + ); + const deletion = withRetirementApi(inputs, request, dependencies, (api) => + executeRetirementDeletes(api, inputs, preflight.report, resumedHead), + ); + const afterIdentity = identityStep( + inputs, + request, + runner, + 'cluster_identity_after', + ); + revalidateRetirementInputs(inputs, request); + const committed = committedHead( + applying, + Object.freeze([ + beforeIdentity, + deletion.deletion, + deletion.absence, + afterIdentity, + ]), + ); + writeHead( + inputs, + request, + runner, + committed, + headResourceVersion, + 'deployment_head_commit', + ); + const receipt = retirementReceiptFromCommitted( + committed, + inputs.headNamespace, + inputs, + ); + writeNoReplace(output, receipt); + return receipt; +} + function existingReceipt(request, commandDigest) { if (!fs.existsSync(request.output)) return null; const file = readCanonicalJson( @@ -2264,6 +3622,57 @@ function auditReceipt(command) { }); } +function auditRetirementReceipt(command) { + const applyFile = verifyExpected( + readCanonicalJson( + command.request.applyCommand.path, + 'retirement apply command', + MAX_COMMAND_BYTES, + ), + command.request.applyCommand.expectedDigest, + 'retirement apply command', + ); + const applyCommand = parseCommand(applyFile.value); + if (applyCommand.operation !== 'cluster.deployment.retirement.apply') { + fail('retirement receipt audit requires a retirement apply command'); + } + const receiptFile = readCanonicalJson( + command.request.receipt.path, + 'retirement receipt', + MAX_REPORT_BYTES, + ); + const receipt = validateRetirementReceipt( + receiptFile.value, + applyFile.digest, + ); + if ( + receipt.receiptDigest !== command.request.receipt.expectedDigest || + receipt.mutationId !== applyCommand.request.mutationId || + receipt.preflightDigest !== applyCommand.request.preflight.expectedDigest || + applyCommand.request.output !== command.request.receipt.path + ) { + fail('retirement receipt command binding changed'); + } + return Object.freeze({ + compatible: true, + deploymentFamily: 'cluster', + mutationId: receipt.mutationId, + receiptDigest: receipt.receiptDigest, + preflightDigest: receipt.preflightDigest, + transitionKind: 'retire', + lockDigest: receipt.lock.lockDigest, + manifestDigest: receipt.lock.manifestDigest, + deploymentGeneration: receipt.deploymentHead.generation, + deploymentDigest: receipt.deploymentHead.deploymentDigest, + deploymentHeadStateDigest: receipt.deploymentHead.stateDigest, + retiredResourceCount: receipt.retiredResources.length, + resourceCount: receipt.resourceInventory.length, + clusterUid: receipt.target.clusterUid, + externalResultsReplayed: false, + kubernetesMutation: false, + }); +} + function executeCommand(commandFile, dependencies = {}) { const commandState = readCommand(commandFile); if (commandState.command.operation === 'cluster.deployment.preflight') { @@ -2272,6 +3681,22 @@ function executeCommand(commandFile, dependencies = {}) { if (commandState.command.operation === 'cluster.deployment.apply') { return applyDeployment(commandState, dependencies); } + if ( + commandState.command.operation === 'cluster.deployment.retirement.preflight' + ) { + return createRetirementPreflight(commandState, dependencies); + } + if ( + commandState.command.operation === 'cluster.deployment.retirement.apply' + ) { + return applyRetirement(commandState, dependencies); + } + if ( + commandState.command.operation === + 'cluster.deployment.retirement.receipt.audit' + ) { + return auditRetirementReceipt(commandState.command); + } return auditReceipt(commandState.command); } @@ -2284,6 +3709,8 @@ module.exports = Object.freeze({ LOCK_SCHEMA, PREFLIGHT_SCHEMA, RECEIPT_SCHEMA, + RETIREMENT_PREFLIGHT_SCHEMA, + RETIREMENT_RECEIPT_SCHEMA, QingLong3KubernetesDeploymentCeremonyError, canonicalJson, executeCommand, @@ -2291,4 +3718,6 @@ module.exports = Object.freeze({ validateLockReport, validatePreflight, validateReceipt, + validateRetirementPreflight, + validateRetirementReceipt, }); diff --git a/scripts/ql3-kubernetes-deployment-live-contract.cjs b/scripts/ql3-kubernetes-deployment-live-contract.cjs index fa9f3cc4..e6bd709e 100644 --- a/scripts/ql3-kubernetes-deployment-live-contract.cjs +++ b/scripts/ql3-kubernetes-deployment-live-contract.cjs @@ -49,7 +49,9 @@ function privateFile(directory, name, contents) { function executablePath(input) { const candidates = [ input, - '/Applications/Docker.app/Contents/Resources/bin/kubectl', + ...(input === 'kubectl' + ? ['/Applications/Docker.app/Contents/Resources/bin/kubectl'] + : []), ...(process.env.PATH ?? '') .split(path.delimiter) .filter(Boolean) @@ -76,9 +78,9 @@ function references() { } function lockedArtifacts() { - const releaseSetDigest = digest('d342-live-release-set'); - const catalogManifestDigest = digest('d342-live-catalog-manifest'); - const catalogReportDigest = digest('d342-live-catalog-report'); + const releaseSetDigest = digest('d343-live-release-set'); + const catalogManifestDigest = digest('d343-live-catalog-manifest'); + const catalogReportDigest = digest('d343-live-catalog-report'); const imageReferences = references(); const annotations = { 'qinglong.io/release-set-digest': releaseSetDigest, @@ -132,6 +134,16 @@ function lockedArtifacts() { }, data: { image: imageReferences.admin }, }, + { + apiVersion: 'v1', + kind: 'ConfigMap', + metadata: { + name: 'ql3-retirement-live-target', + namespace: NAMESPACE, + annotations, + }, + data: { purpose: 'uid-resource-version-retirement-live-contract' }, + }, ]; const manifest = `${resources .map((resource) => JSON.stringify(resource)) @@ -164,7 +176,7 @@ function lockedArtifacts() { count: role === 'admin' ? 2 : 1, })), manifest: { - inputDigest: digest('d342-live-source-render'), + inputDigest: digest('d343-live-source-render'), outputDigest: digest(manifest), resources: resources.length, changedResources: 5, @@ -217,6 +229,7 @@ async function main() { const ceremonyDirectory = fs.realpathSync(fixture.temporary); const kubeconfig = fs.realpathSync(fixture.kubeconfig); const kubectl = executablePath(fixture.kubectlBinary); + const curl = executablePath('curl'); const clusterUid = fixture .kubectl( ['get', 'namespace', 'kube-system', '-o=jsonpath={.metadata.uid}'], @@ -307,21 +320,21 @@ async function main() { }, ); const audit = executeCommand(auditCommand); - const deploymentHeadConfigMap = fixture.kubectlJson([ + const installedHeadConfigMap = fixture.kubectlJson([ 'get', 'configmap', HEAD_NAME, '-n', NAMESPACE, ]); - const deploymentHead = JSON.parse( - deploymentHeadConfigMap.data?.[HEAD_DATA_KEY], + const installedHead = JSON.parse( + installedHeadConfigMap.data?.[HEAD_DATA_KEY], ); if ( - deploymentHead.phase !== 'committed' || - deploymentHead.generation !== 1 || - deploymentHead.stateDigest !== receipt.deploymentHead.stateDigest || - deploymentHead.deployment?.deploymentDigest !== + installedHead.phase !== 'committed' || + installedHead.generation !== 1 || + installedHead.stateDigest !== receipt.deploymentHead.stateDigest || + installedHead.deployment?.deploymentDigest !== receipt.deploymentHead.deploymentDigest ) { fail('committed deployment head is invalid'); @@ -351,6 +364,117 @@ async function main() { fail('server-side apply field manager is unavailable'); } } + const retirementTarget = { + apiVersion: 'v1', + kind: 'ConfigMap', + namespace: NAMESPACE, + name: 'ql3-retirement-live-target', + }; + const retirementCommon = { + lockedManifest: common.lockedManifest, + lockReport: common.lockReport, + kubectl: common.kubectl, + curl: { + path: curl, + expectedDigest: digest(fs.readFileSync(curl)), + }, + kubeconfig: common.kubeconfig, + context: CONTEXT, + expectedClusterUid: clusterUid, + expectedHead: { + generation: receipt.deploymentHead.generation, + deploymentDigest: receipt.deploymentHead.deploymentDigest, + lockDigest: receipt.deploymentHead.lockDigest, + stateDigest: receipt.deploymentHead.stateDigest, + }, + targets: [retirementTarget], + }; + const retirementPreflightPath = path.join( + ceremonyDirectory, + 'retirement-preflight.json', + ); + const retirementPreflightCommand = writeCommand( + ceremonyDirectory, + 'retirement-preflight-command.json', + 'cluster.deployment.retirement.preflight', + { + preflightId: crypto.randomUUID(), + ...retirementCommon, + output: retirementPreflightPath, + }, + ); + const retirementPreflight = executeCommand(retirementPreflightCommand); + const retirementReceiptPath = path.join( + ceremonyDirectory, + 'retirement-receipt.json', + ); + const retirementApplyCommand = writeCommand( + ceremonyDirectory, + 'retirement-apply-command.json', + 'cluster.deployment.retirement.apply', + { + mutationId: crypto.randomUUID(), + preflight: { + path: retirementPreflightPath, + expectedDigest: retirementPreflight.preflightDigest, + }, + ...retirementCommon, + output: retirementReceiptPath, + }, + ); + const retirementReceipt = executeCommand(retirementApplyCommand); + const retirementAuditCommand = writeCommand( + ceremonyDirectory, + 'retirement-audit-command.json', + 'cluster.deployment.retirement.receipt.audit', + { + applyCommand: { + path: retirementApplyCommand, + expectedDigest: digest(fs.readFileSync(retirementApplyCommand)), + }, + receipt: { + path: retirementReceiptPath, + expectedDigest: retirementReceipt.receiptDigest, + }, + }, + ); + const retirementAudit = executeCommand(retirementAuditCommand); + const retiredName = fixture + .kubectl( + [ + 'get', + 'configmap', + retirementTarget.name, + '-n', + NAMESPACE, + '--ignore-not-found=true', + '-o=name', + ], + { capture: true, quiet: true }, + ) + .stdout.trim(); + const deploymentHeadConfigMap = fixture.kubectlJson([ + 'get', + 'configmap', + HEAD_NAME, + '-n', + NAMESPACE, + ]); + const deploymentHead = JSON.parse( + deploymentHeadConfigMap.data?.[HEAD_DATA_KEY], + ); + if ( + retiredName !== '' || + deploymentHead.phase !== 'committed' || + deploymentHead.generation !== 2 || + deploymentHead.transition.kind !== 'retire' || + deploymentHead.stateDigest !== + retirementReceipt.deploymentHead.stateDigest || + deploymentHead.deployment.resources.length !== + artifacts.report.manifest.resources - 1 + ) { + fail('committed resource retirement head is invalid'); + } const version = JSON.parse( fixture.kubectl(['version', '-o=json'], { capture: true, @@ -382,6 +506,18 @@ async function main() { preflightDigest: preflight.preflightDigest, receiptDigest: receipt.receiptDigest, receiptAuditCompatible: audit.compatible, + retirement: { + preflightDigest: retirementPreflight.preflightDigest, + receiptDigest: retirementReceipt.receiptDigest, + receiptAuditCompatible: retirementAudit.compatible, + targetCount: retirementReceipt.retiredResources.length, + targetAbsent: retiredName === '', + uidResourceVersionDeletePreconditions: + retirementReceipt.verification.uidResourceVersionDeletePreconditions, + deploymentHeadCas: retirementReceipt.verification.deploymentHeadCas, + inventoryCount: retirementReceipt.resourceInventory.length, + unixSocketProxy: retirementReceipt.verification.unixSocketProxy, + }, serverSideDryRun: preflight.verification.serverSideDryRun, serverSideApply: receipt.verification.serverSideApply, convergenceRead: receipt.verification.convergenceRead, diff --git a/test/back/ql3KubernetesDeploymentCeremony.test.cjs b/test/back/ql3KubernetesDeploymentCeremony.test.cjs index 6e5b35d4..bd0966db 100644 --- a/test/back/ql3KubernetesDeploymentCeremony.test.cjs +++ b/test/back/ql3KubernetesDeploymentCeremony.test.cjs @@ -15,6 +15,8 @@ const { HEAD_NAME, PREFLIGHT_SCHEMA, RECEIPT_SCHEMA, + RETIREMENT_PREFLIGHT_SCHEMA, + RETIREMENT_RECEIPT_SCHEMA, canonicalJson, executeCommand, parseCommand, @@ -198,6 +200,7 @@ function fixture(t, options = {}) { '#!/bin/sh\nexit 97\n', 0o700, ); + const curl = privateFile(directory, 'curl', '#!/bin/sh\nexit 96\n', 0o700); const kubeconfig = privateFile( directory, 'kubeconfig.yaml', @@ -225,6 +228,7 @@ users: lockedManifest, lockReportPath, kubectl, + curl, kubeconfig, report, manifestContents, @@ -445,6 +449,168 @@ function performTransition( }; } +const RETIREMENT_TARGET = Object.freeze({ + apiVersion: 'v1', + kind: 'ConfigMap', + namespace: 'qinglong-system', + name: 'ql3-extra-release-resource', +}); + +function retirementRequest(value, expectedHead) { + return { + lockedManifest: { + path: value.lockedManifest, + expectedDigest: value.report.manifest.outputDigest, + }, + lockReport: { + path: value.lockReportPath, + expectedDigest: value.report.lockDigest, + }, + kubectl: { + path: value.kubectl, + expectedDigest: fileDigest(value.kubectl), + }, + curl: { + path: value.curl, + expectedDigest: fileDigest(value.curl), + }, + kubeconfig: { + path: value.kubeconfig, + expectedDigest: fileDigest(value.kubeconfig), + }, + context: CONTEXT, + expectedClusterUid: CLUSTER_UID, + expectedHead, + targets: [RETIREMENT_TARGET], + }; +} + +function apiObjectPath(resource) { + const namespace = encodeURIComponent(resource.metadata.namespace); + const name = encodeURIComponent(resource.metadata.name); + if (resource.apiVersion === 'v1' && resource.kind === 'ConfigMap') { + return `/api/v1/namespaces/${namespace}/configmaps/${name}`; + } + throw new Error('unsupported fake API resource'); +} + +class FakeRetirementApi { + constructor(manifestContents) { + this.objects = new Map(); + this.requests = []; + this.loseNextDeleteResponse = false; + const list = JSON.parse(convergenceList(manifestContents)); + for (const resource of list.items) { + if (resource.apiVersion === 'v1' && resource.kind === 'ConfigMap') { + this.objects.set(apiObjectPath(resource), resource); + } + } + } + + request(method, requestPath, body) { + this.requests.push({ method, requestPath, body }); + if (method === 'GET' && requestPath === '/api/v1') { + return { + status: 200, + body: JSON.stringify({ + groupVersion: 'v1', + resources: [ + { + name: 'configmaps', + namespaced: true, + kind: 'ConfigMap', + verbs: ['delete', 'get', 'list'], + }, + ], + }), + }; + } + const live = this.objects.get(requestPath); + if (method === 'GET') { + return live === undefined + ? { status: 404, body: JSON.stringify({ kind: 'Status' }) } + : { status: 200, body: JSON.stringify(live) }; + } + if (method !== 'DELETE') throw new Error('unexpected fake API method'); + if (live === undefined) { + return { status: 404, body: JSON.stringify({ kind: 'Status' }) }; + } + if ( + body?.preconditions?.uid !== live.metadata.uid || + body?.preconditions?.resourceVersion !== live.metadata.resourceVersion + ) { + return { status: 409, body: JSON.stringify({ kind: 'Status' }) }; + } + if (body.dryRun?.includes('All')) { + return { status: 200, body: JSON.stringify({ kind: 'Status' }) }; + } + this.objects.delete(requestPath); + if (this.loseNextDeleteResponse) { + this.loseNextDeleteResponse = false; + throw new Error('simulated response loss'); + } + return { status: 200, body: JSON.stringify({ kind: 'Status' }) }; + } +} + +function prepareRetirement(t) { + const value = fixture(t, { extraResource: true }); + const runner = successfulRunner([], {}, value.manifestContents); + const installed = performTransition( + value, + runner, + 'install', + commonRequest(value).expectedHead, + 'retirement-install', + ); + const expectedHead = expectedFromReceipt(installed.receipt); + const api = new FakeRetirementApi(value.manifestContents); + const preflightPath = path.join(value.directory, 'retirement-preflight.json'); + const preflightCommand = writeCommand( + value.directory, + 'retirement-preflight-command.json', + 'cluster.deployment.retirement.preflight', + { + preflightId: crypto.randomUUID(), + ...retirementRequest(value, expectedHead), + output: preflightPath, + }, + ); + const preflight = executeCommand(preflightCommand, { + runProcess: runner, + retirementApi: api, + }); + return { + ...value, + runner, + installed, + expectedHead, + api, + preflight, + preflightPath, + preflightCommand, + }; +} + +function retirementApplyCommand(value, suffix = 'retirement') { + const receiptPath = path.join(value.directory, `${suffix}-receipt.json`); + const applyCommand = writeCommand( + value.directory, + `${suffix}-apply-command.json`, + 'cluster.deployment.retirement.apply', + { + mutationId: crypto.randomUUID(), + preflight: { + path: value.preflightPath, + expectedDigest: value.preflight.preflightDigest, + }, + ...retirementRequest(value, value.expectedHead), + output: receiptPath, + }, + ); + return { applyCommand, receiptPath }; +} + test('preflight binds one catalog lock to one Kubernetes target without mutation', (t) => { const value = prepare(t); assert.equal(value.preflight.schema, PREFLIGHT_SCHEMA); @@ -1017,6 +1183,168 @@ test('receipt audit rejects a different command or recomputed receipt', (t) => { assert.throws(() => executeCommand(auditCommand)); }); +test('retirement preflight binds exact UID and resourceVersion without mutation', (t) => { + const value = prepareRetirement(t); + assert.equal(value.preflight.schema, RETIREMENT_PREFLIGHT_SCHEMA); + assert.deepEqual(value.preflight.activeResourceInventory.length, 6); + assert.deepEqual(value.preflight.survivorResourceInventory.length, 5); + assert.deepEqual(value.preflight.retirementTargets[0], { + ...RETIREMENT_TARGET, + uid: '123e4567-e89b-42d3-a456-426614174105', + resourceVersion: '6', + }); + assert.equal(value.preflight.verification.kubernetesMutation, false); + const deletes = value.api.requests.filter( + (request) => request.method === 'DELETE', + ); + assert.equal(deletes.length, 1); + assert.deepEqual(deletes[0].body.preconditions, { + uid: value.preflight.retirementTargets[0].uid, + resourceVersion: value.preflight.retirementTargets[0].resourceVersion, + }); + assert.deepEqual(deletes[0].body.dryRun, ['All']); + assert.equal( + value.api.objects.has( + apiObjectPath({ + ...RETIREMENT_TARGET, + metadata: { + namespace: RETIREMENT_TARGET.namespace, + name: RETIREMENT_TARGET.name, + }, + }), + ), + true, + ); +}); + +test('retirement applies preconditioned deletion, closes inventory and audits receipt', (t) => { + const value = prepareRetirement(t); + const { applyCommand, receiptPath } = retirementApplyCommand(value); + const receipt = executeCommand(applyCommand, { + runProcess: value.runner, + retirementApi: value.api, + }); + assert.equal(receipt.schema, RETIREMENT_RECEIPT_SCHEMA); + assert.equal(receipt.resourceInventory.length, 5); + assert.equal(receipt.retiredResources.length, 1); + assert.equal( + receipt.verification.uidResourceVersionDeletePreconditions, + true, + ); + assert.equal(receipt.verification.deletionAbsenceConfirmed, true); + assert.equal(receipt.deploymentHead.generation, 2); + assert.equal( + JSON.parse(value.runner.server.head.data[HEAD_DATA_KEY]).transition.kind, + 'retire', + ); + const actualDelete = value.api.requests.find( + (request) => + request.method === 'DELETE' && request.body?.dryRun === undefined, + ); + assert.deepEqual(actualDelete.body.preconditions, { + uid: receipt.retiredResources[0].uid, + resourceVersion: receipt.retiredResources[0].resourceVersion, + }); + + const auditCommand = writeCommand( + value.directory, + 'retirement-audit-command.json', + 'cluster.deployment.retirement.receipt.audit', + { + applyCommand: { + path: applyCommand, + expectedDigest: fileDigest(applyCommand), + }, + receipt: { path: receiptPath, expectedDigest: receipt.receiptDigest }, + }, + ); + const audit = executeCommand(auditCommand); + assert.equal(audit.compatible, true); + assert.equal(audit.transitionKind, 'retire'); + assert.equal(audit.retiredResourceCount, 1); + assert.equal(audit.resourceCount, 5); + assert.equal(audit.kubernetesMutation, false); +}); + +test('retirement rejects a replacement UID before acquiring the deployment head', (t) => { + const value = prepareRetirement(t); + const { applyCommand, receiptPath } = retirementApplyCommand( + value, + 'replacement', + ); + const requestPath = [...value.api.objects.keys()].find((candidate) => + candidate.endsWith('/ql3-extra-release-resource'), + ); + const live = value.api.objects.get(requestPath); + value.api.objects.set(requestPath, { + ...live, + metadata: { ...live.metadata, uid: crypto.randomUUID() }, + }); + assert.throws(() => + executeCommand(applyCommand, { + runProcess: value.runner, + retirementApi: value.api, + }), + ); + assert.equal(fs.existsSync(receiptPath), false); + const head = JSON.parse(value.runner.server.head.data[HEAD_DATA_KEY]); + assert.equal(head.phase, 'committed'); + assert.equal(head.generation, 1); +}); + +test('retirement resumes exact intent after a lost delete response', (t) => { + const value = prepareRetirement(t); + const { applyCommand, receiptPath } = retirementApplyCommand( + value, + 'response-loss', + ); + value.api.loseNextDeleteResponse = true; + assert.throws(() => + executeCommand(applyCommand, { + runProcess: value.runner, + retirementApi: value.api, + }), + ); + assert.equal(fs.existsSync(receiptPath), false); + assert.equal( + JSON.parse(value.runner.server.head.data[HEAD_DATA_KEY]).phase, + 'applying', + ); + const receipt = executeCommand(applyCommand, { + runProcess: value.runner, + retirementApi: value.api, + }); + assert.equal(receipt.deploymentHead.generation, 2); + assert.equal(receipt.resourceInventory.length, 5); + assert.equal( + JSON.parse(value.runner.server.head.data[HEAD_DATA_KEY]).phase, + 'committed', + ); +}); + +test('rollback may restore the exact inventory retired at the same release version', (t) => { + const value = prepareRetirement(t); + const { applyCommand } = retirementApplyCommand(value, 'restore-source'); + const retired = executeCommand(applyCommand, { + runProcess: value.runner, + retirementApi: value.api, + }); + const restored = performTransition( + value, + value.runner, + 'rollback', + expectedFromReceipt(retired), + 'retirement-restore', + ); + assert.equal(restored.receipt.transitionKind, 'rollback'); + assert.equal(restored.receipt.deploymentHead.generation, 3); + assert.equal(restored.receipt.resourceInventory.length, 6); + assert.deepEqual( + restored.receipt.resourceInventory, + value.installed.receipt.resourceInventory, + ); +}); + test('command surface is closed and lock reports require exact canonical identity', () => { assert.equal( commandFile(['--command-file=/private/command.json']), @@ -1068,6 +1396,56 @@ test('repository exposes the reviewed ceremony and removes the bare apply handof ); }); +test('CI runs the isolated deployment and retirement live contract', () => { + const root = path.resolve(__dirname, '../..'); + const workflowPath = path.join( + root, + '.github/workflows/ql3-kubernetes-deployment-live.yml', + ); + const workflowSource = fs.readFileSync(workflowPath, 'utf8'); + const workflow = yaml.load(workflowSource); + const job = workflow.jobs?.['kubernetes-deployment-live']; + assert.equal(job?.['runs-on'], 'ubuntu-24.04'); + assert.equal(job?.['timeout-minutes'], 25); + assert.match(workflowSource, /rancher\/k3s:v1\.34\.3-k3s1/u); + assert.match(workflowSource, /kubectl v1\.34\.3/u); + assert.match( + workflowSource, + /node --test test\/back\/ql3KubernetesDeploymentCeremony\.test\.cjs/u, + ); + assert.match( + workflowSource, + /node scripts\/ql3-kubernetes-deployment-live-contract\.cjs/u, + ); + assert.match(workflowSource, /uidResourceVersionDeletePreconditions/u); + assert.match(workflowSource, /ql3-deploy-live-/u); +}); + +test('live contract never resolves curl through the Docker kubectl fallback', function liveToolResolutionKeepsCurlDistinct() { + const root = path.resolve(__dirname, '../..'); + const source = fs.readFileSync( + path.join(root, 'scripts/ql3-kubernetes-deployment-live-contract.cjs'), + 'utf8', + ); + assert.match( + source, + /const candidates = \[\s*input,\s*\.\.\.\(input === 'kubectl'\s*\? \['\/Applications\/Docker\.app\/Contents\/Resources\/bin\/kubectl'\]\s*: \[\]\),\s*\.\.\.\(process\.env\.PATH/u, + ); +}); + +test('private retirement proxy rejects every unrelated mutation method', function privateRetirementProxyRejectsUnrelatedMutations() { + const root = path.resolve(__dirname, '../..'); + const source = fs.readFileSync( + path.join(root, 'scripts/lib/ql3-kubernetes-deployment-ceremony.cjs'), + 'utf8', + ); + assert.match( + source, + /--reject-methods=\^\(POST\|PUT\|PATCH\|CONNECT\|OPTIONS\|TRACE\)\$/u, + ); + assert.equal(source.includes('--reject-methods=POST,PUT,PATCH'), false); +}); + test('thin CLI uses the pinned executable and keeps failures low-sensitive', (t) => { const value = fixture(t); fs.writeFileSync(