feat(ql3): fence Kubernetes resource retirement

This commit is contained in:
whyour
2026-08-17 23:45:34 +08:00
parent 2e848ef7c0
commit 8496b56bed
8 changed files with 2242 additions and 34 deletions
@@ -0,0 +1,90 @@
name: QingLong 3.0 Kubernetes deployment live contract
on:
push:
branches:
- next
pull_request:
branches:
- next
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ql3-kubernetes-deployment-live-${{ github.ref }}
cancel-in-progress: true
jobs:
kubernetes-deployment-live:
name: Deployment Head and fenced resource retirement
runs-on: ubuntu-24.04
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24.18.0'
cache: pnpm
cache-dependency-path: pnpm-lock.yaml
- name: Install workspace dependencies without lifecycle scripts
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install verified kubectl v1.34.3
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl" \
https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl.sha256" \
https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl.sha256
test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \
"$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)"
chmod 0755 "${RUNNER_TEMP}/kubectl"
- name: Preload the digest-reviewed K3s distribution
run: docker pull rancher/k3s:v1.34.3-k3s1
- name: Recheck the closed deployment ceremony contract
run: node --test test/back/ql3KubernetesDeploymentCeremony.test.cjs
- name: Prove install, UID/resourceVersion retirement and Head convergence
env:
QL3_KUBECTL_BIN: ${{ runner.temp }}/kubectl
QL3_DEPLOYMENT_LIVE_REPORT: ${{ runner.temp }}/ql3-kubernetes-deployment/report.json
shell: bash
run: |
set -euo pipefail
umask 077
install -d -m 0700 "$(dirname "${QL3_DEPLOYMENT_LIVE_REPORT}")"
node scripts/ql3-kubernetes-deployment-live-contract.cjs > \
"${QL3_DEPLOYMENT_LIVE_REPORT}"
node -e \
"const fs=require('node:fs');const report=JSON.parse(fs.readFileSync(process.env.QL3_DEPLOYMENT_LIVE_REPORT,'utf8'));if(report.cleanupComplete!==true||report.receiptAuditCompatible!==true||report.retirement?.receiptAuditCompatible!==true||report.retirement?.targetAbsent!==true||report.retirement?.uidResourceVersionDeletePreconditions!==true||report.retirement?.deploymentHeadCas!==true||report.retirement?.unixSocketProxy!==true)process.exit(1)"
- name: Verify isolated cleanup
if: always()
shell: bash
run: |
set -euo pipefail
test -z "$(docker ps -aq --filter name=ql3-deploy-live-)"
test -z "$(docker network ls -q --filter name=ql3-deploy-live-)"
- name: Upload content-free deployment evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ql3-kubernetes-deployment-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/ql3-kubernetes-deployment/report.json
if-no-files-found: warn
retention-days: 14
compression-level: 9
overwrite: false
include-hidden-files: false
+24 -2
View File
@@ -6,10 +6,32 @@
- 目标版本:QingLong 3.x - 目标版本:QingLong 3.x
- 作者:QingLong Maintainers - 作者:QingLong Maintainers
- 创建日期:2026-07-17 - 创建日期:2026-07-17
- 最后更新:2026-08-16 - 最后更新:2026-08-17
- 讨论范围:架构与演进路线,不包含最终 UI 视觉方案 - 讨论范围:架构与演进路线,不包含最终 UI 视觉方案
最新增量证据(2026-08-16): 最新增量证据(2026-08-17):
- D-343/ADR-0435(已接受;真实公开 catalog 运行仍待 release Gate):在 D-342 的 resource-inventory-closed Head 上增加独立
`cluster.deployment.retirement.preflight|apply|receipt.audit`。退役目标只能来自 current locked manifest 与 committed active inventory
最多 64 个显式 namespaced resource;固定 Head、cluster-scoped resource 以及 Secret/PVC/ServiceAccount 均拒绝进入通用删除面,survivor
inventory 必须非空并继续覆盖 `control|control-ai|admin|worker` 四类 authority。preflight 通过 API discovery 绑定每个对象的 exact
UID/resourceVersion、desired subset 与 `qinglong3-catalog-lock` Apply ownership,再发送带同一 Preconditions 的服务端 DeleteOptions
dry-runapply 在删除前复验 observation,以 Head resourceVersion CAS 取得唯一 `applying` 意图,逐对象使用 Background DELETE 并确认
old UID absent,之后才提交缩小 inventory 的下一代 committed Head 与 no-replace receipt。替换 UID、resourceVersion/字段漂移、
terminating/finalizer、Head 冲突均失败关闭;DELETE 已生效但响应丢失时,只允许相同 command/mutation/preflight 从 durable applying
Head 和 absent target 恢复。同版本 rollback 可精确恢复 retirement 前一个 inventory,仍走完整 server-side apply/convergence。
pinned kubectl 只在 ceremony 期间启动 owner-private Unix socket proxy,由 pinned curl 发送带 body 的 DELETE;不开放 TCP,不读取
ambient HOME,完成即清理。没有新增 package、第三方依赖、controller、CRD、webhook、ServiceAccount、Pod、数据库、migration、Pool、
watcher、timer 或常驻进程,Local/Edge/Standalone 零导入、零稳态成本。独立 GitHub Actions job 固定 kubectl/K3s 版本并验证真实
install、退役、Head 收敛、content-free evidence 与隔离清理;定向 deployment ceremony 22/22,完整 backend 1,339 项为 1,337 pass/
2 条件 skip/0 fail18-package clean build/test 退出 0。架构/部署审计保持 compatiblepackage boundary 为 18 packages 且
`singleSourcePackages=[]``shallowSourcePackages=[]`14 档 Local artifact 全部 compatible 且字节数与 D-342 完全一致。三节点 K3s
live 使用 digest-reviewed `rancher/k3s:v1.34.3-k3s1`,在 linux/arm64 上得到 3 个 Ready node,真实 install 后 Head generation
`1→2`、inventory `7→6`、目标 absentserver-side dry-run/apply、UID/resourceVersion DELETE preconditions、Unix socket proxy、Head
CAS 与两个 receipt audit 均通过,`cleanupComplete=true` 且 Docker container/network 残留为 0;最终 retirement receipt digest 为
`sha256:d3145b3fee7a1191458b5c3a930c0063a0e1e446d3a196565bcdd20d9da06ec5`。PostgreSQL HA 在 18.6/arm64 上完成
142/142 gates、timeline `1→2` 与独立 evidence auditmode-0600 报告 SHA-256 为
`89fd4fb47f82f35d3819d1fd9540cf3764f65fa68ac6561effd16778bee4ab8a`container/network/volume 残留为 0。
- D-342/ADR-0434(已接受;资源退休 ceremony 与真实公开 catalog 运行仍待后续 Gate):Cluster deployment command、preflight 与 - D-342/ADR-0434(已接受;资源退休 ceremony 与真实公开 catalog 运行仍待后续 Gate):Cluster deployment command、preflight 与
receipt 在尚未发布前直接升为 v2,并以目标 namespace 中唯一 `qinglong3-deployment-head` ConfigMap 建立部署顺序 authority。每次 receipt 在尚未发布前直接升为 v2,并以目标 namespace 中唯一 `qinglong3-deployment-head` ConfigMap 建立部署顺序 authority。每次
@@ -0,0 +1,96 @@
# ADR-0435UID/resourceVersion 围栏的 Kubernetes 资源退役 Ceremony
- 状态:Accepted
- 日期:2026-08-17
- 关联 RFCQL-RFC-0001 D-03、D-14、D-342、D-343
- 关联 ADRADR-0431、ADR-0433、ADR-0434
- 修正:ADR-0434 第 7 项中“资源退休仍待后续 ceremony”的未完成边界
## 上下文
ADR-0434 已让 Deployment Head 保存完整有序 inventory,并拒绝 upgrade 隐式遗漏资源,但仍没有安全缩小 inventory 的路径。普通
`kubectl delete` 不执行 `resourceVersion` 比较;只按 kind/namespace/name 删除会在旧对象已被替换时误删新 UID。`kubectl apply
--prune` 也无法把每个删除绑定到 preflight 所观察的 UID/resourceVersion、当前 Head 与独立 receipt。
删除还是 Kubernetes 多对象非事务操作。请求在 API Server 已接受后丢失响应、对象被 finalizer 留在 terminating、并发 operator
推进 Head,都会产生不能靠“再跑一次 delete”猜测的窗口。QingLong 3.0 因而需要显式、短生命周期、可恢复的 retirement ceremony
同时不能给低配 Local/Edge 设备增加 Kubernetes 依赖或常驻成本。
## 决策
1. 在既有 `qinglong/kubernetes-deployment-command@v2` 中增加
`cluster.deployment.retirement.preflight|apply|receipt.audit`,不新增 package、controller、CRD、数据库或常驻进程。退役和普通
install/upgrade/rollback 使用同一个 Deployment Head CAS authority。
2. 调用方只能提交 current locked manifest 中、current committed inventory 内的显式有序 target identity。每次最多 64 个且只允许
namespaced resourceHead ConfigMap及 `v1 Secret|PersistentVolumeClaim|ServiceAccount` 禁止进入此通用 ceremony。集群级对象、数据与
credential authority 必须使用各自专用 ceremony。
3. preflight 重新验证 catalog-bound lock、pinned kubectl/curl、owner-private kubeconfig、cluster UID 与 exact expected Head,从 API
discovery 得到唯一 plural/scope 映射,逐个 GET 并记录 exact UID/resourceVersion、期望字段与
`qinglong3-catalog-lock` Apply ownership,再以 `DeleteOptions.dryRun=[All]`、Background propagation 和 UID/resourceVersion
preconditions 执行服务端 dry-run。preflight 不改变 Kubernetes 状态。
4. survivor inventory 必须非空并继续包含 `control|control-ai|admin|worker` 四类 immutable image authority。退役集合必须精确等于
`active - survivor`,不能通过手写 receipt 或隐式 prune 改写 inventory。
5. apply 在任何删除前重读目标对象并要求 UID/resourceVersion 仍与 preflight 完全相同,然后用 Head ConfigMap 的 opaque
resourceVersion 把唯一意图 CAS 为 `phase=applying`。之后逐对象发送带相同 UID/resourceVersion preconditions 的 DELETE;仅在每个
old UID 已确认 404 absent 后,才把缩小后的 inventory 作为下一 generation 的 `committed` Head 发布,并 no-replace 写 receipt。
6. DELETE 不使用 force、grace-period=0 或 foreground 级联;固定 Background propagation。对象处于 terminating、precondition 409、
UID 替换、resourceVersion 漂移、字段漂移、API discovery 歧义、Head 冲突或输入替换均失败关闭,不发布成功 receipt。
7. 若 DELETE 已完成但响应丢失,Head 仍保存 exact applying intent。相同 command/mutation/preflight 重放时,只有目标已经 absent 才可把
该次删除记为 recovered 并继续提交;不同 UID、仍 terminating 或任何非同意图都不能自动接管。若 Head 已 committed 而本地 receipt
丢失,相同 command 可从 Head 确定性重建。
8. 普通 kubectl 缺少带 body 的安全 DELETE 接口,因此一次性启动 pinned kubectl 的 owner-private Unix socket proxy,并用 pinned
curl 发送 JSON DeleteOptions。proxy 只接受 `/api|/apis`,拒绝 POST/PUT/PATCH 等无关方法,使用独立 HOME/XDG/TMP,完成即关闭并删除
socket;不监听 TCP,不读取 ambient HOME。
9. retirement receipt 显式声明 `crossResourceAtomicity=false`
`recovery=resume_exact_retirement_from_target_head`、UID/resourceVersion preconditions、absence、survivor role closure 与 Head CAS。
offline audit 重建 command/receipt digest 闭包,但保持 `externalResultsReplayed=false``kubernetesMutation=false`
10. Head schema 在 3.0 尚未正式发布时原地升为 v2;普通 rollback 可在 current transition 为 retire 时,以同一 release version 精确恢复
Head.previous 的 lock 和完整 inventory。恢复对象仍走原 server-side apply/convergence ceremony,不把 DELETE 本身假装成可逆事务。
## 部署与资源影响
- Local/Edge/Standalone 不导入该脚本和 Kubernetes/YAML/semver 依赖,产物与稳态进程不变。
- Cluster 每次 retirement 临时产生一个私有目录、一个 Unix socket proxy 与顺序有界 API 请求;完成后全部清理。没有 listener 常驻、
watcher、timer、ServiceAccount、Pod、sidecar、Pool、migration 或第三方依赖。
- 最多 64 个 target,按顺序执行 discovery/read/dry-run/delete/absence;这是明确的运维时成本,不进入 runtime hot path。
- 多资源删除仍非事务。Head 提供的是唯一意图、精确恢复与可审计 inventory closure,不宣称 API Server 跨对象原子性。
## 被拒绝的替代方案
### `kubectl delete -f` 或按名称删除
拒绝。普通命令不会绑定 preflight 观察到的 `resourceVersion`,旧操作可能删除已经替换的新 UID。
### `kubectl apply --prune`
拒绝。隐式集合差不能形成逐对象 UID/resourceVersion 删除证明,也不能表达响应丢失恢复、finalizer 和 Head inventory commit 顺序。
### 常驻 controller/finalizer
拒绝。QingLong 不是通用 Kubernetes desired-state controller;常驻 reconciler 会扩大凭据、升级、资源占用和低配部署边界。
### force delete 或 `grace-period=0`
拒绝。它会绕过正常终止/级联语义,扩大数据丢失和依赖对象孤儿风险,也不能弥补多对象非事务事实。
## 验证
- 定向 Node 契约 22/22:原 14 项部署 Head 契约保持通过,新增 exact UID/resourceVersion dry-run、preconditioned delete、inventory
closure、offline audit、替换 UID fail-closed、DELETE 响应丢失恢复、同版本精确 rollback、独立 CI live workflow、curl/kubectl 工具解析隔离
与锚定 proxy method 正则闭包;
- 隔离三节点 K3s v1.34.3+k3s1 live 门在 linux/arm64 同时执行 install 与一个 namespaced ConfigMap 退役,验证真实 DeleteOptions
preconditions、对象 absent、Head generation 1→2、inventory 7→6、两个 receipt audit 与临时 Docker/Unix socket 清理;最终 retirement
receipt digest 为 `sha256:d3145b3fee7a1191458b5c3a930c0063a0e1e446d3a196565bcdd20d9da06ec5`,门后 container/network
残留均为 0。该门固定进入独立 GitHub Actions job,并上传 content-free evidence
- 完整 backend 为 1,339 total/1,337 pass/2 条件 skip/0 fail18-package clean build/test 退出 0。架构/边界审计保持
compatible14 档 Local artifact 字节数与 D-342 一致;
- PostgreSQL 18.6/arm64 HA 门为 142/142 gates、timeline 1→2,独立 evidence audit compatiblemode-0600 报告 SHA-256 为
`89fd4fb47f82f35d3819d1fd9540cf3764f65fa68ac6561effd16778bee4ab8a`,门后 container/network/volume 残留均为 0。完整证据记录在
QL-RFC-0001 D-343。
## 规范依据
- [Kubernetes API conceptsdeletion、resourceVersion 与条件操作](https://kubernetes.io/docs/reference/using-api/api-concepts/)
- [Kubernetes DeleteOptionsUID/resourceVersion Preconditions](https://kubernetes.io/docs/reference/kubernetes-api/definitions/delete-options-v1-meta/)
- [kubectl delete:默认不执行 resourceVersion 检查](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_delete/)
- [Kubernetes cascading deletionBackground propagation](https://kubernetes.io/docs/tasks/administer-cluster/use-cascading-deletion/)
+1
View File
@@ -438,6 +438,7 @@
| [ADR-0432](./ADR-0432-target-side-catalog-bound-local-compose-revisions.md) | 目标侧 Catalog-bound Local Compose 修订 | Accepted | | [ADR-0432](./ADR-0432-target-side-catalog-bound-local-compose-revisions.md) | 目标侧 Catalog-bound Local Compose 修订 | Accepted |
| [ADR-0433](./ADR-0433-catalog-bound-kubernetes-target-deployment-ceremony.md) | Catalog-bound Kubernetes 目标部署 Ceremony | Accepted | | [ADR-0433](./ADR-0433-catalog-bound-kubernetes-target-deployment-ceremony.md) | Catalog-bound Kubernetes 目标部署 Ceremony | Accepted |
| [ADR-0434](./ADR-0434-resource-inventory-closed-kubernetes-deployment-head.md) | Resource-inventory-closed Kubernetes Deployment Head | Accepted | | [ADR-0434](./ADR-0434-resource-inventory-closed-kubernetes-deployment-head.md) | Resource-inventory-closed Kubernetes Deployment Head | Accepted |
| [ADR-0435](./ADR-0435-uid-resource-version-fenced-kubernetes-resource-retirement.md) | UID/resourceVersion 围栏的 Kubernetes 资源退役 Ceremony | Accepted |
## 规则 ## 规则
@@ -299,6 +299,62 @@ cluster UID,最后以同一 ConfigMap 的新 resourceVersion 提交 `committed
mutation 前失败关闭。`rollback` 只接受 Head 中精确的上一部署 lock,且当前/目标 inventory 必须完全相同;涉及资源退休时须等待 mutation 前失败关闭。`rollback` 只接受 Head 中精确的上一部署 lock,且当前/目标 inventory 必须完全相同;涉及资源退休时须等待
独立的 UID/resourceVersion-precondition retirement ceremony,不能把遗留对象伪装成回滚成功。 独立的 UID/resourceVersion-precondition retirement ceremony,不能把遗留对象伪装成回滚成功。
### Kubernetes 资源退役
缩小 active inventory 必须显式使用 `cluster.deployment.retirement.preflight`,不能先手工删除,也不能使用 `kubectl delete -f`
`kubectl apply --prune`、force delete 或 `--grace-period=0`。只允许退役 current locked manifest 与 committed Head inventory 中的
namespaced 非 credential/data 对象;`Secret``PersistentVolumeClaim``ServiceAccount`、cluster-scoped resource 和固定 Head
ConfigMap 均须走专用运维流程。survivor inventory 必须仍包含 control、control-ai、admin、worker 四类 authority。
除上文相同的 locked manifest、lock report、kubectl、kubeconfig、context、cluster UID 与 expected Head 外,retirement command
还必须固定 realpath 后的 curl executable digest,并提交至多 64 个按 `apiVersion/kind/namespace/name` 排序的 target
```json
{
"schemaVersion": 1,
"schema": "qinglong/kubernetes-deployment-command@v2",
"operation": "cluster.deployment.retirement.preflight",
"request": {
"preflightId": "<new UUID>",
"lockedManifest": "<same path/digest object>",
"lockReport": "<same path/digest object>",
"kubectl": "<same path/digest object>",
"curl": {
"path": "<canonical absolute curl>",
"expectedDigest": "<SHA-256 of curl bytes>"
},
"kubeconfig": "<same path/digest object>",
"context": "<same context>",
"expectedClusterUid": "<same cluster UID>",
"expectedHead": "<exact current committed Head fields>",
"targets": [
{
"apiVersion": "v1",
"kind": "ConfigMap",
"namespace": "qinglong-system",
"name": "obsolete-release-resource"
}
],
"output": "<unused canonical absolute retirement-preflight.json>"
}
}
```
示例中的 `"<same path/digest object>"``"<exact current committed Head fields>"` 是文档占位符,实际 command 必须展开为上文
展示的 JSON object,且仍须是单行 canonical JSON。运行同一个 `pnpm cluster-deployment:ql3 -- --command-file=...` 入口。preflight
逐对象读取 exact UID/resourceVersion 与 Apply ownership,并向 API Server 发送带 Preconditions 的 `DeleteOptions` dry-run;成功报告
保持 `kubernetesMutation:false`。人工核对 active/survivor inventory、target observation 和两个 tool digest 后,生成
`cluster.deployment.retirement.apply` command:把 `preflightId` 改为新 `mutationId`,增加
`preflight={path,expectedDigest}`,其余 authority 与 targets 必须逐字一致,output 改为新的 retirement receipt 路径。
apply 在删除前再次读取对象并 CAS Head 为 `applying`,随后通过 owner-private Unix socket proxy 发送带同一 UID/resourceVersion
preconditions 和 Background propagation 的 DELETE。只有 old UID 已确认 absent 才会提交缩小 inventory 的下一代 Head。若 DELETE
响应丢失,保留 command 与 preflight 原文件并原样重跑;只有相同 applying intent 且对象已经 absent 才可恢复。对象被新 UID 替换、
resourceVersion 漂移、处于 terminating 或 Head 已被其他意图推进时必须人工处理,不能改 command 绕过围栏。
退役收据使用 `cluster.deployment.retirement.receipt.audit` 离线审计,结构与普通 audit 相同,但 applyCommand 必须指向 retirement
apply command。成功 audit 只证明 command/receipt digest 闭包,不重放外部删除结果。
离线审计使用 `cluster.deployment.receipt.audit` command,其中 `applyCommand.expectedDigest` 是 apply command 文件完整字节的 离线审计使用 `cluster.deployment.receipt.audit` command,其中 `applyCommand.expectedDigest` 是 apply command 文件完整字节的
SHA-256`receipt.expectedDigest` 是 receipt 内的 `receiptDigest`。审计不会访问 Kubernetes API,结果必须保持 SHA-256`receipt.expectedDigest` 是 receipt 内的 `receiptDigest`。审计不会访问 Kubernetes API,结果必须保持
`externalResultsReplayed:false``kubernetesMutation:false` `externalResultsReplayed:false``kubernetesMutation:false`
File diff suppressed because it is too large Load Diff
@@ -49,7 +49,9 @@ function privateFile(directory, name, contents) {
function executablePath(input) { function executablePath(input) {
const candidates = [ const candidates = [
input, input,
'/Applications/Docker.app/Contents/Resources/bin/kubectl', ...(input === 'kubectl'
? ['/Applications/Docker.app/Contents/Resources/bin/kubectl']
: []),
...(process.env.PATH ?? '') ...(process.env.PATH ?? '')
.split(path.delimiter) .split(path.delimiter)
.filter(Boolean) .filter(Boolean)
@@ -76,9 +78,9 @@ function references() {
} }
function lockedArtifacts() { function lockedArtifacts() {
const releaseSetDigest = digest('d342-live-release-set'); const releaseSetDigest = digest('d343-live-release-set');
const catalogManifestDigest = digest('d342-live-catalog-manifest'); const catalogManifestDigest = digest('d343-live-catalog-manifest');
const catalogReportDigest = digest('d342-live-catalog-report'); const catalogReportDigest = digest('d343-live-catalog-report');
const imageReferences = references(); const imageReferences = references();
const annotations = { const annotations = {
'qinglong.io/release-set-digest': releaseSetDigest, 'qinglong.io/release-set-digest': releaseSetDigest,
@@ -132,6 +134,16 @@ function lockedArtifacts() {
}, },
data: { image: imageReferences.admin }, data: { image: imageReferences.admin },
}, },
{
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: 'ql3-retirement-live-target',
namespace: NAMESPACE,
annotations,
},
data: { purpose: 'uid-resource-version-retirement-live-contract' },
},
]; ];
const manifest = `${resources const manifest = `${resources
.map((resource) => JSON.stringify(resource)) .map((resource) => JSON.stringify(resource))
@@ -164,7 +176,7 @@ function lockedArtifacts() {
count: role === 'admin' ? 2 : 1, count: role === 'admin' ? 2 : 1,
})), })),
manifest: { manifest: {
inputDigest: digest('d342-live-source-render'), inputDigest: digest('d343-live-source-render'),
outputDigest: digest(manifest), outputDigest: digest(manifest),
resources: resources.length, resources: resources.length,
changedResources: 5, changedResources: 5,
@@ -217,6 +229,7 @@ async function main() {
const ceremonyDirectory = fs.realpathSync(fixture.temporary); const ceremonyDirectory = fs.realpathSync(fixture.temporary);
const kubeconfig = fs.realpathSync(fixture.kubeconfig); const kubeconfig = fs.realpathSync(fixture.kubeconfig);
const kubectl = executablePath(fixture.kubectlBinary); const kubectl = executablePath(fixture.kubectlBinary);
const curl = executablePath('curl');
const clusterUid = fixture const clusterUid = fixture
.kubectl( .kubectl(
['get', 'namespace', 'kube-system', '-o=jsonpath={.metadata.uid}'], ['get', 'namespace', 'kube-system', '-o=jsonpath={.metadata.uid}'],
@@ -307,21 +320,21 @@ async function main() {
}, },
); );
const audit = executeCommand(auditCommand); const audit = executeCommand(auditCommand);
const deploymentHeadConfigMap = fixture.kubectlJson([ const installedHeadConfigMap = fixture.kubectlJson([
'get', 'get',
'configmap', 'configmap',
HEAD_NAME, HEAD_NAME,
'-n', '-n',
NAMESPACE, NAMESPACE,
]); ]);
const deploymentHead = JSON.parse( const installedHead = JSON.parse(
deploymentHeadConfigMap.data?.[HEAD_DATA_KEY], installedHeadConfigMap.data?.[HEAD_DATA_KEY],
); );
if ( if (
deploymentHead.phase !== 'committed' || installedHead.phase !== 'committed' ||
deploymentHead.generation !== 1 || installedHead.generation !== 1 ||
deploymentHead.stateDigest !== receipt.deploymentHead.stateDigest || installedHead.stateDigest !== receipt.deploymentHead.stateDigest ||
deploymentHead.deployment?.deploymentDigest !== installedHead.deployment?.deploymentDigest !==
receipt.deploymentHead.deploymentDigest receipt.deploymentHead.deploymentDigest
) { ) {
fail('committed deployment head is invalid'); fail('committed deployment head is invalid');
@@ -351,6 +364,117 @@ async function main() {
fail('server-side apply field manager is unavailable'); fail('server-side apply field manager is unavailable');
} }
} }
const retirementTarget = {
apiVersion: 'v1',
kind: 'ConfigMap',
namespace: NAMESPACE,
name: 'ql3-retirement-live-target',
};
const retirementCommon = {
lockedManifest: common.lockedManifest,
lockReport: common.lockReport,
kubectl: common.kubectl,
curl: {
path: curl,
expectedDigest: digest(fs.readFileSync(curl)),
},
kubeconfig: common.kubeconfig,
context: CONTEXT,
expectedClusterUid: clusterUid,
expectedHead: {
generation: receipt.deploymentHead.generation,
deploymentDigest: receipt.deploymentHead.deploymentDigest,
lockDigest: receipt.deploymentHead.lockDigest,
stateDigest: receipt.deploymentHead.stateDigest,
},
targets: [retirementTarget],
};
const retirementPreflightPath = path.join(
ceremonyDirectory,
'retirement-preflight.json',
);
const retirementPreflightCommand = writeCommand(
ceremonyDirectory,
'retirement-preflight-command.json',
'cluster.deployment.retirement.preflight',
{
preflightId: crypto.randomUUID(),
...retirementCommon,
output: retirementPreflightPath,
},
);
const retirementPreflight = executeCommand(retirementPreflightCommand);
const retirementReceiptPath = path.join(
ceremonyDirectory,
'retirement-receipt.json',
);
const retirementApplyCommand = writeCommand(
ceremonyDirectory,
'retirement-apply-command.json',
'cluster.deployment.retirement.apply',
{
mutationId: crypto.randomUUID(),
preflight: {
path: retirementPreflightPath,
expectedDigest: retirementPreflight.preflightDigest,
},
...retirementCommon,
output: retirementReceiptPath,
},
);
const retirementReceipt = executeCommand(retirementApplyCommand);
const retirementAuditCommand = writeCommand(
ceremonyDirectory,
'retirement-audit-command.json',
'cluster.deployment.retirement.receipt.audit',
{
applyCommand: {
path: retirementApplyCommand,
expectedDigest: digest(fs.readFileSync(retirementApplyCommand)),
},
receipt: {
path: retirementReceiptPath,
expectedDigest: retirementReceipt.receiptDigest,
},
},
);
const retirementAudit = executeCommand(retirementAuditCommand);
const retiredName = fixture
.kubectl(
[
'get',
'configmap',
retirementTarget.name,
'-n',
NAMESPACE,
'--ignore-not-found=true',
'-o=name',
],
{ capture: true, quiet: true },
)
.stdout.trim();
const deploymentHeadConfigMap = fixture.kubectlJson([
'get',
'configmap',
HEAD_NAME,
'-n',
NAMESPACE,
]);
const deploymentHead = JSON.parse(
deploymentHeadConfigMap.data?.[HEAD_DATA_KEY],
);
if (
retiredName !== '' ||
deploymentHead.phase !== 'committed' ||
deploymentHead.generation !== 2 ||
deploymentHead.transition.kind !== 'retire' ||
deploymentHead.stateDigest !==
retirementReceipt.deploymentHead.stateDigest ||
deploymentHead.deployment.resources.length !==
artifacts.report.manifest.resources - 1
) {
fail('committed resource retirement head is invalid');
}
const version = JSON.parse( const version = JSON.parse(
fixture.kubectl(['version', '-o=json'], { fixture.kubectl(['version', '-o=json'], {
capture: true, capture: true,
@@ -382,6 +506,18 @@ async function main() {
preflightDigest: preflight.preflightDigest, preflightDigest: preflight.preflightDigest,
receiptDigest: receipt.receiptDigest, receiptDigest: receipt.receiptDigest,
receiptAuditCompatible: audit.compatible, receiptAuditCompatible: audit.compatible,
retirement: {
preflightDigest: retirementPreflight.preflightDigest,
receiptDigest: retirementReceipt.receiptDigest,
receiptAuditCompatible: retirementAudit.compatible,
targetCount: retirementReceipt.retiredResources.length,
targetAbsent: retiredName === '',
uidResourceVersionDeletePreconditions:
retirementReceipt.verification.uidResourceVersionDeletePreconditions,
deploymentHeadCas: retirementReceipt.verification.deploymentHeadCas,
inventoryCount: retirementReceipt.resourceInventory.length,
unixSocketProxy: retirementReceipt.verification.unixSocketProxy,
},
serverSideDryRun: preflight.verification.serverSideDryRun, serverSideDryRun: preflight.verification.serverSideDryRun,
serverSideApply: receipt.verification.serverSideApply, serverSideApply: receipt.verification.serverSideApply,
convergenceRead: receipt.verification.convergenceRead, convergenceRead: receipt.verification.convergenceRead,
@@ -15,6 +15,8 @@ const {
HEAD_NAME, HEAD_NAME,
PREFLIGHT_SCHEMA, PREFLIGHT_SCHEMA,
RECEIPT_SCHEMA, RECEIPT_SCHEMA,
RETIREMENT_PREFLIGHT_SCHEMA,
RETIREMENT_RECEIPT_SCHEMA,
canonicalJson, canonicalJson,
executeCommand, executeCommand,
parseCommand, parseCommand,
@@ -198,6 +200,7 @@ function fixture(t, options = {}) {
'#!/bin/sh\nexit 97\n', '#!/bin/sh\nexit 97\n',
0o700, 0o700,
); );
const curl = privateFile(directory, 'curl', '#!/bin/sh\nexit 96\n', 0o700);
const kubeconfig = privateFile( const kubeconfig = privateFile(
directory, directory,
'kubeconfig.yaml', 'kubeconfig.yaml',
@@ -225,6 +228,7 @@ users:
lockedManifest, lockedManifest,
lockReportPath, lockReportPath,
kubectl, kubectl,
curl,
kubeconfig, kubeconfig,
report, report,
manifestContents, manifestContents,
@@ -445,6 +449,168 @@ function performTransition(
}; };
} }
const RETIREMENT_TARGET = Object.freeze({
apiVersion: 'v1',
kind: 'ConfigMap',
namespace: 'qinglong-system',
name: 'ql3-extra-release-resource',
});
function retirementRequest(value, expectedHead) {
return {
lockedManifest: {
path: value.lockedManifest,
expectedDigest: value.report.manifest.outputDigest,
},
lockReport: {
path: value.lockReportPath,
expectedDigest: value.report.lockDigest,
},
kubectl: {
path: value.kubectl,
expectedDigest: fileDigest(value.kubectl),
},
curl: {
path: value.curl,
expectedDigest: fileDigest(value.curl),
},
kubeconfig: {
path: value.kubeconfig,
expectedDigest: fileDigest(value.kubeconfig),
},
context: CONTEXT,
expectedClusterUid: CLUSTER_UID,
expectedHead,
targets: [RETIREMENT_TARGET],
};
}
function apiObjectPath(resource) {
const namespace = encodeURIComponent(resource.metadata.namespace);
const name = encodeURIComponent(resource.metadata.name);
if (resource.apiVersion === 'v1' && resource.kind === 'ConfigMap') {
return `/api/v1/namespaces/${namespace}/configmaps/${name}`;
}
throw new Error('unsupported fake API resource');
}
class FakeRetirementApi {
constructor(manifestContents) {
this.objects = new Map();
this.requests = [];
this.loseNextDeleteResponse = false;
const list = JSON.parse(convergenceList(manifestContents));
for (const resource of list.items) {
if (resource.apiVersion === 'v1' && resource.kind === 'ConfigMap') {
this.objects.set(apiObjectPath(resource), resource);
}
}
}
request(method, requestPath, body) {
this.requests.push({ method, requestPath, body });
if (method === 'GET' && requestPath === '/api/v1') {
return {
status: 200,
body: JSON.stringify({
groupVersion: 'v1',
resources: [
{
name: 'configmaps',
namespaced: true,
kind: 'ConfigMap',
verbs: ['delete', 'get', 'list'],
},
],
}),
};
}
const live = this.objects.get(requestPath);
if (method === 'GET') {
return live === undefined
? { status: 404, body: JSON.stringify({ kind: 'Status' }) }
: { status: 200, body: JSON.stringify(live) };
}
if (method !== 'DELETE') throw new Error('unexpected fake API method');
if (live === undefined) {
return { status: 404, body: JSON.stringify({ kind: 'Status' }) };
}
if (
body?.preconditions?.uid !== live.metadata.uid ||
body?.preconditions?.resourceVersion !== live.metadata.resourceVersion
) {
return { status: 409, body: JSON.stringify({ kind: 'Status' }) };
}
if (body.dryRun?.includes('All')) {
return { status: 200, body: JSON.stringify({ kind: 'Status' }) };
}
this.objects.delete(requestPath);
if (this.loseNextDeleteResponse) {
this.loseNextDeleteResponse = false;
throw new Error('simulated response loss');
}
return { status: 200, body: JSON.stringify({ kind: 'Status' }) };
}
}
function prepareRetirement(t) {
const value = fixture(t, { extraResource: true });
const runner = successfulRunner([], {}, value.manifestContents);
const installed = performTransition(
value,
runner,
'install',
commonRequest(value).expectedHead,
'retirement-install',
);
const expectedHead = expectedFromReceipt(installed.receipt);
const api = new FakeRetirementApi(value.manifestContents);
const preflightPath = path.join(value.directory, 'retirement-preflight.json');
const preflightCommand = writeCommand(
value.directory,
'retirement-preflight-command.json',
'cluster.deployment.retirement.preflight',
{
preflightId: crypto.randomUUID(),
...retirementRequest(value, expectedHead),
output: preflightPath,
},
);
const preflight = executeCommand(preflightCommand, {
runProcess: runner,
retirementApi: api,
});
return {
...value,
runner,
installed,
expectedHead,
api,
preflight,
preflightPath,
preflightCommand,
};
}
function retirementApplyCommand(value, suffix = 'retirement') {
const receiptPath = path.join(value.directory, `${suffix}-receipt.json`);
const applyCommand = writeCommand(
value.directory,
`${suffix}-apply-command.json`,
'cluster.deployment.retirement.apply',
{
mutationId: crypto.randomUUID(),
preflight: {
path: value.preflightPath,
expectedDigest: value.preflight.preflightDigest,
},
...retirementRequest(value, value.expectedHead),
output: receiptPath,
},
);
return { applyCommand, receiptPath };
}
test('preflight binds one catalog lock to one Kubernetes target without mutation', (t) => { test('preflight binds one catalog lock to one Kubernetes target without mutation', (t) => {
const value = prepare(t); const value = prepare(t);
assert.equal(value.preflight.schema, PREFLIGHT_SCHEMA); assert.equal(value.preflight.schema, PREFLIGHT_SCHEMA);
@@ -1017,6 +1183,168 @@ test('receipt audit rejects a different command or recomputed receipt', (t) => {
assert.throws(() => executeCommand(auditCommand)); assert.throws(() => executeCommand(auditCommand));
}); });
test('retirement preflight binds exact UID and resourceVersion without mutation', (t) => {
const value = prepareRetirement(t);
assert.equal(value.preflight.schema, RETIREMENT_PREFLIGHT_SCHEMA);
assert.deepEqual(value.preflight.activeResourceInventory.length, 6);
assert.deepEqual(value.preflight.survivorResourceInventory.length, 5);
assert.deepEqual(value.preflight.retirementTargets[0], {
...RETIREMENT_TARGET,
uid: '123e4567-e89b-42d3-a456-426614174105',
resourceVersion: '6',
});
assert.equal(value.preflight.verification.kubernetesMutation, false);
const deletes = value.api.requests.filter(
(request) => request.method === 'DELETE',
);
assert.equal(deletes.length, 1);
assert.deepEqual(deletes[0].body.preconditions, {
uid: value.preflight.retirementTargets[0].uid,
resourceVersion: value.preflight.retirementTargets[0].resourceVersion,
});
assert.deepEqual(deletes[0].body.dryRun, ['All']);
assert.equal(
value.api.objects.has(
apiObjectPath({
...RETIREMENT_TARGET,
metadata: {
namespace: RETIREMENT_TARGET.namespace,
name: RETIREMENT_TARGET.name,
},
}),
),
true,
);
});
test('retirement applies preconditioned deletion, closes inventory and audits receipt', (t) => {
const value = prepareRetirement(t);
const { applyCommand, receiptPath } = retirementApplyCommand(value);
const receipt = executeCommand(applyCommand, {
runProcess: value.runner,
retirementApi: value.api,
});
assert.equal(receipt.schema, RETIREMENT_RECEIPT_SCHEMA);
assert.equal(receipt.resourceInventory.length, 5);
assert.equal(receipt.retiredResources.length, 1);
assert.equal(
receipt.verification.uidResourceVersionDeletePreconditions,
true,
);
assert.equal(receipt.verification.deletionAbsenceConfirmed, true);
assert.equal(receipt.deploymentHead.generation, 2);
assert.equal(
JSON.parse(value.runner.server.head.data[HEAD_DATA_KEY]).transition.kind,
'retire',
);
const actualDelete = value.api.requests.find(
(request) =>
request.method === 'DELETE' && request.body?.dryRun === undefined,
);
assert.deepEqual(actualDelete.body.preconditions, {
uid: receipt.retiredResources[0].uid,
resourceVersion: receipt.retiredResources[0].resourceVersion,
});
const auditCommand = writeCommand(
value.directory,
'retirement-audit-command.json',
'cluster.deployment.retirement.receipt.audit',
{
applyCommand: {
path: applyCommand,
expectedDigest: fileDigest(applyCommand),
},
receipt: { path: receiptPath, expectedDigest: receipt.receiptDigest },
},
);
const audit = executeCommand(auditCommand);
assert.equal(audit.compatible, true);
assert.equal(audit.transitionKind, 'retire');
assert.equal(audit.retiredResourceCount, 1);
assert.equal(audit.resourceCount, 5);
assert.equal(audit.kubernetesMutation, false);
});
test('retirement rejects a replacement UID before acquiring the deployment head', (t) => {
const value = prepareRetirement(t);
const { applyCommand, receiptPath } = retirementApplyCommand(
value,
'replacement',
);
const requestPath = [...value.api.objects.keys()].find((candidate) =>
candidate.endsWith('/ql3-extra-release-resource'),
);
const live = value.api.objects.get(requestPath);
value.api.objects.set(requestPath, {
...live,
metadata: { ...live.metadata, uid: crypto.randomUUID() },
});
assert.throws(() =>
executeCommand(applyCommand, {
runProcess: value.runner,
retirementApi: value.api,
}),
);
assert.equal(fs.existsSync(receiptPath), false);
const head = JSON.parse(value.runner.server.head.data[HEAD_DATA_KEY]);
assert.equal(head.phase, 'committed');
assert.equal(head.generation, 1);
});
test('retirement resumes exact intent after a lost delete response', (t) => {
const value = prepareRetirement(t);
const { applyCommand, receiptPath } = retirementApplyCommand(
value,
'response-loss',
);
value.api.loseNextDeleteResponse = true;
assert.throws(() =>
executeCommand(applyCommand, {
runProcess: value.runner,
retirementApi: value.api,
}),
);
assert.equal(fs.existsSync(receiptPath), false);
assert.equal(
JSON.parse(value.runner.server.head.data[HEAD_DATA_KEY]).phase,
'applying',
);
const receipt = executeCommand(applyCommand, {
runProcess: value.runner,
retirementApi: value.api,
});
assert.equal(receipt.deploymentHead.generation, 2);
assert.equal(receipt.resourceInventory.length, 5);
assert.equal(
JSON.parse(value.runner.server.head.data[HEAD_DATA_KEY]).phase,
'committed',
);
});
test('rollback may restore the exact inventory retired at the same release version', (t) => {
const value = prepareRetirement(t);
const { applyCommand } = retirementApplyCommand(value, 'restore-source');
const retired = executeCommand(applyCommand, {
runProcess: value.runner,
retirementApi: value.api,
});
const restored = performTransition(
value,
value.runner,
'rollback',
expectedFromReceipt(retired),
'retirement-restore',
);
assert.equal(restored.receipt.transitionKind, 'rollback');
assert.equal(restored.receipt.deploymentHead.generation, 3);
assert.equal(restored.receipt.resourceInventory.length, 6);
assert.deepEqual(
restored.receipt.resourceInventory,
value.installed.receipt.resourceInventory,
);
});
test('command surface is closed and lock reports require exact canonical identity', () => { test('command surface is closed and lock reports require exact canonical identity', () => {
assert.equal( assert.equal(
commandFile(['--command-file=/private/command.json']), commandFile(['--command-file=/private/command.json']),
@@ -1068,6 +1396,56 @@ test('repository exposes the reviewed ceremony and removes the bare apply handof
); );
}); });
test('CI runs the isolated deployment and retirement live contract', () => {
const root = path.resolve(__dirname, '../..');
const workflowPath = path.join(
root,
'.github/workflows/ql3-kubernetes-deployment-live.yml',
);
const workflowSource = fs.readFileSync(workflowPath, 'utf8');
const workflow = yaml.load(workflowSource);
const job = workflow.jobs?.['kubernetes-deployment-live'];
assert.equal(job?.['runs-on'], 'ubuntu-24.04');
assert.equal(job?.['timeout-minutes'], 25);
assert.match(workflowSource, /rancher\/k3s:v1\.34\.3-k3s1/u);
assert.match(workflowSource, /kubectl v1\.34\.3/u);
assert.match(
workflowSource,
/node --test test\/back\/ql3KubernetesDeploymentCeremony\.test\.cjs/u,
);
assert.match(
workflowSource,
/node scripts\/ql3-kubernetes-deployment-live-contract\.cjs/u,
);
assert.match(workflowSource, /uidResourceVersionDeletePreconditions/u);
assert.match(workflowSource, /ql3-deploy-live-/u);
});
test('live contract never resolves curl through the Docker kubectl fallback', function liveToolResolutionKeepsCurlDistinct() {
const root = path.resolve(__dirname, '../..');
const source = fs.readFileSync(
path.join(root, 'scripts/ql3-kubernetes-deployment-live-contract.cjs'),
'utf8',
);
assert.match(
source,
/const candidates = \[\s*input,\s*\.\.\.\(input === 'kubectl'\s*\? \['\/Applications\/Docker\.app\/Contents\/Resources\/bin\/kubectl'\]\s*: \[\]\),\s*\.\.\.\(process\.env\.PATH/u,
);
});
test('private retirement proxy rejects every unrelated mutation method', function privateRetirementProxyRejectsUnrelatedMutations() {
const root = path.resolve(__dirname, '../..');
const source = fs.readFileSync(
path.join(root, 'scripts/lib/ql3-kubernetes-deployment-ceremony.cjs'),
'utf8',
);
assert.match(
source,
/--reject-methods=\^\(POST\|PUT\|PATCH\|CONNECT\|OPTIONS\|TRACE\)\$/u,
);
assert.equal(source.includes('--reject-methods=POST,PUT,PATCH'), false);
});
test('thin CLI uses the pinned executable and keeps failures low-sensitive', (t) => { test('thin CLI uses the pinned executable and keeps failures low-sensitive', (t) => {
const value = fixture(t); const value = fixture(t);
fs.writeFileSync( fs.writeFileSync(