From 2485d61e8a24532d8e9d57508a23f3dfaf72f6ed Mon Sep 17 00:00:00 2001 From: whyour Date: Fri, 14 Aug 2026 13:57:16 +0800 Subject: [PATCH] test(ql3): gate failed plugin upgrades before activation --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 1 + ...-plugin-package-candidate-qualification.md | 20 +- docs/adr/README.md | 2 +- ...l3-plugin-package-recovery-e2e-fixture.cjs | 544 +++++++++++++----- ...gin-package-recovery-e2e-live-contract.cjs | 344 +++++++++-- ...ginPackageRecoveryE2ELiveContract.test.cjs | 81 +-- 6 files changed, 736 insertions(+), 256 deletions(-) diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index e4c7f79a..17984da2 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -35,6 +35,7 @@ - 2026-08-14 终态恢复更新：Secret Action controller 不再把所有终态 Job 仅计为瞬时 `recoveryRequired`。Job 到达 Complete/Failed 后已停止执行，controller 会用 started execution 的原 lease fence 复验不可变业务结果：首次 binding 必须与 approval plan、`startedAtMs` 推导出的 binding 完全一致；transition 必须与 plan、authority evidence、commit time 推导出的 receipt 完全一致。精确 durable result 存在时补写 `succeeded`，即使 Job 已被 TTL 清理也能收敛；Failed 且无 durable mutation 时写 `failed`；Complete 但无 receipt 时以 `indeterminate` 写 `blocked`。Job 在 start barrier 前终态或审批过期且尚未创建时，controller 复用既有 claim→release fence 写 `blocked`，不让坏 Job 永久占据 reconciler 页首。任何 stored result 漂移继续抛出 conflict，`executing + Job 缺失 + receipt 缺失` 继续要求人工处理，绝不自动重建可能已产生副作用的动作。该切片不修改共享 execution schema、PostgreSQL migration 或角色权限，不新增 package、连接与常驻进程；Cluster controller 复用现有 package-executor Pool，Edge 零变化。controller/process 定向 21/21，cluster-admin 全包 348 项为 345 pass/3 条件 skip/0 fail；完整 18-package 串行 build/test 退出 0；backend 1196 项为 1194 pass/2 条件 skip/0 fail；package boundary、cluster dependency、edge import、cluster deployment 均无 finding，部署/包边界聚焦测试 61/61。PostgreSQL `18.4` arm64 physical HA 125 项、timeline `1→2` 通过，报告 SHA-256 为 `bec512767fbbd7774baa9366698f60c25c8b017ed66f459b154d143fe86293bc`，临时 Docker 资源已清理。 - 2026-08-14 人工恢复更新（ADR-0397，已接受）：上述唯一保留的 `executing + Job/receipt 均缺失` 不确定窗口现在具有显式 Cluster 产品处置路径。既有 Approval management mTLS/OIDC endpoint 新增 `approval.recover.inspect|resolve`，只接受五分钟内 `multi_factor|hardware` User、独立 `approval.recover` 权限、二次认证、exact execution version/digest 和外部 evidence SHA-256。只允许 Secret binding/transition action；`confirm_failed` 写 failed，`abandon_unknown` 写 blocked，永远禁止人工 succeeded、Job 重建或 execution 重置。PostgreSQL `pg-0065`/capability v64 新增不可变 resolution ledger 与单个 SECURITY DEFINER resolver，在同一事务内锁 Policy/execution fence、写 allowed audit、推进终态并写 receipt；Approval manager 只有 dispatch/execution/resolution SELECT 与函数 EXECUTE，没有 execution UPDATE。通用 execution repository 与 Worker Credential 调用链保持不变。真实 PostgreSQL 18.4 已从空库完成 65 migration，证明原子提交、exact replay 不重复审计和 direct UPDATE `42501`；实现不新增 package、依赖、Pod、Pool、daemon、timer、watcher 或 Edge/Standalone 负担。18-package clean build/test 退出 0，backend 1,194 pass/2 skip/0 fail，package/dependency/edge/deployment 审计零 finding；新 migration 与 repository 内聚到 `approved-action` 领域，migration ledger 直属源码保持审定上限 65。PostgreSQL 18.4 arm64 physical HA 125 项 gate、timeline `1→2` 通过，报告 SHA-256 为 `6d4921cba74475d15722a13c6a8034793c0ee25681bc7dcaf91024927c5752fe`，临时 Docker 资源已清理。 - 2026-08-14 升级失败收口（ADR-0398，进行中）：已存在旧 active 的 `upgrade|reinstall|rollback` 不再先切 active pointer 再物化 Package 资源。共享 activation prerequisite sequence 固定为 Secret binding/transition receipt 就绪后，从 staged install 与 immutable lock 构建候选 generation，按既有字节上限完成 Manifest、Task、Workflow、Prompt、Tool 语义物化，并以 generation digest 预发布 immutable revision；只有成功后才进入 active pointer CAS。确定性候选错误把本次 install 写为 `failed(activation_fact_conflict)`，`activeLockDigest` 保持 `previousActiveLockDigest` 且 publisher 不被调用；瞬时文件/OCI/数据库故障保持 staged 重试。generation 1 没有旧版本可保留，Secret-aware 首次安装继续使用 ADR-0395 的 post-activation B1 binding ceremony，不被候选门错误拒绝。激活后的 publication recovery 只复用 revision 做 generation-fenced reconciliation。实现复用既有 18 个 package、双方言 repository、Local 单 SQLite authority 与 Cluster caller-driven recovery Job/单 Pool，不新增 migration、表、依赖、daemon、timer、watcher、listener、连接池或常驻 cache。定向 Runtime Core 21/21、Runtime Core 全量 548/548、Local Application 47 pass/4 条件 skip、Cluster Admin 347 pass/3 条件 skip；18-package clean build/test 退出 0，backend 1196 项为 1194 pass/2 条件 skip/0 fail，package boundary、cluster dependency、cluster deployment 与 edge import 审计均通过。PostgreSQL `18.4` arm64 physical HA 125 项、timeline `1→2` 通过，报告 SHA-256 `75d7a52be75c22b2aacf32f2d7e2c432a467ebaab4d639668ff3a4b98767a17e`。真实 Kubernetes 失败升级未移动 active pointer/head 的现场门与固定物理低配设备证据仍待闭合。 + - 2026-08-14 失败升级现场门更新：既有 PostgreSQL/OCI/Kubernetes recovery E2E 已升级为 report v2。门先激活 signed OCI generation 1，再排入包含合法 Task 与循环 Workflow 的 generation 2；第一次 recovery 必须因 transition receipt 缺失而失败并保持 staged，提交 content-free receipt 后，第二次 recovery 必须确定性写入 `failed(activation_fact_conflict)`，且 generation 2 materialized revision 数量为 0。门在升级前后逐字比较 active ConfigMap UID、`resourceVersion` 与完整 `active.json`，从而排除“先切 pointer、再补偿”的假安全；OCI v1 六路径各取一次、v2 六路径各取两次，全部要求 HTTPS、exact Basic authentication 与零 redirect。runtime rollout 只绑定最终成功 recovery Job，ConfigMap-only RBAC 与 runtime 数据库隔离保持不变。源码契约 7/7、18-package clean build/test、backend 1194 pass/2 skip/0 fail、package/dependency/deployment/edge 审计均通过；PostgreSQL 18.4 arm64 physical HA 125 gate、timeline `1→2` 通过，报告 SHA-256 `8560469694c67776e5e4c70977f8bde8d4f5635f8e7d1c293ef449dc6da59f72`。本机 admin/control 镜像构建成功，但固定 Kind 1.32.8 节点镜像不在缓存且受限网络无法取得，门在创建节点前中止并确认无遗留集群/容器；因此远端 Kubernetes 成功记录与固定物理低配设备证据仍阻断 ADR Accepted。 - D-302/ADR-0390（已接受） Cluster operator context 增加无网络、无 mutation 的内建 `ql3-cluster-admin context validate` 预检。它先复用 owner-private context reader，再让每个 entry 经过与真实请求相同的 production HTTPS/Kubernetes configuration preparation，验证精确 route、hostname、CA、 diff --git a/docs/adr/ADR-0398-pre-activation-plugin-package-candidate-qualification.md b/docs/adr/ADR-0398-pre-activation-plugin-package-candidate-qualification.md index 3c0258ad..c4dc22b3 100644 --- a/docs/adr/ADR-0398-pre-activation-plugin-package-candidate-qualification.md +++ b/docs/adr/ADR-0398-pre-activation-plugin-package-candidate-qualification.md @@ -63,13 +63,19 @@ ConfigMap 和数据库 head 之间会出现第二次分布式提交窗口，并 - Runtime Core 定向 21/21 通过，覆盖前置条件顺序、候选预物化、exact replay、无效语义拒绝、generation 1 B1 兼容和升级失败保留 旧 `activeLockDigest`；拒绝路径的 activation publisher 调用次数为零。 -- Local Application 47 pass/4 条件 skip；Cluster Admin 347 pass/3 条件 skip。Cluster 首次在 - 沙箱内运行的 32 个失败均为回环监听 `EPERM`，在允许本机监听的环境重跑后全部通过。 -- 18-package clean build/test 退出 0；Runtime Core 548/548；backend 1196 项为 - 1194 pass/2 条件 skip/0 fail。package boundary 保持 18 个 package 且 +- PostgreSQL/OCI/Kubernetes 现场门已升级为 `qinglong/plugin-package-recovery-e2e-live-contract@v2`：先用真实 signed OCI package + 激活 generation 1，再创建包含合法 Task 与循环 Workflow 的 generation 2；第一次 recovery 必须因 transition receipt 缺失而以 + `ClusterPluginPackageRecoveryRequiredError` 失败并留下 `staged`，提交 content-free transition receipt 后，第二次 recovery 必须把升级写为 + `failed(activation_fact_conflict)`，且 generation 2 materialized revision 数量仍为 0。现场门逐字比较 active ConfigMap 的 UID、 + `resourceVersion` 与完整 `active.json`，因此不能用“错误切换后再补偿回来”冒充旧版本未移动；OCI v1 六个路径各读取一次，v2 六个路径 + 各读取两次，全部要求 HTTPS、exact Basic authentication、200 且无 redirect。最终 runtime rollout 仍只绑定最后一个成功 recovery Job， + recovery ServiceAccount 继续只有 ConfigMap `get|create|update`，runtime 角色仍不能读取安装 authority。 +- 18-package clean build/test 在允许 loopback TLS 的环境退出 0；backend 1196 项为 + 1194 pass/2 条件 skip/0 fail。新增/更新的 recovery E2E 源码契约 7/7，Runtime Core 定向 21/21。package boundary 保持 18 个 package 且 `singleSourcePackages=[]`、`shallowSourcePackages=[]`；cluster dependency、cluster deployment 与 edge import 审计均无 finding。 - PostgreSQL `18.4` arm64 physical HA 通过 125 项门，timeline `1→2`，报告 SHA-256 - `75d7a52be75c22b2aacf32f2d7e2c432a467ebaab4d639668ff3a4b98767a17e`，临时 Docker - 资源已清理。真实 Kubernetes 失败升级未移动 active pointer/head 的现场门与固定物理低配设备 - 证据仍待完成，因此本 ADR 保持 Proposed。 + `8560469694c67776e5e4c70977f8bde8d4f5635f8e7d1c293ef449dc6da59f72`，临时 Docker + 资源已清理。本机已成功构建现场门所需 admin/control 镜像，但固定 `kindest/node:v1.32.8` 不在本地缓存，受限网络拉取数分钟无进度； + 门在创建任何 Kind 节点前被中止，并确认没有遗留集群或容器。因此 v2 门的代码与离线契约已完成，但仍不能计为真实 Kubernetes + 现场通过；远端 CI 成功记录与固定物理低配设备证据仍待完成，本 ADR 保持 Proposed。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 14de7d6e..b1d1b194 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -401,7 +401,7 @@ | [ADR-0395](./ADR-0395-owner-confirmed-plugin-package-secret-binding.md) | Owner 确认的 Plugin Package Secret 首次绑定 | Proposed | | [ADR-0396](./ADR-0396-generation-transition-plugin-package-secret-binding.md) | 按 Package Generation 切换 Plugin Package Secret Binding | Proposed | | [ADR-0397](./ADR-0397-explicit-cluster-secret-action-manual-recovery.md) | Cluster Secret Action 显式人工恢复 | Accepted（实现、单节点 PostgreSQL、完整 workspace/后端/边界与 physical HA 门完成） | -| [ADR-0398](./ADR-0398-pre-activation-plugin-package-candidate-qualification.md) | Plugin Package 激活前候选资格校验与自动保留旧版本 | Proposed（实现、全量/HA 门完成，Kubernetes 失败升级与物理低配门待闭合） | +| [ADR-0398](./ADR-0398-pre-activation-plugin-package-candidate-qualification.md) | Plugin Package 激活前候选资格校验与自动保留旧版本 | Proposed（v2 现场门已编排、全量/HA 门完成；远端 Kubernetes 成功记录与物理低配门待闭合） | ## 规则 diff --git a/scripts/ql3-plugin-package-recovery-e2e-fixture.cjs b/scripts/ql3-plugin-package-recovery-e2e-fixture.cjs index b9a6a7f6..111e3783 100644 --- a/scripts/ql3-plugin-package-recovery-e2e-fixture.cjs +++ b/scripts/ql3-plugin-package-recovery-e2e-fixture.cjs @@ -6,14 +6,9 @@ const fs = require('node:fs'); const https = require('node:https'); const path = require('node:path'); const { createRequire } = require('node:module'); -const { - createHash, - generateKeyPairSync, - sign, -} = require('node:crypto'); +const { createHash, generateKeyPairSync, sign } = require('node:crypto'); -const FIXTURE_SCHEMA = - 'qinglong/plugin-package-recovery-e2e-fixture@v1'; +const FIXTURE_SCHEMA = 'qinglong/plugin-package-recovery-e2e-fixture@v1'; const REGISTRY_EVENT_SCHEMA = 'qinglong/plugin-package-recovery-e2e-registry-event@v1'; const OCI_MANIFEST = 'application/vnd.oci.image.manifest.v1+json'; @@ -74,10 +69,7 @@ function tarHeader(entryPath, size) { Buffer.from('ustar\0').copy(header, 257); Buffer.from('00').copy(header, 263); const checksum = header.reduce((total, byte) => total + byte, 0); - Buffer.from(`${checksum.toString(8).padStart(6, '0')}\0 `).copy( - header, - 148, - ); + Buffer.from(`${checksum.toString(8).padStart(6, '0')}\0 `).copy(header, 148); return header; } @@ -92,18 +84,17 @@ function canonicalTar(entries) { return Buffer.concat(parts); } -function pluginManifest(architecture) { - const { - PLUGIN_PACKAGE_API_VERSION, - PLUGIN_PACKAGE_KIND, - } = ql3Require('@qinglong/runtime-core/plugin-package'); +function pluginManifest(architecture, version, invalidUpgrade = false) { + const { PLUGIN_PACKAGE_API_VERSION, PLUGIN_PACKAGE_KIND } = ql3Require( + '@qinglong/runtime-core/plugin-package', + ); return { apiVersion: PLUGIN_PACKAGE_API_VERSION, kind: PLUGIN_PACKAGE_KIND, metadata: { name: 'e2e-monitor', displayName: 'E2E Monitor', - version: '1.0.0', + version, description: 'One bounded end-to-end recovery package', license: 'Apache-2.0', }, @@ -121,13 +112,55 @@ function pluginManifest(architecture) { permissions: { network: { allowedHosts: [] }, secrets: [], - tools: [], + tools: invalidUpgrade ? ['system.command'] : [], }, - contents: { tasks: [], workflows: [], prompts: [], tools: [] }, + contents: invalidUpgrade + ? { + tasks: ['tasks/noop.json'], + workflows: ['workflows/cycle.json'], + prompts: [], + tools: [], + } + : { tasks: [], workflows: [], prompts: [], tools: [] }, }, }; } +function invalidUpgradeResources() { + return Object.freeze({ + 'tasks/noop.json': Object.freeze({ + schema: 'qinglong/plugin-package-task-resource@v1', + id: 'noop', + name: 'No-op', + labels: Object.freeze({}), + enabled: true, + kind: 'command', + spec: Object.freeze({ + schema: 'qinglong/command@v1', + config: Object.freeze({ + command: Object.freeze({ + kind: 'argv', + file: '/usr/bin/printf', + args: Object.freeze(['ok']), + }), + environment: Object.freeze([]), + timeoutMs: 30_000, + }), + }), + }), + 'workflows/cycle.json': Object.freeze({ + schema: 'qinglong/plugin-package-workflow-resource@v1', + id: 'cycle', + name: 'Rejected cyclic workflow', + enabled: true, + steps: Object.freeze([ + Object.freeze({ id: 'first', task: 'noop', needs: ['second'] }), + Object.freeze({ id: 'second', task: 'noop', needs: ['first'] }), + ]), + }), + }); +} + function route(path, mediaType, body) { return Object.freeze({ path, @@ -137,44 +170,38 @@ function route(path, mediaType, body) { }); } -function createFixture({ registry, architecture, createdAtMs = Date.now() }) { - if ( - typeof registry !== 'string' || - !/^[a-z0-9](?:[-a-z0-9.]{0,251}[a-z0-9])?$/.test(registry) || - !['amd64', 'arm64'].includes(architecture) || - !Number.isSafeInteger(createdAtMs) || - createdAtMs < 1 - ) { - throw new TypeError('Plugin Package E2E fixture options are invalid'); - } - const { - PluginPackagePublisherTrustRegistry, - PLUGIN_PACKAGE_SIGNATURE_SCHEMA, - pluginPackageContentTreeDigest, - pluginPackagePublisherSignaturePayload, - } = ql3Require('@qinglong/runtime-core/plugin-package-bundle'); - const { - planPluginPackageInstall, - } = ql3Require('@qinglong/runtime-core/plugin-package'); - const { - createPluginPackageLock, - pluginPackageInstallActionDigest, - pluginPackageInstallPlanDigest, - serializePluginPackageManifest, - } = ql3Require('@qinglong/runtime-core/plugin-package-install'); +function packageMaterial(registry, manifest, resourceValues) { + const { pluginPackageContentTreeDigest } = ql3Require( + '@qinglong/runtime-core/plugin-package-bundle', + ); + const { serializePluginPackageManifest } = ql3Require( + '@qinglong/runtime-core/plugin-package-install', + ); const { PLUGIN_PACKAGE_OCI_ARTIFACT_TYPE, PLUGIN_PACKAGE_OCI_CONFIG_MEDIA_TYPE, PLUGIN_PACKAGE_OCI_SIGNATURE_ARTIFACT_TYPE, PLUGIN_PACKAGE_OCI_SIGNATURE_CONFIG_MEDIA_TYPE, } = ql3Require('@qinglong/cluster-admin/plugin-package-oci-stage'); - - const manifest = pluginManifest(architecture); + const resourceEntries = Object.entries(resourceValues) + .map(([entryPath, value]) => ({ + path: entryPath, + body: jsonBytes(value), + })) + .sort((left, right) => left.path.localeCompare(right.path)); + const contentDigest = pluginPackageContentTreeDigest( + resourceEntries.map((entry) => ({ + path: entry.path, + bytes: entry.body.byteLength, + digest: sha256(entry.body), + })), + ); const artifact = canonicalTar([ { path: 'package.json', body: Buffer.from(serializePluginPackageManifest(manifest), 'utf8'), }, + ...resourceEntries, ]); const packageConfig = jsonBytes({ schema: 'qinglong/plugin-package-oci-config@v1', @@ -201,6 +228,142 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) { }; const packageManifestBytes = jsonBytes(packageManifestValue); const packageManifestDigest = sha256(packageManifestBytes); + return Object.freeze({ + manifest, + source: Object.freeze({ + kind: 'oci', + locator: `oci://${registry}/${REPOSITORY}@sha256:${packageManifestDigest}`, + artifactDigest, + artifactBytes: artifact.byteLength, + contentDigest, + }), + packageConfig, + packageConfigDigest, + packageManifestBytes, + packageManifestDigest, + artifact, + artifactDigest, + mediaTypes: Object.freeze({ + packageConfig: PLUGIN_PACKAGE_OCI_CONFIG_MEDIA_TYPE, + signatureArtifact: PLUGIN_PACKAGE_OCI_SIGNATURE_ARTIFACT_TYPE, + signatureConfig: PLUGIN_PACKAGE_OCI_SIGNATURE_CONFIG_MEDIA_TYPE, + }), + }); +} + +function signedPackageRoutes(material, lock, privateKey) { + const { + PLUGIN_PACKAGE_SIGNATURE_SCHEMA, + pluginPackagePublisherSignaturePayload, + } = ql3Require('@qinglong/runtime-core/plugin-package-bundle'); + const signature = { + schema: PLUGIN_PACKAGE_SIGNATURE_SCHEMA, + publisher: PUBLISHER, + keyId: KEY_ID, + signature: sign( + null, + pluginPackagePublisherSignaturePayload(lock, PUBLISHER, KEY_ID), + privateKey, + ).toString('base64url'), + }; + const signatureConfig = jsonBytes(signature); + const signatureConfigDigest = sha256(signatureConfig); + const signatureManifestValue = { + schemaVersion: 2, + mediaType: OCI_MANIFEST, + artifactType: material.mediaTypes.signatureArtifact, + config: { + mediaType: material.mediaTypes.signatureConfig, + digest: `sha256:${signatureConfigDigest}`, + size: signatureConfig.byteLength, + }, + layers: [], + subject: { + mediaType: OCI_MANIFEST, + digest: `sha256:${material.packageManifestDigest}`, + size: material.packageManifestBytes.byteLength, + }, + }; + const signatureManifestBytes = jsonBytes(signatureManifestValue); + const signatureManifestDigest = sha256(signatureManifestBytes); + const referrers = jsonBytes({ + schemaVersion: 2, + mediaType: OCI_INDEX, + manifests: [ + { + mediaType: OCI_MANIFEST, + digest: `sha256:${signatureManifestDigest}`, + size: signatureManifestBytes.byteLength, + artifactType: material.mediaTypes.signatureArtifact, + annotations: { + 'qinglong.io/plugin-package-lock-digest': lock.lockDigest, + }, + }, + ], + }); + const prefix = `/v2/${REPOSITORY}`; + return Object.freeze([ + route( + `${prefix}/manifests/sha256:${material.packageManifestDigest}`, + OCI_MANIFEST, + material.packageManifestBytes, + ), + route( + `${prefix}/blobs/sha256:${material.packageConfigDigest}`, + material.mediaTypes.packageConfig, + material.packageConfig, + ), + route( + `${prefix}/referrers/sha256:${ + material.packageManifestDigest + }?artifactType=${encodeURIComponent( + material.mediaTypes.signatureArtifact, + )}`, + OCI_INDEX, + referrers, + ), + route( + `${prefix}/manifests/sha256:${signatureManifestDigest}`, + OCI_MANIFEST, + signatureManifestBytes, + ), + route( + `${prefix}/blobs/sha256:${signatureConfigDigest}`, + material.mediaTypes.signatureConfig, + signatureConfig, + ), + route( + `${prefix}/blobs/sha256:${material.artifactDigest}`, + BUNDLE, + material.artifact, + ), + ]); +} + +function createFixture({ registry, architecture, createdAtMs = Date.now() }) { + if ( + typeof registry !== 'string' || + !/^[a-z0-9](?:[-a-z0-9.]{0,251}[a-z0-9])?$/.test(registry) || + !['amd64', 'arm64'].includes(architecture) || + !Number.isSafeInteger(createdAtMs) || + createdAtMs < 1 + ) { + throw new TypeError('Plugin Package E2E fixture options are invalid'); + } + const { PluginPackagePublisherTrustRegistry } = ql3Require( + '@qinglong/runtime-core/plugin-package-bundle', + ); + const { planPluginPackageInstall } = ql3Require( + '@qinglong/runtime-core/plugin-package', + ); + const { + createPluginPackageLock, + pluginPackageInstallActionDigest, + pluginPackageInstallPlanDigest, + } = ql3Require('@qinglong/runtime-core/plugin-package-install'); + const { createPluginPackageResourceGenerationFromReferences } = ql3Require( + '@qinglong/runtime-core/plugin-package-resource-generation', + ); const environment = { qinglongVersion: '3.0.0-alpha.0', architecture, @@ -209,33 +372,28 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) { availableMemoryBytes: 128 * 1024 * 1024, availableDiskBytes: 256 * 1024 * 1024, }; - const plan = planPluginPackageInstall(manifest, environment); - const source = { - kind: 'oci', - locator: `oci://${registry}/${REPOSITORY}@sha256:${packageManifestDigest}`, - artifactDigest, - artifactBytes: artifact.byteLength, - contentDigest: pluginPackageContentTreeDigest([]), - }; - const action = { - lockId: 'lock-plugin-recovery-e2e', + const initialManifest = pluginManifest(architecture, '1.0.0'); + const initialMaterial = packageMaterial(registry, initialManifest, {}); + const initialPlan = planPluginPackageInstall(initialManifest, environment); + const initialAction = { + lockId: 'lock-plugin-recovery-e2e-initial', projectId: 'default', - manifest, - plan, + manifest: initialManifest, + plan: initialPlan, environment, - source, + source: initialMaterial.source, architecture, deploymentProfile: 'cluster-control', targetGeneration: 1, }; - const lock = createPluginPackageLock({ - ...action, + const initialLock = createPluginPackageLock({ + ...initialAction, approval: { - requestId: 'approval-plugin-recovery-e2e', + requestId: 'approval-plugin-recovery-e2e-initial', requestVersion: 1, - dispatchId: 'dispatch-plugin-recovery-e2e', - actionDigest: pluginPackageInstallActionDigest(action), - previewDigest: pluginPackageInstallPlanDigest(plan), + dispatchId: 'dispatch-plugin-recovery-e2e-initial', + actionDigest: pluginPackageInstallActionDigest(initialAction), + previewDigest: pluginPackageInstallPlanDigest(initialPlan), approvedBy: { type: 'user', id: 'e2e-owner' }, approvedAtMs: createdAtMs - 1, expiresAtMs: createdAtMs + 60 * 60 * 1000, @@ -243,6 +401,46 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) { }, createdAtMs, }); + const upgradeCreatedAtMs = createdAtMs + 10; + const upgradeManifest = pluginManifest(architecture, '2.0.0', true); + const upgradeMaterial = packageMaterial( + registry, + upgradeManifest, + invalidUpgradeResources(), + ); + const upgradePlan = planPluginPackageInstall( + upgradeManifest, + environment, + initialManifest, + ); + const upgradeAction = { + lockId: 'lock-plugin-recovery-e2e-upgrade', + projectId: 'default', + manifest: upgradeManifest, + plan: upgradePlan, + environment, + previousManifest: initialManifest, + source: upgradeMaterial.source, + architecture, + deploymentProfile: 'cluster-control', + targetGeneration: 2, + previousLockDigest: initialLock.lockDigest, + }; + const upgradeLock = createPluginPackageLock({ + ...upgradeAction, + approval: { + requestId: 'approval-plugin-recovery-e2e-upgrade', + requestVersion: 1, + dispatchId: 'dispatch-plugin-recovery-e2e-upgrade', + actionDigest: pluginPackageInstallActionDigest(upgradeAction), + previewDigest: pluginPackageInstallPlanDigest(upgradePlan), + approvedBy: { type: 'user', id: 'e2e-owner' }, + approvedAtMs: upgradeCreatedAtMs - 1, + expiresAtMs: upgradeCreatedAtMs + 60 * 60 * 1000, + fence: { projectVersion: 1, bindingVersion: 1 }, + }, + createdAtMs: upgradeCreatedAtMs, + }); const { publicKey, privateKey } = generateKeyPairSync('ed25519'); const publicKeyPem = publicKey.export({ format: 'pem', type: 'spki' }); const trust = { @@ -258,94 +456,58 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) { ], }; new PluginPackagePublisherTrustRegistry(trust.keys); - const signature = { - schema: PLUGIN_PACKAGE_SIGNATURE_SCHEMA, - publisher: PUBLISHER, - keyId: KEY_ID, - signature: sign( - null, - pluginPackagePublisherSignaturePayload(lock, PUBLISHER, KEY_ID), - privateKey, - ).toString('base64url'), - }; - const signatureConfig = jsonBytes(signature); - const signatureConfigDigest = sha256(signatureConfig); - const signatureManifestValue = { - schemaVersion: 2, - mediaType: OCI_MANIFEST, - artifactType: PLUGIN_PACKAGE_OCI_SIGNATURE_ARTIFACT_TYPE, - config: { - mediaType: PLUGIN_PACKAGE_OCI_SIGNATURE_CONFIG_MEDIA_TYPE, - digest: `sha256:${signatureConfigDigest}`, - size: signatureConfig.byteLength, - }, - layers: [], - subject: { - mediaType: OCI_MANIFEST, - digest: `sha256:${packageManifestDigest}`, - size: packageManifestBytes.byteLength, - }, - }; - const signatureManifestBytes = jsonBytes(signatureManifestValue); - const signatureManifestDigest = sha256(signatureManifestBytes); - const referrers = jsonBytes({ - schemaVersion: 2, - mediaType: OCI_INDEX, - manifests: [ - { - mediaType: OCI_MANIFEST, - digest: `sha256:${signatureManifestDigest}`, - size: signatureManifestBytes.byteLength, - artifactType: PLUGIN_PACKAGE_OCI_SIGNATURE_ARTIFACT_TYPE, - annotations: { - 'qinglong.io/plugin-package-lock-digest': lock.lockDigest, - }, - }, - ], + const initialRoutes = signedPackageRoutes( + initialMaterial, + initialLock, + privateKey, + ); + const upgradeRoutes = signedPackageRoutes( + upgradeMaterial, + upgradeLock, + privateKey, + ); + const initial = Object.freeze({ + installationId: 'install-plugin-recovery-e2e-initial', + manifest: initialManifest, + lock: initialLock, + generation: createPluginPackageResourceGenerationFromReferences({ + installationId: 'install-plugin-recovery-e2e-initial', + projectId: initialLock.projectId, + packageName: initialLock.packageName, + lockDigest: initialLock.lockDigest, + generation: initialLock.targetGeneration, + previousActiveLockDigest: null, + contentDigest: initialLock.source.contentDigest, + resources: initialLock.resources, + }), + routes: initialRoutes, + }); + const upgrade = Object.freeze({ + installationId: 'install-plugin-recovery-e2e-upgrade', + manifest: upgradeManifest, + lock: upgradeLock, + generation: createPluginPackageResourceGenerationFromReferences({ + installationId: 'install-plugin-recovery-e2e-upgrade', + projectId: upgradeLock.projectId, + packageName: upgradeLock.packageName, + lockDigest: upgradeLock.lockDigest, + generation: upgradeLock.targetGeneration, + previousActiveLockDigest: initialLock.lockDigest, + contentDigest: upgradeLock.source.contentDigest, + resources: upgradeLock.resources, + }), + routes: upgradeRoutes, }); - const prefix = `/v2/${REPOSITORY}`; - const routes = [ - route( - `${prefix}/manifests/sha256:${packageManifestDigest}`, - OCI_MANIFEST, - packageManifestBytes, - ), - route( - `${prefix}/blobs/sha256:${packageConfigDigest}`, - PLUGIN_PACKAGE_OCI_CONFIG_MEDIA_TYPE, - packageConfig, - ), - route( - `${prefix}/referrers/sha256:${packageManifestDigest}?artifactType=${encodeURIComponent( - PLUGIN_PACKAGE_OCI_SIGNATURE_ARTIFACT_TYPE, - )}`, - OCI_INDEX, - referrers, - ), - route( - `${prefix}/manifests/sha256:${signatureManifestDigest}`, - OCI_MANIFEST, - signatureManifestBytes, - ), - route( - `${prefix}/blobs/sha256:${signatureConfigDigest}`, - PLUGIN_PACKAGE_OCI_SIGNATURE_CONFIG_MEDIA_TYPE, - signatureConfig, - ), - route( - `${prefix}/blobs/sha256:${artifactDigest}`, - BUNDLE, - artifact, - ), - ]; return Object.freeze({ schema: FIXTURE_SCHEMA, registry, repository: REPOSITORY, architecture, - lock, + lock: initialLock, + initial, + upgrade, trust, - routes, + routes: Object.freeze([...initialRoutes, ...upgradeRoutes]), }); } @@ -355,7 +517,10 @@ function readFixture(filePath) { !value || value.schema !== FIXTURE_SCHEMA || !Array.isArray(value.routes) || - !value.lock || + !value.initial?.lock || + !value.initial?.generation || + !value.upgrade?.lock || + !value.upgrade?.generation || !value.trust ) { throw new Error('Plugin Package E2E fixture is invalid'); @@ -459,17 +624,27 @@ async function runSeed() { assertPostgresPackageExecutorSchemaReady, createPostgresDatabaseOpener, loadPostgresConnectionEnvironment, + PostgresPluginPackageSecretBindingTransitionRepository, } = ql3Require('@qinglong/cluster-postgres/package-executor'); - const { - PostgresPluginPackageInstallRepository, - } = ql3Require('@qinglong/cluster-postgres/plugin-package-install'); + const { PostgresPluginPackageInstallRepository } = ql3Require( + '@qinglong/cluster-postgres/plugin-package-install', + ); const { createPluginPackageInstall, normalizePluginPackageLock, pluginPackageInstallCreate, } = ql3Require('@qinglong/runtime-core/plugin-package-install'); + const { createPluginPackageSecretBindingTarget } = ql3Require( + '@qinglong/runtime-core/plugin-package-secret-binding', + ); + const { createPluginPackageSecretBindingTransitionPlan } = ql3Require( + '@qinglong/runtime-core/plugin-package-secret-binding-transition-plan', + ); const fixture = readFixture(process.env.QL3_E2E_FIXTURE_FILE); - const lock = normalizePluginPackageLock(fixture.lock); + const mode = process.env.QL3_E2E_MODE; + if (!['seed-initial', 'seed-upgrade', 'commit-transition'].includes(mode)) { + throw new Error('Plugin Package E2E seed mode is invalid'); + } const connection = loadPostgresConnectionEnvironment(process.env, { host: 'QL3_E2E_POSTGRES_HOST', port: 'QL3_E2E_POSTGRES_PORT', @@ -489,21 +664,72 @@ async function runSeed() { })(); try { await assertPostgresPackageExecutorSchemaReady(database.pool); + if (mode === 'commit-transition') { + const plannedAtMs = Date.now(); + const transitionPlan = createPluginPackageSecretBindingTransitionPlan({ + previousTarget: createPluginPackageSecretBindingTarget( + fixture.initial.generation, + fixture.initial.manifest, + ), + previousBinding: null, + previousAttemptGeneration: 1, + nextGeneration: fixture.upgrade.generation, + nextManifest: fixture.upgrade.manifest, + assignments: [], + plannedAtMs, + }); + const result = + await new PostgresPluginPackageSecretBindingTransitionRepository( + database.pool, + ).apply({ + transitionPlan, + evidenceDigest: transitionPlan.transitionDigest, + committedAtMs: plannedAtMs + 1, + }); + process.stdout.write( + `${JSON.stringify({ + schema: 'qinglong/plugin-package-recovery-e2e-transition-result@v1', + event: 'transition_completed', + status: result.status, + generationDigest: transitionPlan.nextTarget.generationDigest, + transitionDigest: transitionPlan.transitionDigest, + bindingDigest: result.receipt.bindingDigest, + receiptDigest: result.receipt.receiptDigest, + })}\n`, + ); + return; + } + const selected = + mode === 'seed-initial' ? fixture.initial : fixture.upgrade; + const lock = normalizePluginPackageLock(selected.lock); const repository = new PostgresPluginPackageInstallRepository( database.pool, ); + const previous = await repository.find(lock.projectId, lock.packageName); + if ( + (mode === 'seed-initial' && previous !== null) || + (mode === 'seed-upgrade' && + (previous?.state !== 'active' || + previous.installationId !== fixture.initial.installationId || + previous.lockDigest !== fixture.initial.lock.lockDigest)) + ) { + throw new Error('Plugin Package E2E previous install head is invalid'); + } const record = createPluginPackageInstall(lock, { - installationId: 'install-plugin-recovery-e2e', - mutationId: 'mutation-plugin-recovery-e2e-create', + installationId: selected.installationId, + mutationId: `mutation-plugin-recovery-e2e-${ + mode === 'seed-initial' ? 'initial' : 'upgrade' + }-create`, occurredAtMs: lock.createdAtMs + 1, }); const result = await repository.create( - pluginPackageInstallCreate(lock, record, null), + pluginPackageInstallCreate(lock, record, previous), ); process.stdout.write( `${JSON.stringify({ schema: 'qinglong/plugin-package-recovery-e2e-seed-result@v1', event: 'seed_completed', + phase: mode === 'seed-initial' ? 'initial' : 'upgrade', status: result.status, state: result.record.state, installationId: result.record.installationId, @@ -521,11 +747,17 @@ async function main() { await runRegistry(); return; } - if (process.env.QL3_E2E_MODE === 'seed') { + if ( + ['seed-initial', 'seed-upgrade', 'commit-transition'].includes( + process.env.QL3_E2E_MODE, + ) + ) { await runSeed(); return; } - throw new Error('QL3_E2E_MODE must be registry or seed'); + throw new Error( + 'QL3_E2E_MODE must be registry, seed-initial, seed-upgrade or commit-transition', + ); } if (require.main === module) { diff --git a/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs b/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs index 49b9dfaa..bc9fbbda 100644 --- a/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs +++ b/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs @@ -35,7 +35,13 @@ const POSTGRES_REPOSITORY_DIGEST = `postgres@${POSTGRES_INDEX_DIGEST}`; const DEFAULT_ADMIN_IMAGE = 'qinglong3-cluster-admin:ql3-plugin-recovery-e2e'; const DEFAULT_CONTROL_IMAGE = 'qinglong3-cluster-control:ql3-plugin-recovery-e2e'; -const REPORT_SCHEMA = 'qinglong/plugin-package-recovery-e2e-live-contract@v1'; +const REPORT_SCHEMA = 'qinglong/plugin-package-recovery-e2e-live-contract@v2'; +const INITIAL_SEED_JOB = 'ql3-plugin-package-e2e-seed-initial'; +const INITIAL_RECOVERY_JOB = 'ql3-plugin-package-recovery-initial'; +const UPGRADE_SEED_JOB = 'ql3-plugin-package-e2e-seed-upgrade'; +const UPGRADE_STAGE_JOB = 'ql3-plugin-package-recovery-stage-upgrade'; +const TRANSITION_JOB = 'ql3-plugin-package-e2e-transition'; +const UPGRADE_REJECTION_JOB = 'ql3-plugin-package-recovery-reject-upgrade'; const SAFE_CLUSTER = /^ql3-plugin-recovery-e2e(?:-[a-z0-9](?:[-a-z0-9]{0,24}[a-z0-9])?)?$/; @@ -727,11 +733,11 @@ function migrationJob() { return job; } -function seedJob() { +function seedJob(name, mode) { return { apiVersion: 'batch/v1', kind: 'Job', - metadata: { name: 'ql3-plugin-package-e2e-seed', namespace: NAMESPACE }, + metadata: { name, namespace: NAMESPACE }, spec: { backoffLimit: 0, activeDeadlineSeconds: 300, @@ -754,7 +760,7 @@ function seedJob() { imagePullPolicy: 'Never', command: ['node', '/opt/ql3-e2e/fixture.cjs'], env: [ - { name: 'QL3_E2E_MODE', value: 'seed' }, + { name: 'QL3_E2E_MODE', value: mode }, { name: 'QL3_E2E_FIXTURE_FILE', value: '/opt/ql3-e2e/fixture.json', @@ -811,7 +817,7 @@ function seedJob() { }; } -function recoveryResources(fixture) { +function recoveryResources(fixture, jobName) { const rbac = readYamlDocuments( path.join( ROOT, @@ -830,6 +836,7 @@ function recoveryResources(fixture) { 'deploy/kubernetes/ql3-cluster/operations/plugin-package-recovery/base/recover-job.yaml', ), ); + job.metadata.name = jobName; job.metadata.namespace = NAMESPACE; job.spec.activeDeadlineSeconds = 300; const container = job.spec.template.spec.containers[0]; @@ -956,7 +963,14 @@ function recoveryResources(fixture) { ]; } -function waitForJob(name, timeoutMs = 5 * 60 * 1000) { +function waitForJob( + name, + expectedStatus = 'complete', + timeoutMs = 5 * 60 * 1000, +) { + if (!['complete', 'failed'].includes(expectedStatus)) { + fail('expected Job status is invalid'); + } const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const job = kubectlJson(['-n', NAMESPACE, 'get', 'job', name]); @@ -964,11 +978,17 @@ function waitForJob(name, timeoutMs = 5 * 60 * 1000) { (condition) => condition.type === 'Complete' && condition.status === 'True', ); - if (complete) return job; + if (complete) { + if (expectedStatus !== 'complete') { + fail(`${name} completed but failure was required`); + } + return job; + } const failed = job.status?.conditions?.find( (condition) => condition.type === 'Failed' && condition.status === 'True', ); if (failed) { + if (expectedStatus === 'failed') return job; const logs = kubectl( ['-n', NAMESPACE, 'logs', `job/${name}`, '--all-containers=true'], { capture: true, quiet: true, allowFailure: true }, @@ -1039,7 +1059,72 @@ function canI(verb, resource) { return result.stdout === 'yes'; } -function databaseEvidence(lockDigest) { +function upgradeStageEvidence(fixture, transitionReceiptCount) { + const generationDigest = fixture.upgrade.generation.generationDigest; + assert.match(generationDigest, /^[0-9a-f]{64}$/); + const sql = ` +SELECT json_build_object( + 'state', ( + SELECT state FROM ql3.plugin_package_installs + WHERE installation_id = '${fixture.upgrade.installationId}' + ), + 'previousActiveLockDigest', ( + SELECT previous_active_lock_digest FROM ql3.plugin_package_installs + WHERE installation_id = '${fixture.upgrade.installationId}' + ), + 'activeLockDigest', ( + SELECT active_lock_digest FROM ql3.plugin_package_installs + WHERE installation_id = '${fixture.upgrade.installationId}' + ), + 'mutationCount', ( + SELECT count(*) FROM ql3.plugin_package_install_mutations + WHERE installation_id = '${fixture.upgrade.installationId}' + ), + 'transitionReceiptCount', ( + SELECT count(*) + FROM ql3.plugin_package_secret_binding_transition_receipts + WHERE generation_digest = '${generationDigest}' + ), + 'candidateRevisionCount', ( + SELECT count(*) FROM ql3.plugin_package_materialized_revisions + WHERE generation_digest = '${generationDigest}' + ) +)::text; +`.trim(); + const output = kubectl( + [ + '-n', + NAMESPACE, + 'exec', + POSTGRES_NAME, + '--', + 'psql', + '--username', + 'postgres', + '--dbname', + 'qinglong', + '--tuples-only', + '--no-align', + '--command', + sql, + ], + { capture: true, quiet: true }, + ).stdout; + const value = JSON.parse(output); + assert.equal(value.state, 'staged'); + assert.equal(value.previousActiveLockDigest, fixture.initial.lock.lockDigest); + assert.equal(value.activeLockDigest, fixture.initial.lock.lockDigest); + assert.equal(value.mutationCount, 2); + assert.equal(value.transitionReceiptCount, transitionReceiptCount); + assert.equal(value.candidateRevisionCount, 0); + return value; +} + +function databaseEvidence(fixture) { + const initialGenerationDigest = fixture.initial.generation.generationDigest; + const upgradeGenerationDigest = fixture.upgrade.generation.generationDigest; + assert.match(initialGenerationDigest, /^[0-9a-f]{64}$/); + assert.match(upgradeGenerationDigest, /^[0-9a-f]{64}$/); const sql = ` SELECT json_build_object( 'migrationCount', (SELECT count(*) FROM ql3.schema_migrations), @@ -1048,25 +1133,63 @@ SELECT json_build_object( FROM ql3.schema_capabilities WHERE contract_name = 'control-core' ), - 'state', ( + 'initialState', ( SELECT state FROM ql3.plugin_package_installs - WHERE installation_id = 'install-plugin-recovery-e2e' + WHERE installation_id = '${fixture.initial.installationId}' ), - 'lockDigest', ( - SELECT lock_digest - FROM ql3.plugin_package_installs - WHERE installation_id = 'install-plugin-recovery-e2e' - ), - 'activeLockDigest', ( + 'initialActiveLockDigest', ( SELECT active_lock_digest FROM ql3.plugin_package_installs - WHERE installation_id = 'install-plugin-recovery-e2e' + WHERE installation_id = '${fixture.initial.installationId}' ), - 'mutationCount', ( + 'upgradeState', ( + SELECT state + FROM ql3.plugin_package_installs + WHERE installation_id = '${fixture.upgrade.installationId}' + ), + 'upgradePreviousActiveLockDigest', ( + SELECT previous_active_lock_digest + FROM ql3.plugin_package_installs + WHERE installation_id = '${fixture.upgrade.installationId}' + ), + 'upgradeActiveLockDigest', ( + SELECT active_lock_digest + FROM ql3.plugin_package_installs + WHERE installation_id = '${fixture.upgrade.installationId}' + ), + 'upgradeFailureReason', ( + SELECT record_json #>> '{failure,reason}' + FROM ql3.plugin_package_installs + WHERE installation_id = '${fixture.upgrade.installationId}' + ), + 'initialMutationCount', ( SELECT count(*) FROM ql3.plugin_package_install_mutations - WHERE installation_id = 'install-plugin-recovery-e2e' + WHERE installation_id = '${fixture.initial.installationId}' + ), + 'upgradeMutationCount', ( + SELECT count(*) + FROM ql3.plugin_package_install_mutations + WHERE installation_id = '${fixture.upgrade.installationId}' + ), + 'headInstallationId', ( + SELECT installation_id + FROM ql3.plugin_package_install_heads + WHERE project_id = 'default' AND package_name = 'e2e-monitor' + ), + 'transitionReceiptCount', ( + SELECT count(*) + FROM ql3.plugin_package_secret_binding_transition_receipts + WHERE generation_digest = '${upgradeGenerationDigest}' + ), + 'initialRevisionCount', ( + SELECT count(*) FROM ql3.plugin_package_materialized_revisions + WHERE generation_digest = '${initialGenerationDigest}' + ), + 'upgradeRevisionCount', ( + SELECT count(*) FROM ql3.plugin_package_materialized_revisions + WHERE generation_digest = '${upgradeGenerationDigest}' ), 'recoverableCount', ( SELECT count(*) @@ -1095,17 +1218,28 @@ SELECT json_build_object( { capture: true, quiet: true }, ).stdout; const value = JSON.parse(output); - assert.equal(value.migrationCount, 22); - assert.equal(value.capabilityVersion, 21); - assert.equal(value.state, 'active'); - assert.equal(value.lockDigest, lockDigest); - assert.equal(value.activeLockDigest, lockDigest); - assert.equal(value.mutationCount, 4); + assert.equal(value.migrationCount, 65); + assert.equal(value.capabilityVersion, 64); + assert.equal(value.initialState, 'active'); + assert.equal(value.initialActiveLockDigest, fixture.initial.lock.lockDigest); + assert.equal(value.upgradeState, 'failed'); + assert.equal( + value.upgradePreviousActiveLockDigest, + fixture.initial.lock.lockDigest, + ); + assert.equal(value.upgradeActiveLockDigest, fixture.initial.lock.lockDigest); + assert.equal(value.upgradeFailureReason, 'activation_fact_conflict'); + assert.equal(value.initialMutationCount, 4); + assert.equal(value.upgradeMutationCount, 3); + assert.equal(value.headInstallationId, fixture.upgrade.installationId); + assert.equal(value.transitionReceiptCount, 1); + assert.equal(value.initialRevisionCount, 1); + assert.equal(value.upgradeRevisionCount, 0); assert.equal(value.recoverableCount, 0); return value; } -function activePointerEvidence(lockDigest) { +function activePointerEvidence(fixture) { const values = kubectlJson([ '-n', NAMESPACE, @@ -1117,13 +1251,14 @@ function activePointerEvidence(lockDigest) { assert.equal(values.items.length, 1); const configMap = values.items[0]; const pointer = JSON.parse(configMap.data['active.json']); - assert.equal(pointer.intent.installationId, 'install-plugin-recovery-e2e'); - assert.equal(pointer.intent.lockDigest, lockDigest); + assert.equal(pointer.intent.installationId, fixture.initial.installationId); + assert.equal(pointer.intent.lockDigest, fixture.initial.lock.lockDigest); assert.equal(pointer.receipt.generation, 1); return Object.freeze({ name: configMap.metadata.name, uid: configMap.metadata.uid, resourceVersion: configMap.metadata.resourceVersion, + activeJson: configMap.data['active.json'], intentDigest: pointer.intent.intentDigest, activationRef: pointer.receipt.activationRef, }); @@ -1141,10 +1276,17 @@ function registryEvidence(fixture) { .map((line) => JSON.parse(line)) .filter((value) => value.schema === REGISTRY_EVENT_SCHEMA); const packageRequests = events.filter((event) => event.path !== '/v2/'); - assert.equal(packageRequests.length, fixture.routes.length); + const expectedPaths = [ + ...fixture.initial.routes.map((routeValue) => routeValue.path), + ...fixture.upgrade.routes.flatMap((routeValue) => [ + routeValue.path, + routeValue.path, + ]), + ].sort(); + assert.equal(packageRequests.length, expectedPaths.length); assert.deepEqual( packageRequests.map((event) => event.path).sort(), - fixture.routes.map((routeValue) => routeValue.path).sort(), + expectedPaths, ); assert.ok(packageRequests.every((event) => event.status === 200)); assert.ok(packageRequests.every((event) => event.authenticated === true)); @@ -1154,6 +1296,8 @@ function registryEvidence(fixture) { authenticatedRequestCount: packageRequests.length, requestCount: packageRequests.length, uniquePaths: new Set(packageRequests.map((event) => event.path)).size, + initialRequestCount: fixture.initial.routes.length, + upgradeRequestCount: fixture.upgrade.routes.length * 2, redirects: 0, }); } @@ -1572,36 +1716,109 @@ async function main() { (value) => value.event === 'migration_completed', ); - apply(seedJob(), 'persist one durable queued Plugin Package installation'); - waitForJob('ql3-plugin-package-e2e-seed'); - const seed = lastJsonLine( - jobLog('ql3-plugin-package-e2e-seed'), + apply( + seedJob(INITIAL_SEED_JOB, 'seed-initial'), + 'persist initial durable queued Plugin Package installation', + ); + waitForJob(INITIAL_SEED_JOB); + const initialSeed = lastJsonLine( + jobLog(INITIAL_SEED_JOB), (value) => value.event === 'seed_completed', ); - assert.equal(seed.status, 'created'); - assert.equal(seed.state, 'queued'); - assert.equal(seed.lockDigest, fixture.lock.lockDigest); + assert.equal(initialSeed.phase, 'initial'); + assert.equal(initialSeed.status, 'created'); + assert.equal(initialSeed.state, 'queued'); + assert.equal(initialSeed.lockDigest, fixture.initial.lock.lockDigest); - for (const resource of recoveryResources(fixture)) { - apply(resource, `apply recovery ${resource.kind}`); + for (const resource of recoveryResources(fixture, INITIAL_RECOVERY_JOB)) { + apply(resource, `apply initial recovery ${resource.kind}`); } - const recovered = waitForJob('ql3-plugin-package-recovery'); - const recoveryLog = jobLog('ql3-plugin-package-recovery'); - const completed = lastJsonLine( - recoveryLog, + const initialRecovered = waitForJob(INITIAL_RECOVERY_JOB); + const initialCompleted = lastJsonLine( + jobLog(INITIAL_RECOVERY_JOB), (value) => value.event === 'recovery_completed', ); - assert.equal(completed.recovery.safeToAdmit, true); - assert.equal(completed.recovery.remaining, false); - assert.equal(completed.recovery.manualRequired, 0); + assert.equal(initialCompleted.recovery.safeToAdmit, true); + assert.equal(initialCompleted.recovery.remaining, false); + assert.equal(initialCompleted.recovery.manualRequired, 0); + const pointerBeforeUpgrade = activePointerEvidence(fixture); - const database = databaseEvidence(fixture.lock.lockDigest); - const pointer = activePointerEvidence(fixture.lock.lockDigest); + apply( + seedJob(UPGRADE_SEED_JOB, 'seed-upgrade'), + 'persist invalid durable queued Plugin Package upgrade', + ); + waitForJob(UPGRADE_SEED_JOB); + const upgradeSeed = lastJsonLine( + jobLog(UPGRADE_SEED_JOB), + (value) => value.event === 'seed_completed', + ); + assert.equal(upgradeSeed.phase, 'upgrade'); + assert.equal(upgradeSeed.status, 'created'); + assert.equal(upgradeSeed.state, 'queued'); + assert.equal(upgradeSeed.lockDigest, fixture.upgrade.lock.lockDigest); + + for (const resource of recoveryResources(fixture, UPGRADE_STAGE_JOB)) { + apply(resource, `apply upgrade staging recovery ${resource.kind}`); + } + const stagedUpgrade = waitForJob(UPGRADE_STAGE_JOB, 'failed'); + const stagedFailureCondition = stagedUpgrade.status.conditions.find( + (condition) => condition.type === 'Failed' && condition.status === 'True', + ); + assert.ok(stagedFailureCondition?.lastTransitionTime); + const stageFailure = lastJsonLine( + jobLog(UPGRADE_STAGE_JOB), + (value) => value.event === 'recovery_failed', + ); + assert.equal( + stageFailure.name, + 'ClusterPluginPackageRecoveryRequiredError', + ); + const stagedDatabase = upgradeStageEvidence(fixture, 0); + assert.deepEqual(activePointerEvidence(fixture), pointerBeforeUpgrade); + + apply( + seedJob(TRANSITION_JOB, 'commit-transition'), + 'commit durable no-secret binding transition receipt', + ); + const committedTransition = waitForJob(TRANSITION_JOB); + const transition = lastJsonLine( + jobLog(TRANSITION_JOB), + (value) => value.event === 'transition_completed', + ); + assert.equal(transition.status, 'created'); + assert.equal( + transition.generationDigest, + fixture.upgrade.generation.generationDigest, + ); + assert.equal(transition.bindingDigest, null); + upgradeStageEvidence(fixture, 1); + + for (const resource of recoveryResources(fixture, UPGRADE_REJECTION_JOB)) { + apply(resource, `apply upgrade rejection recovery ${resource.kind}`); + } + const rejectedUpgrade = waitForJob(UPGRADE_REJECTION_JOB); + const rejectionCompleted = lastJsonLine( + jobLog(UPGRADE_REJECTION_JOB), + (value) => value.event === 'recovery_completed', + ); + assert.equal(rejectionCompleted.recovery.safeToAdmit, true); + assert.equal(rejectionCompleted.recovery.remaining, false); + assert.equal(rejectionCompleted.recovery.manualRequired, 0); + + const database = databaseEvidence(fixture); + const pointerAfterRejection = activePointerEvidence(fixture); + assert.deepEqual(pointerAfterRejection, pointerBeforeUpgrade); const oci = registryEvidence(fixture); const rbac = recoveryRbacEvidence(); assertRuntimeCannotReadPluginAuthority(); - const runtime = applyRuntimeAfterRecovery(recovered, migrated, secrets); - const recoveryImageId = jobImageId('ql3-plugin-package-recovery'); + const runtime = applyRuntimeAfterRecovery( + rejectedUpgrade, + migrated, + secrets, + ); + const initialRecoveryImageId = jobImageId(INITIAL_RECOVERY_JOB); + const stageRecoveryImageId = jobImageId(UPGRADE_STAGE_JOB); + const rejectionRecoveryImageId = jobImageId(UPGRADE_REJECTION_JOB); const migrationImageId = jobImageId('ql3-cluster-migration'); const postgresPod = kubectlJson([ '-n', @@ -1621,21 +1838,40 @@ async function main() { controlBuildId: imageId(CONTROL_IMAGE), postgresRepositoryDigest: POSTGRES_REPOSITORY_DIGEST, migrationImageId, - recoveryImageId, + initialRecoveryImageId, + stageRecoveryImageId, + rejectionRecoveryImageId, postgresImageId: postgresPod.status.containerStatuses[0].imageID, }), ordering: Object.freeze({ migrationJobUid: migrated.metadata.uid, migrationCompletedAt: migrated.status.completionTime, - recoveryJobUid: recovered.metadata.uid, - recoveryCompletedAt: recovered.status.completionTime, + initialRecoveryJobUid: initialRecovered.metadata.uid, + initialRecoveryCompletedAt: initialRecovered.status.completionTime, + upgradeStageJobUid: stagedUpgrade.metadata.uid, + upgradeStageFailedAt: stagedFailureCondition.lastTransitionTime, + transitionJobUid: committedTransition.metadata.uid, + transitionCompletedAt: committedTransition.status.completionTime, + rejectionRecoveryJobUid: rejectedUpgrade.metadata.uid, + rejectionRecoveryCompletedAt: rejectedUpgrade.status.completionTime, runtimeCreatedAt: runtime.creationTimestamp, runtimeBoundRecoveryJobUid: runtime.recoveryJobUid, }), + failedUpgrade: Object.freeze({ + stageFailure: Object.freeze({ + jobUid: stagedUpgrade.metadata.uid, + reason: stageFailure.name, + durableState: stagedDatabase.state, + }), + transitionReceiptDigest: transition.receiptDigest, + rejectionReason: database.upgradeFailureReason, + candidateRevisionCount: database.upgradeRevisionCount, + activePointerUnchanged: true, + }), database, oci, kubernetes: Object.freeze({ - activePointer: pointer, + activePointer: pointerAfterRejection, rbac, }), runtime, diff --git a/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs b/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs index 0d0fa776..8dd0b83d 100644 --- a/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs +++ b/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs @@ -23,10 +23,7 @@ const workflow = yaml.load( ); test('E2E live gate is opt-in and owns only one exact disposable Kind cluster', () => { - assert.match( - live, - /QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE !== '1'/, - ); + assert.match(live, /QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE !== '1'/); assert.match(live, /\^ql3-plugin-recovery-e2e/); assert.match(live, /Refusing to reuse or delete pre-existing Kind cluster/); assert.match(live, /kind\(\['delete', 'cluster', '--name', clusterName\]/); @@ -50,41 +47,59 @@ test('fixture uses a real HTTPS and content-addressed OCI Distribution surface', assert.match(live, /requestCount: packageRequests\.length/); }); -test('gate runs the committed migration and recovery binaries around a durable queued record', () => { - assert.match( - live, - /operations\/base\/migrate-job\.yaml/, - ); - assert.match( - live, - /plugin-package-recovery\/base\/recover-job\.yaml/, - ); +test('gate runs migration, healthy activation and a durable rejected upgrade', () => { + assert.match(live, /operations\/base\/migrate-job\.yaml/); + assert.match(live, /plugin-package-recovery\/base\/recover-job\.yaml/); assert.match(fixture, /PostgresPluginPackageInstallRepository/); + assert.match( + fixture, + /PostgresPluginPackageSecretBindingTransitionRepository/, + ); assert.match(fixture, /createPluginPackageInstall/); assert.match(fixture, /pluginPackageInstallCreate/); - assert.match(live, /seed\.state, 'queued'/); - assert.match(live, /value\.migrationCount, 22/); - assert.match(live, /value\.capabilityVersion, 21/); + assert.match(live, /initialSeed\.state, 'queued'/); + assert.match(live, /upgradeSeed\.state, 'queued'/); + assert.match(live, /value\.migrationCount, 65/); + assert.match(live, /value\.capabilityVersion, 64/); assert.match(live, /postgresEnvironment\(\s*'PACKAGE_EXECUTOR'/); assert.match(fixture, /assertPostgresPackageExecutorSchemaReady/); - assert.match(live, /value\.state, 'active'/); + assert.match(live, /value\.initialState, 'active'/); + assert.match(live, /value\.upgradeState, 'failed'/); + assert.match(live, /value\.upgradeFailureReason, 'activation_fact_conflict'/); + assert.match(live, /value\.upgradeRevisionCount, 0/); assert.match(live, /value\.recoverableCount, 0/); }); -test('deployment controller observes successful recovery before creating runtime', () => { - const recoveryWait = live.indexOf( - "waitForJob('ql3-plugin-package-recovery')", +test('deployment controller rejects the upgrade before creating runtime', () => { + const failedStageWait = live.indexOf( + "waitForJob(UPGRADE_STAGE_JOB, 'failed')", + ); + const transitionWait = live.indexOf('waitForJob(TRANSITION_JOB)'); + const rejectionWait = live.indexOf('waitForJob(UPGRADE_REJECTION_JOB)'); + const pointerProof = live.indexOf( + 'assert.deepEqual(pointerAfterRejection, pointerBeforeUpgrade)', ); const runtimeApply = live.indexOf( - 'applyRuntimeAfterRecovery(recovered, migrated, secrets)', + 'const runtime = applyRuntimeAfterRecovery(', + ); + assert.ok(failedStageWait > 0); + assert.ok(transitionWait > failedStageWait); + assert.ok(rejectionWait > transitionWait); + assert.ok(pointerProof > rejectionWait); + assert.ok(runtimeApply > pointerProof); + assert.match(live, /activePointerUnchanged: true/); + assert.match( + live, + /stageFailure\.name,[\s\S]*'ClusterPluginPackageRecoveryRequiredError'/, ); - assert.ok(recoveryWait > 0); - assert.ok(runtimeApply > recoveryWait); assert.match(live, /qinglong\.io\/plugin-recovery-job-uid/); assert.match(live, /qinglong\.io\/plugin-recovery-completed-at/); assert.match(live, /rollout[\s\S]*status/); assert.match(live, /availableReplicas, 2/); - assert.match(live, /new Set\(pods\.items\.map\(\(pod\) => pod\.spec\.nodeName\)\)/); + assert.match( + live, + /new Set\(pods\.items\.map\(\(pod\) => pod\.spec\.nodeName\)\)/, + ); }); test('recovery Job keeps exact ConfigMap-only RBAC and runtime cannot read install authority', () => { @@ -95,10 +110,7 @@ test('recovery Job keeps exact ConfigMap-only RBAC and runtime cannot read insta assert.match(live, /listConfigMaps: false/); assert.match(live, /deleteConfigMaps: false/); assert.match(live, /getSecrets: false/); - assert.match( - live, - /SELECT count\(\*\) FROM ql3\.plugin_package_installs/, - ); + assert.match(live, /SELECT count\(\*\) FROM ql3\.plugin_package_installs/); assert.match(live, /permission denied/i); }); @@ -112,8 +124,7 @@ test('package script and independent CI job execute the full gate', () => { assert.equal(job['timeout-minutes'], 35); assert.ok( job.steps.some( - (step) => - step.run === 'pnpm test:plugin-package-recovery-e2e:ql3', + (step) => step.run === 'pnpm test:plugin-package-recovery-e2e:ql3', ), ); assert.ok( @@ -129,14 +140,8 @@ test('private Registry evidence uses one exact Secret file and authenticated req assert.match(fixture, /request\.headers\.authorization !== authorization/); assert.match(fixture, /www-authenticate/); assert.match(fixture, /authenticated,/); - assert.match( - live, - /QL3_PLUGIN_PACKAGE_REGISTRY_CREDENTIAL_FILE/, - ); - assert.match( - live, - /qinglong\/plugin-package-registry-credentials@v1/, - ); + assert.match(live, /QL3_PLUGIN_PACKAGE_REGISTRY_CREDENTIAL_FILE/); + assert.match(live, /qinglong\/plugin-package-registry-credentials@v1/); assert.match(live, /secretName: 'ql3-e2e-registry-auth'/); assert.match(live, /defaultMode: 288/); assert.match(live, /authentication: 'exact-registry-basic'/);