diff --git a/scripts/ql3-plugin-package-recovery-e2e-fixture.cjs b/scripts/ql3-plugin-package-recovery-e2e-fixture.cjs index aa7fa2d2..8496ea74 100644 --- a/scripts/ql3-plugin-package-recovery-e2e-fixture.cjs +++ b/scripts/ql3-plugin-package-recovery-e2e-fixture.cjs @@ -340,6 +340,172 @@ function signedPackageRoutes(material, lock, privateKey) { ]); } +function createInstallAuthority(actionInput, phase, proposedAtMs, eventBase) { + const { + consumeApprovalRequest, + createApprovalRequest, + decideApprovalRequest, + } = ql3Require('@qinglong/runtime-core/approved-action'); + const { + createPluginPackageInstallProposal, + resolvePluginPackageInstallProposal, + } = ql3Require('@qinglong/runtime-core/plugin-package-proposal'); + const requester = { type: 'user', id: 'plugin-recovery-e2e-owner' }; + const reviewer = { type: 'user', id: 'plugin-recovery-e2e-reviewer' }; + const consumer = { type: 'system', id: 'plugin-recovery-e2e-dispatcher' }; + const fence = { projectVersion: 1, bindingVersion: 1 }; + const requestedAtMs = proposedAtMs + 1; + const decidedAtMs = proposedAtMs + 2; + const consumedAtMs = proposedAtMs + 3; + const lockCreatedAtMs = proposedAtMs + 4; + const expiresAtMs = proposedAtMs + 60 * 60 * 1000; + const actionRef = `plugin-package:e2e-monitor:${phase}`; + const proposal = createPluginPackageInstallProposal({ + actionRef, + actionInput, + proposedBy: requester, + proposalFence: fence, + createdAtMs: proposedAtMs, + }); + const action = { + actionRef: proposal.actionRef, + actionType: proposal.actionType, + permission: proposal.permission, + actionDigest: proposal.actionDigest, + previewDigest: proposal.previewDigest, + }; + const request = createApprovalRequest({ + id: `approval-plugin-recovery-e2e-${phase}`, + projectId: proposal.projectId, + action, + risk: 'high', + decisionMode: 'separation_of_duty', + requestedBy: requester, + requestedAtMs, + expiresAtMs, + requestFence: fence, + }); + const decision = { + expectedVersion: 1, + decisionId: `decision-plugin-recovery-e2e-${phase}`, + decision: 'approved', + reasonCode: 'reviewed', + principal: { + subject: reviewer, + authenticationId: `auth-plugin-recovery-e2e-reviewer-${phase}`, + authenticatedAtMs: proposedAtMs, + expiresAtMs, + assurance: 'multi_factor', + }, + decidedAtMs, + authorizationFence: fence, + }; + const decided = decideApprovalRequest(request, decision); + const consumption = { + expectedVersion: 2, + consumptionId: `consume-plugin-recovery-e2e-${phase}`, + dispatchId: `dispatch-plugin-recovery-e2e-${phase}`, + action, + requestedBy: requester, + consumedBy: consumer, + consumedAtMs, + authorizationFence: fence, + }; + const consumed = consumeApprovalRequest(decided, consumption); + const audit = ( + offset, + requestId, + operationId, + subject, + authenticationId, + outcome, + reasons, + occurredAtMs, + ) => ({ + eventId: `10000000-0000-4000-8000-${String( + eventBase + offset, + ).padStart(12, '0')}`, + requestId, + operationId, + projectId: proposal.projectId, + subject, + authenticationId, + outcome, + reasons, + fence, + occurredAtMs, + }); + return Object.freeze({ + requester, + reviewer, + consumer, + fence, + proposalCommand: { + proposal, + audit: audit( + 0, + proposal.actionRef, + 'plugin_package.propose', + requester, + `auth-plugin-recovery-e2e-owner-${phase}`, + 'allowed', + ['package_proposal'], + proposedAtMs, + ), + }, + requestCommand: { + request, + audit: audit( + 1, + `request-plugin-recovery-e2e-${phase}`, + 'approval.request', + requester, + `auth-plugin-recovery-e2e-owner-${phase}`, + 'approval_required', + ['package_review'], + requestedAtMs, + ), + }, + decisionCommand: { + requestId: request.id, + ...decision, + audit: audit( + 2, + `decision-plugin-recovery-e2e-${phase}`, + 'approval.decide', + reviewer, + decision.principal.authenticationId, + 'allowed', + ['role_grant'], + decidedAtMs, + ), + }, + consumptionCommand: { + requestId: request.id, + ...consumption, + audit: audit( + 3, + `consume-plugin-recovery-e2e-${phase}`, + 'approval.consume', + consumer, + `auth-plugin-recovery-e2e-dispatcher-${phase}`, + 'allowed', + ['role_grant'], + consumedAtMs, + ), + }, + dispatch: consumed.dispatch, + lock: resolvePluginPackageInstallProposal( + proposal, + consumed.dispatch, + lockCreatedAtMs, + ), + admissionAuditEventId: `10000000-0000-4000-8000-${String( + eventBase + 4, + ).padStart(12, '0')}`, + }); +} + function createFixture({ registry, architecture, createdAtMs = Date.now() }) { if ( typeof registry !== 'string' || @@ -356,11 +522,6 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) { const { planPluginPackageInstall } = ql3Require( '@qinglong/runtime-core/plugin-package', ); - const { - createPluginPackageLock, - pluginPackageInstallActionDigest, - pluginPackageInstallPlanDigest, - } = ql3Require('@qinglong/runtime-core/plugin-package-install'); const { createPluginPackageResourceGenerationFromReferences } = ql3Require( '@qinglong/runtime-core/plugin-package-resource-generation', ); @@ -386,22 +547,14 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) { deploymentProfile: 'cluster-control', targetGeneration: 1, }; - const initialLock = createPluginPackageLock({ - ...initialAction, - approval: { - requestId: 'approval-plugin-recovery-e2e-initial', - requestVersion: 1, - dispatchId: 'dispatch-plugin-recovery-e2e-initial', - actionDigest: pluginPackageInstallActionDigest(initialAction), - previewDigest: pluginPackageInstallPlanDigest(initialPlan), - approvedBy: { type: 'user', id: 'e2e-owner' }, - approvedAtMs: createdAtMs - 1, - expiresAtMs: createdAtMs + 60 * 60 * 1000, - fence: { projectVersion: 1, bindingVersion: 1 }, - }, + const initialAuthority = createInstallAuthority( + initialAction, + 'initial', createdAtMs, - }); - const upgradeCreatedAtMs = createdAtMs + 10; + 1, + ); + const initialLock = initialAuthority.lock; + const upgradeCreatedAtMs = initialLock.createdAtMs + 10; const upgradeManifest = pluginManifest(architecture, '2.0.0', true); const upgradeMaterial = packageMaterial( registry, @@ -426,21 +579,13 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) { targetGeneration: 2, previousLockDigest: initialLock.lockDigest, }; - const upgradeLock = createPluginPackageLock({ - ...upgradeAction, - approval: { - requestId: 'approval-plugin-recovery-e2e-upgrade', - requestVersion: 1, - dispatchId: 'dispatch-plugin-recovery-e2e-upgrade', - actionDigest: pluginPackageInstallActionDigest(upgradeAction), - previewDigest: pluginPackageInstallPlanDigest(upgradePlan), - approvedBy: { type: 'user', id: 'e2e-owner' }, - approvedAtMs: upgradeCreatedAtMs - 1, - expiresAtMs: upgradeCreatedAtMs + 60 * 60 * 1000, - fence: { projectVersion: 1, bindingVersion: 1 }, - }, - createdAtMs: upgradeCreatedAtMs, - }); + const upgradeAuthority = createInstallAuthority( + upgradeAction, + 'upgrade', + upgradeCreatedAtMs, + 11, + ); + const upgradeLock = upgradeAuthority.lock; const { publicKey, privateKey } = generateKeyPairSync('ed25519'); const publicKeyPem = publicKey.export({ format: 'pem', type: 'spki' }); const trust = { @@ -467,6 +612,7 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) { privateKey, ); const initial = Object.freeze({ + authority: initialAuthority, installationId: 'install-plugin-recovery-e2e-initial', manifest: initialManifest, lock: initialLock, @@ -483,6 +629,7 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) { routes: initialRoutes, }); const upgrade = Object.freeze({ + authority: upgradeAuthority, installationId: 'install-plugin-recovery-e2e-upgrade', manifest: upgradeManifest, lock: upgradeLock, @@ -518,8 +665,12 @@ function readFixture(filePath) { value.schema !== FIXTURE_SCHEMA || !Array.isArray(value.routes) || !value.initial?.lock || + !value.initial?.authority?.proposalCommand?.proposal || + !value.initial?.authority?.dispatch || !value.initial?.generation || !value.upgrade?.lock || + !value.upgrade?.authority?.proposalCommand?.proposal || + !value.upgrade?.authority?.dispatch || !value.upgrade?.generation || !value.trust ) { @@ -624,32 +775,31 @@ async function runSeed() { assertPostgresPackageExecutorSchemaReady, createPostgresDatabaseOpener, loadPostgresConnectionEnvironment, - PostgresPluginPackageSecretBindingTransitionRepository, + PostgresApprovedActionExecutionRepository, } = ql3Require('@qinglong/cluster-postgres/package-executor'); const { assertPostgresPackageManagerSchemaReady, + PostgresApprovalRequestRepository, PostgresPluginPackagePublisherTrustAuthorityRepository, } = ql3Require('@qinglong/cluster-postgres/package-manager'); + const { PostgresProjectPolicyRepository } = ql3Require( + '@qinglong/cluster-postgres/project-policy', + ); const { PostgresPluginPackageInstallRepository } = ql3Require( '@qinglong/cluster-postgres/plugin-package-install', ); - const { - createPluginPackageInstall, - normalizePluginPackageLock, - pluginPackageInstallCreate, - } = ql3Require('@qinglong/runtime-core/plugin-package-install'); - const { createPluginPackageSecretBindingTarget } = ql3Require( - '@qinglong/runtime-core/plugin-package-secret-binding', + const { PostgresPluginPackageInstallProposalRepository } = ql3Require( + '@qinglong/cluster-postgres/plugin-package-proposal', ); - const { createPluginPackageSecretBindingTransitionPlan } = ql3Require( - '@qinglong/runtime-core/plugin-package-secret-binding-transition-plan', + const { normalizePluginPackageLock } = ql3Require( + '@qinglong/runtime-core/plugin-package-install', ); const { createPluginPackagePublisherTrustSnapshot } = ql3Require( '@qinglong/runtime-core/plugin-package-publisher-trust', ); const fixture = readFixture(process.env.QL3_E2E_FIXTURE_FILE); const mode = process.env.QL3_E2E_MODE; - if (!['seed-initial', 'seed-upgrade', 'commit-transition'].includes(mode)) { + if (!['seed-initial', 'seed-upgrade'].includes(mode)) { throw new Error('Plugin Package E2E seed mode is invalid'); } const connection = loadPostgresConnectionEnvironment(process.env, { @@ -671,72 +821,82 @@ async function runSeed() { })(); try { await assertPostgresPackageExecutorSchemaReady(database.pool); - if (mode === 'commit-transition') { - const plannedAtMs = Date.now(); - const transitionPlan = createPluginPackageSecretBindingTransitionPlan({ - previousTarget: createPluginPackageSecretBindingTarget( - fixture.initial.generation, - fixture.initial.manifest, - ), - previousBinding: null, - previousAttemptGeneration: 1, - nextGeneration: fixture.upgrade.generation, - nextManifest: fixture.upgrade.manifest, - assignments: [], - plannedAtMs, - }); - const result = - await new PostgresPluginPackageSecretBindingTransitionRepository( - database.pool, - ).apply({ - transitionPlan, - evidenceDigest: transitionPlan.transitionDigest, - committedAtMs: plannedAtMs + 1, - }); - process.stdout.write( - `${JSON.stringify({ - schema: 'qinglong/plugin-package-recovery-e2e-transition-result@v1', - event: 'transition_completed', - status: result.status, - generationDigest: transitionPlan.nextTarget.generationDigest, - transitionDigest: transitionPlan.transitionDigest, - bindingDigest: result.receipt.bindingDigest, - receiptDigest: result.receipt.receiptDigest, - })}\n`, - ); - return; - } const selected = mode === 'seed-initial' ? fixture.initial : fixture.upgrade; - let publisherTrustStatus = null; - if (mode === 'seed-initial') { - const packageManagerConnection = loadPostgresConnectionEnvironment( - process.env, - { - host: 'QL3_E2E_POSTGRES_HOST', - port: 'QL3_E2E_POSTGRES_PORT', - database: 'QL3_E2E_POSTGRES_DATABASE', - user: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_USER', - password: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_PASSWORD', - }, + const runtimeConnection = loadPostgresConnectionEnvironment(process.env, { + host: 'QL3_E2E_POSTGRES_HOST', + port: 'QL3_E2E_POSTGRES_PORT', + database: 'QL3_E2E_POSTGRES_DATABASE', + user: 'QL3_E2E_POSTGRES_RUNTIME_USER', + password: 'QL3_E2E_POSTGRES_RUNTIME_PASSWORD', + }); + const runtimeDatabase = await createPostgresDatabaseOpener({ + role: 'runtime', + connection: { ...runtimeConnection, tls: { mode: 'disable' } }, + pool: { + applicationName: 'qinglong3-plugin-package-e2e-policy-seed', + maxConnections: 1, + connectionTimeoutMs: 15_000, + }, + onPoolError() {}, + })(); + try { + const policies = new PostgresProjectPolicyRepository( + runtimeDatabase.pool, ); - const packageManagerDatabase = await createPostgresDatabaseOpener({ - role: 'package-manager', - connection: { - ...packageManagerConnection, - tls: { mode: 'disable' }, - }, - pool: { - applicationName: 'qinglong3-plugin-package-e2e-trust-seed', - maxConnections: 1, - connectionTimeoutMs: 15_000, - }, - onPoolError() {}, - })(); - try { - await assertPostgresPackageManagerSchemaReady( - packageManagerDatabase.pool, - ); + for (const [subject, role] of [ + [selected.authority.requester, 'owner'], + [selected.authority.reviewer, 'admin'], + ]) { + const policy = await policies.append({ + expectedCurrentVersion: 0, + binding: { + projectId: selected.lock.projectId, + subject, + version: 1, + state: 'active', + role, + mutationId: `plugin-recovery-e2e-grant-${role}`, + changedBy: { type: 'system', id: 'plugin-recovery-e2e-bootstrap' }, + createdAtMs: 1, + }, + }); + if (!['inserted', 'existing'].includes(policy.status)) { + throw new Error('Plugin Package E2E policy seed was not durable'); + } + } + } finally { + await runtimeDatabase.close(); + } + let publisherTrustStatus = null; + const packageManagerConnection = loadPostgresConnectionEnvironment( + process.env, + { + host: 'QL3_E2E_POSTGRES_HOST', + port: 'QL3_E2E_POSTGRES_PORT', + database: 'QL3_E2E_POSTGRES_DATABASE', + user: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_USER', + password: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_PASSWORD', + }, + ); + const packageManagerDatabase = await createPostgresDatabaseOpener({ + role: 'package-manager', + connection: { + ...packageManagerConnection, + tls: { mode: 'disable' }, + }, + pool: { + applicationName: 'qinglong3-plugin-package-e2e-management-seed', + maxConnections: 1, + connectionTimeoutMs: 15_000, + }, + onPoolError() {}, + })(); + try { + await assertPostgresPackageManagerSchemaReady( + packageManagerDatabase.pool, + ); + if (mode === 'seed-initial') { const observed = await new PostgresPluginPackagePublisherTrustAuthorityRepository( packageManagerDatabase.pool, @@ -754,9 +914,17 @@ async function runSeed() { ); } publisherTrustStatus = observed.status; - } finally { - await packageManagerDatabase.close(); } + await new PostgresPluginPackageInstallProposalRepository( + packageManagerDatabase.pool, + ).createProposal(selected.authority.proposalCommand); + const approvals = new PostgresApprovalRequestRepository( + packageManagerDatabase.pool, + ); + await approvals.create(selected.authority.requestCommand); + await approvals.decide(selected.authority.decisionCommand); + } finally { + await packageManagerDatabase.close(); } const lock = normalizePluginPackageLock(selected.lock); const repository = new PostgresPluginPackageInstallRepository( @@ -772,16 +940,68 @@ async function runSeed() { ) { throw new Error('Plugin Package E2E previous install head is invalid'); } - const record = createPluginPackageInstall(lock, { + const consumed = await new PostgresApprovalRequestRepository( + database.pool, + ).consume(selected.authority.consumptionCommand); + if ( + JSON.stringify(consumed.dispatch) !== + JSON.stringify(selected.authority.dispatch) + ) { + throw new Error('Plugin Package E2E durable dispatch drifted'); + } + const executions = new PostgresApprovedActionExecutionRepository( + database.pool, + ); + const executionAtMs = Date.now(); + const owner = 'plugin_recovery_e2e_dispatcher'; + const leaseToken = `lease-plugin-recovery-e2e-${ + mode === 'seed-initial' ? 'initial' : 'upgrade' + }`; + const claimed = await executions.claimExecution({ + dispatchId: consumed.dispatch.id, + owner, + leaseToken, + nowMs: executionAtMs, + leaseDurationMs: 60_000, + }); + if (claimed.status !== 'claimed') { + throw new Error('Plugin Package E2E execution was not claimable'); + } + const started = await executions.startExecution({ + dispatchId: consumed.dispatch.id, + approvalRequestId: consumed.dispatch.approvalRequestId, + actionDigest: consumed.dispatch.action.actionDigest, + owner, + leaseToken, + expectedVersion: claimed.snapshot.execution.version, + startedAtMs: executionAtMs + 1, + }); + const admittedAtMs = executionAtMs + 2; + const result = await repository.admit({ + lock, + proposalDigest: + selected.authority.proposalCommand.proposal.proposalDigest, + execution: started.execution, installationId: selected.installationId, mutationId: `mutation-plugin-recovery-e2e-${ mode === 'seed-initial' ? 'initial' : 'upgrade' - }-create`, - occurredAtMs: lock.createdAtMs + 1, + }-admit`, + admittedAtMs, + audit: { + eventId: selected.authority.admissionAuditEventId, + requestId: consumed.dispatch.id, + operationId: 'plugin_package.admit', + projectId: lock.projectId, + subject: selected.authority.consumer, + authenticationId: `auth-plugin-recovery-e2e-dispatcher-${ + mode === 'seed-initial' ? 'initial' : 'upgrade' + }`, + outcome: 'allowed', + reasons: ['approved_action'], + fence: selected.authority.fence, + occurredAtMs: admittedAtMs, + }, }); - const result = await repository.create( - pluginPackageInstallCreate(lock, record, previous), - ); process.stdout.write( `${JSON.stringify({ schema: 'qinglong/plugin-package-recovery-e2e-seed-result@v1', @@ -806,15 +1026,13 @@ async function main() { return; } if ( - ['seed-initial', 'seed-upgrade', 'commit-transition'].includes( - process.env.QL3_E2E_MODE, - ) + ['seed-initial', 'seed-upgrade'].includes(process.env.QL3_E2E_MODE) ) { await runSeed(); return; } throw new Error( - 'QL3_E2E_MODE must be registry, seed-initial, seed-upgrade or commit-transition', + 'QL3_E2E_MODE must be registry, seed-initial or seed-upgrade', ); } diff --git a/scripts/ql3-plugin-package-recovery-e2e-live-audit.cjs b/scripts/ql3-plugin-package-recovery-e2e-live-audit.cjs index 24a557c0..6c7c5c23 100644 --- a/scripts/ql3-plugin-package-recovery-e2e-live-audit.cjs +++ b/scripts/ql3-plugin-package-recovery-e2e-live-audit.cjs @@ -5,7 +5,7 @@ const fs = require('node:fs'); const path = require('node:path'); -const FIXTURE = 'qinglong/plugin-package-recovery-e2e-live-contract@v2'; +const FIXTURE = 'qinglong/plugin-package-recovery-e2e-live-contract@v3'; const LIMITATIONS = Object.freeze([ 'isolated PostgreSQL uses explicit TLS disable; production manifests remain verify-full', 'the authenticated HTTPS OCI Distribution fixture implements the immutable GET/referrers surface used by the resolver, not a production registry storage implementation', @@ -13,7 +13,6 @@ const LIMITATIONS = Object.freeze([ ]); const GATE_KEYS = Object.freeze([ 'healthyInitialActivation', - 'missingTransitionFailedClosed', 'invalidUpgradeRejectedBeforeActivation', 'activePointerUidUnchanged', 'activePointerResourceVersionUnchanged', @@ -119,8 +118,8 @@ function validRuntime(report) { ]) && runtime.replicas === 2 && runtime.creationTimestamp === ordering?.runtimeCreatedAt && - runtime.recoveryJobUid === ordering?.rejectionRecoveryJobUid && - runtime.recoveryCompletedAt === ordering?.rejectionRecoveryCompletedAt && + runtime.recoveryJobUid === ordering?.upgradeRecoveryJobUid && + runtime.recoveryCompletedAt === ordering?.upgradeRecoveryCompletedAt && Array.isArray(runtime.nodes) && runtime.nodes.length === 2 && new Set(runtime.nodes).size === 2 && @@ -179,8 +178,7 @@ function validatePluginPackageRecoveryE2ELiveReport(report) { 'postgresRepositoryDigest', 'migrationImageId', 'initialRecoveryImageId', - 'stageRecoveryImageId', - 'rejectionRecoveryImageId', + 'upgradeRecoveryImageId', 'postgresImageId', ]) || !SHA256_ID.test(images?.adminBuildId ?? '') || @@ -192,9 +190,7 @@ function validatePluginPackageRecoveryE2ELiveReport(report) { !imageDigest(images?.migrationImageId) || imageDigest(images?.initialRecoveryImageId) !== imageDigest(images?.migrationImageId) || - imageDigest(images?.stageRecoveryImageId) !== - imageDigest(images?.migrationImageId) || - imageDigest(images?.rejectionRecoveryImageId) !== + imageDigest(images?.upgradeRecoveryImageId) !== imageDigest(images?.migrationImageId) || !imageDigest(images?.postgresImageId) ) { @@ -207,17 +203,13 @@ function validatePluginPackageRecoveryE2ELiveReport(report) { const timeKeys = [ 'migrationCompletedAt', 'initialRecoveryCompletedAt', - 'upgradeStageFailedAt', - 'transitionCompletedAt', - 'rejectionRecoveryCompletedAt', + 'upgradeRecoveryCompletedAt', 'runtimeCreatedAt', ]; const uidKeys = [ 'migrationJobUid', 'initialRecoveryJobUid', - 'upgradeStageJobUid', - 'transitionJobUid', - 'rejectionRecoveryJobUid', + 'upgradeRecoveryJobUid', 'runtimeBoundRecoveryJobUid', ]; const times = timeKeys.map((key) => Date.parse(ordering?.[key])); @@ -225,7 +217,7 @@ function validatePluginPackageRecoveryE2ELiveReport(report) { !exactKeys(ordering, [...uidKeys, ...timeKeys]) || uidKeys.some((key) => !UUID.test(ordering?.[key] ?? '')) || ordering?.runtimeBoundRecoveryJobUid !== - ordering?.rejectionRecoveryJobUid || + ordering?.upgradeRecoveryJobUid || timeKeys.some((key) => !validIso(ordering?.[key])) || times.some((value, index) => index > 0 && value < times[index - 1]) ) { @@ -240,18 +232,12 @@ function validatePluginPackageRecoveryE2ELiveReport(report) { const failed = report?.failedUpgrade; if ( !exactKeys(failed, [ - 'stageFailure', - 'transitionReceiptDigest', + 'recoveryJobUid', 'rejectionReason', 'candidateRevisionCount', 'activePointerUnchanged', ]) || - !exactKeys(failed?.stageFailure, ['jobUid', 'reason', 'durableState']) || - failed?.stageFailure?.jobUid !== ordering?.upgradeStageJobUid || - failed?.stageFailure?.reason !== - 'ClusterPluginPackageRecoveryRequiredError' || - failed?.stageFailure?.durableState !== 'staged' || - !SHA256.test(failed?.transitionReceiptDigest ?? '') || + failed?.recoveryJobUid !== ordering?.upgradeRecoveryJobUid || failed?.rejectionReason !== 'activation_fact_conflict' || failed?.candidateRevisionCount !== 0 || failed?.activePointerUnchanged !== true @@ -277,7 +263,6 @@ function validatePluginPackageRecoveryE2ELiveReport(report) { 'initialMutationCount', 'upgradeMutationCount', 'headInstallationId', - 'transitionReceiptCount', 'initialRevisionCount', 'upgradeRevisionCount', 'recoverableCount', @@ -296,7 +281,6 @@ function validatePluginPackageRecoveryE2ELiveReport(report) { database?.initialMutationCount !== 4 || database?.upgradeMutationCount !== 3 || database?.headInstallationId !== 'install-plugin-recovery-e2e-upgrade' || - database?.transitionReceiptCount !== 1 || database?.initialRevisionCount !== 1 || database?.upgradeRevisionCount !== 0 || database?.recoverableCount !== 0 @@ -323,11 +307,11 @@ function validatePluginPackageRecoveryE2ELiveReport(report) { ]) || oci?.https !== true || oci?.authentication !== 'exact-registry-basic' || - oci?.authenticatedRequestCount !== 18 || - oci?.requestCount !== 18 || + oci?.authenticatedRequestCount !== 12 || + oci?.requestCount !== 12 || oci?.uniquePaths !== 12 || oci?.initialRequestCount !== 6 || - oci?.upgradeRequestCount !== 12 || + oci?.upgradeRequestCount !== 6 || oci?.redirects !== 0 ) { findings.push( diff --git a/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs b/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs index 1146d344..6cf41f88 100644 --- a/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs +++ b/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs @@ -38,13 +38,11 @@ const POSTGRES_RUNTIME_IMAGE = const DEFAULT_ADMIN_IMAGE = 'qinglong3-cluster-admin:ql3-plugin-recovery-e2e'; const DEFAULT_CONTROL_IMAGE = 'qinglong3-cluster-control:ql3-plugin-recovery-e2e'; -const REPORT_SCHEMA = 'qinglong/plugin-package-recovery-e2e-live-contract@v2'; +const REPORT_SCHEMA = 'qinglong/plugin-package-recovery-e2e-live-contract@v3'; const INITIAL_SEED_JOB = 'ql3-plugin-package-e2e-seed-initial'; const INITIAL_RECOVERY_JOB = 'ql3-plugin-package-recovery-initial'; const UPGRADE_SEED_JOB = 'ql3-plugin-package-e2e-seed-upgrade'; -const UPGRADE_STAGE_JOB = 'ql3-plugin-package-recovery-stage-upgrade'; -const TRANSITION_JOB = 'ql3-plugin-package-e2e-transition'; -const UPGRADE_REJECTION_JOB = 'ql3-plugin-package-recovery-reject-upgrade'; +const UPGRADE_RECOVERY_JOB = 'ql3-plugin-package-recovery-reject-upgrade'; const SAFE_CLUSTER = /^ql3-plugin-recovery-e2e(?:-[a-z0-9](?:[-a-z0-9]{0,24}[a-z0-9])?)?$/; @@ -879,6 +877,19 @@ function seedJob(name, mode) { { name: 'QL3_E2E_POSTGRES_HOST', value: POSTGRES_NAME }, { name: 'QL3_E2E_POSTGRES_PORT', value: '5432' }, { name: 'QL3_E2E_POSTGRES_DATABASE', value: 'qinglong' }, + { + name: 'QL3_E2E_POSTGRES_RUNTIME_USER', + value: 'ql3_runtime', + }, + { + name: 'QL3_E2E_POSTGRES_RUNTIME_PASSWORD', + valueFrom: { + secretKeyRef: { + name: 'ql3-e2e-postgres-auth', + key: 'runtime-password', + }, + }, + }, { name: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_USER', value: 'ql3_package_manager', @@ -1179,67 +1190,6 @@ function canI(verb, resource) { return result.stdout === 'yes'; } -function upgradeStageEvidence(fixture, transitionReceiptCount) { - const generationDigest = fixture.upgrade.generation.generationDigest; - assert.match(generationDigest, /^[0-9a-f]{64}$/); - const sql = ` -SELECT json_build_object( - 'state', ( - SELECT state FROM ql3.plugin_package_installs - WHERE installation_id = '${fixture.upgrade.installationId}' - ), - 'previousActiveLockDigest', ( - SELECT previous_active_lock_digest FROM ql3.plugin_package_installs - WHERE installation_id = '${fixture.upgrade.installationId}' - ), - 'activeLockDigest', ( - SELECT active_lock_digest FROM ql3.plugin_package_installs - WHERE installation_id = '${fixture.upgrade.installationId}' - ), - 'mutationCount', ( - SELECT count(*) FROM ql3.plugin_package_install_mutations - WHERE installation_id = '${fixture.upgrade.installationId}' - ), - 'transitionReceiptCount', ( - SELECT count(*) - FROM ql3.plugin_package_secret_binding_transition_receipts - WHERE generation_digest = '${generationDigest}' - ), - 'candidateRevisionCount', ( - SELECT count(*) FROM ql3.plugin_package_materialized_revisions - WHERE generation_digest = '${generationDigest}' - ) -)::text; -`.trim(); - const output = kubectl( - [ - '-n', - NAMESPACE, - 'exec', - POSTGRES_NAME, - '--', - 'psql', - '--username', - 'postgres', - '--dbname', - 'qinglong', - '--tuples-only', - '--no-align', - '--command', - sql, - ], - { capture: true, quiet: true }, - ).stdout; - const value = JSON.parse(output); - assert.equal(value.state, 'staged'); - assert.equal(value.previousActiveLockDigest, fixture.initial.lock.lockDigest); - assert.equal(value.activeLockDigest, fixture.initial.lock.lockDigest); - assert.equal(value.mutationCount, 2); - assert.equal(value.transitionReceiptCount, transitionReceiptCount); - assert.equal(value.candidateRevisionCount, 0); - return value; -} - function databaseEvidence(fixture) { const initialGenerationDigest = fixture.initial.generation.generationDigest; const upgradeGenerationDigest = fixture.upgrade.generation.generationDigest; @@ -1298,11 +1248,6 @@ SELECT json_build_object( FROM ql3.plugin_package_install_heads WHERE project_id = 'default' AND package_name = 'e2e-monitor' ), - 'transitionReceiptCount', ( - SELECT count(*) - FROM ql3.plugin_package_secret_binding_transition_receipts - WHERE generation_digest = '${upgradeGenerationDigest}' - ), 'initialRevisionCount', ( SELECT count(*) FROM ql3.plugin_package_materialized_revisions WHERE generation_digest = '${initialGenerationDigest}' @@ -1352,7 +1297,6 @@ SELECT json_build_object( assert.equal(value.initialMutationCount, 4); assert.equal(value.upgradeMutationCount, 3); assert.equal(value.headInstallationId, fixture.upgrade.installationId); - assert.equal(value.transitionReceiptCount, 1); assert.equal(value.initialRevisionCount, 1); assert.equal(value.upgradeRevisionCount, 0); assert.equal(value.recoverableCount, 0); @@ -1411,10 +1355,7 @@ function registryEvidence(fixture) { const packageRequests = events.filter((event) => event.path !== '/v2/'); const expectedPaths = [ ...fixture.initial.routes.map((routeValue) => routeValue.path), - ...fixture.upgrade.routes.flatMap((routeValue) => [ - routeValue.path, - routeValue.path, - ]), + ...fixture.upgrade.routes.map((routeValue) => routeValue.path), ].sort(); assert.equal(packageRequests.length, expectedPaths.length); assert.deepEqual( @@ -1430,7 +1371,7 @@ function registryEvidence(fixture) { requestCount: packageRequests.length, uniquePaths: new Set(packageRequests.map((event) => event.path)).size, initialRequestCount: fixture.initial.routes.length, - upgradeRequestCount: fixture.upgrade.routes.length * 2, + upgradeRequestCount: fixture.upgrade.routes.length, redirects: 0, }); } @@ -1874,7 +1815,7 @@ async function main(argv = process.argv.slice(2)) { (value) => value.event === 'seed_completed', ); assert.equal(initialSeed.phase, 'initial'); - assert.equal(initialSeed.status, 'created'); + assert.equal(initialSeed.status, 'admitted'); assert.equal(initialSeed.state, 'queued'); assert.equal(initialSeed.lockDigest, fixture.initial.lock.lockDigest); @@ -1901,52 +1842,16 @@ async function main(argv = process.argv.slice(2)) { (value) => value.event === 'seed_completed', ); assert.equal(upgradeSeed.phase, 'upgrade'); - assert.equal(upgradeSeed.status, 'created'); + assert.equal(upgradeSeed.status, 'admitted'); assert.equal(upgradeSeed.state, 'queued'); assert.equal(upgradeSeed.lockDigest, fixture.upgrade.lock.lockDigest); - for (const resource of recoveryResources(fixture, UPGRADE_STAGE_JOB)) { - apply(resource, `apply upgrade staging recovery ${resource.kind}`); - } - const stagedUpgrade = waitForJob(UPGRADE_STAGE_JOB, 'failed'); - const stagedFailureCondition = stagedUpgrade.status.conditions.find( - (condition) => condition.type === 'Failed' && condition.status === 'True', - ); - assert.ok(stagedFailureCondition?.lastTransitionTime); - const stageFailure = lastJsonLine( - jobLog(UPGRADE_STAGE_JOB), - (value) => value.event === 'recovery_failed', - ); - assert.equal( - stageFailure.name, - 'ClusterPluginPackageRecoveryRequiredError', - ); - const stagedDatabase = upgradeStageEvidence(fixture, 0); - assert.deepEqual(activePointerEvidence(fixture), pointerBeforeUpgrade); - - apply( - seedJob(TRANSITION_JOB, 'commit-transition'), - 'commit durable no-secret binding transition receipt', - ); - const committedTransition = waitForJob(TRANSITION_JOB); - const transition = lastJsonLine( - jobLog(TRANSITION_JOB), - (value) => value.event === 'transition_completed', - ); - assert.equal(transition.status, 'created'); - assert.equal( - transition.generationDigest, - fixture.upgrade.generation.generationDigest, - ); - assert.equal(transition.bindingDigest, null); - upgradeStageEvidence(fixture, 1); - - for (const resource of recoveryResources(fixture, UPGRADE_REJECTION_JOB)) { + for (const resource of recoveryResources(fixture, UPGRADE_RECOVERY_JOB)) { apply(resource, `apply upgrade rejection recovery ${resource.kind}`); } - const rejectedUpgrade = waitForJob(UPGRADE_REJECTION_JOB); + const rejectedUpgrade = waitForJob(UPGRADE_RECOVERY_JOB); const rejectionCompleted = lastJsonLine( - jobLog(UPGRADE_REJECTION_JOB), + jobLog(UPGRADE_RECOVERY_JOB), (value) => value.event === 'recovery_completed', ); assert.equal(rejectionCompleted.recovery.safeToAdmit, true); @@ -1965,8 +1870,7 @@ async function main(argv = process.argv.slice(2)) { secrets, ); const initialRecoveryImageId = jobImageId(INITIAL_RECOVERY_JOB); - const stageRecoveryImageId = jobImageId(UPGRADE_STAGE_JOB); - const rejectionRecoveryImageId = jobImageId(UPGRADE_REJECTION_JOB); + const upgradeRecoveryImageId = jobImageId(UPGRADE_RECOVERY_JOB); const migrationImageId = jobImageId('ql3-cluster-migration'); const postgresPod = kubectlJson([ '-n', @@ -1991,8 +1895,7 @@ async function main(argv = process.argv.slice(2)) { postgresRepositoryDigest: POSTGRES_REPOSITORY_DIGEST, migrationImageId, initialRecoveryImageId, - stageRecoveryImageId, - rejectionRecoveryImageId, + upgradeRecoveryImageId, postgresImageId: postgresPod.status.containerStatuses[0].imageID, }), ordering: Object.freeze({ @@ -2000,22 +1903,13 @@ async function main(argv = process.argv.slice(2)) { migrationCompletedAt: migrated.status.completionTime, initialRecoveryJobUid: initialRecovered.metadata.uid, initialRecoveryCompletedAt: initialRecovered.status.completionTime, - upgradeStageJobUid: stagedUpgrade.metadata.uid, - upgradeStageFailedAt: stagedFailureCondition.lastTransitionTime, - transitionJobUid: committedTransition.metadata.uid, - transitionCompletedAt: committedTransition.status.completionTime, - rejectionRecoveryJobUid: rejectedUpgrade.metadata.uid, - rejectionRecoveryCompletedAt: rejectedUpgrade.status.completionTime, + upgradeRecoveryJobUid: rejectedUpgrade.metadata.uid, + upgradeRecoveryCompletedAt: rejectedUpgrade.status.completionTime, runtimeCreatedAt: runtime.creationTimestamp, runtimeBoundRecoveryJobUid: runtime.recoveryJobUid, }), failedUpgrade: Object.freeze({ - stageFailure: Object.freeze({ - jobUid: stagedUpgrade.metadata.uid, - reason: stageFailure.name, - durableState: stagedDatabase.state, - }), - transitionReceiptDigest: transition.receiptDigest, + recoveryJobUid: rejectedUpgrade.metadata.uid, rejectionReason: database.upgradeFailureReason, candidateRevisionCount: database.upgradeRevisionCount, activePointerUnchanged: true, @@ -2029,7 +1923,6 @@ async function main(argv = process.argv.slice(2)) { runtime, gates: Object.freeze({ healthyInitialActivation: true, - missingTransitionFailedClosed: true, invalidUpgradeRejectedBeforeActivation: true, activePointerUidUnchanged: true, activePointerResourceVersionUnchanged: true, diff --git a/test/back/ql3PluginPackageRecoveryE2ELiveAudit.test.cjs b/test/back/ql3PluginPackageRecoveryE2ELiveAudit.test.cjs index 40803ae8..23eb35b7 100644 --- a/test/back/ql3PluginPackageRecoveryE2ELiveAudit.test.cjs +++ b/test/back/ql3PluginPackageRecoveryE2ELiveAudit.test.cjs @@ -42,8 +42,7 @@ function validReport() { 'postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296', migrationImageId: `docker://${admin}`, initialRecoveryImageId: `docker://${admin}`, - stageRecoveryImageId: `docker://${admin}`, - rejectionRecoveryImageId: `docker://${admin}`, + upgradeRecoveryImageId: `docker://${admin}`, postgresImageId: `containerd://sha256:${'e'.repeat(64)}`, }, ordering: { @@ -51,22 +50,13 @@ function validReport() { migrationCompletedAt: '2026-08-14T08:00:01.000Z', initialRecoveryJobUid: '00000000-0000-4000-8000-000000000002', initialRecoveryCompletedAt: '2026-08-14T08:00:02.000Z', - upgradeStageJobUid: '00000000-0000-4000-8000-000000000003', - upgradeStageFailedAt: '2026-08-14T08:00:03.000Z', - transitionJobUid: '00000000-0000-4000-8000-000000000004', - transitionCompletedAt: '2026-08-14T08:00:04.000Z', - rejectionRecoveryJobUid: '00000000-0000-4000-8000-000000000005', - rejectionRecoveryCompletedAt: '2026-08-14T08:00:05.000Z', - runtimeCreatedAt: '2026-08-14T08:00:06.000Z', - runtimeBoundRecoveryJobUid: '00000000-0000-4000-8000-000000000005', + upgradeRecoveryJobUid: '00000000-0000-4000-8000-000000000003', + upgradeRecoveryCompletedAt: '2026-08-14T08:00:03.000Z', + runtimeCreatedAt: '2026-08-14T08:00:04.000Z', + runtimeBoundRecoveryJobUid: '00000000-0000-4000-8000-000000000003', }, failedUpgrade: { - stageFailure: { - jobUid: '00000000-0000-4000-8000-000000000003', - reason: 'ClusterPluginPackageRecoveryRequiredError', - durableState: 'staged', - }, - transitionReceiptDigest: 'f'.repeat(64), + recoveryJobUid: '00000000-0000-4000-8000-000000000003', rejectionReason: 'activation_fact_conflict', candidateRevisionCount: 0, activePointerUnchanged: true, @@ -83,7 +73,6 @@ function validReport() { initialMutationCount: 4, upgradeMutationCount: 3, headInstallationId: 'install-plugin-recovery-e2e-upgrade', - transitionReceiptCount: 1, initialRevisionCount: 1, upgradeRevisionCount: 0, recoverableCount: 0, @@ -91,11 +80,11 @@ function validReport() { oci: { https: true, authentication: 'exact-registry-basic', - authenticatedRequestCount: 18, - requestCount: 18, + authenticatedRequestCount: 12, + requestCount: 12, uniquePaths: 12, initialRequestCount: 6, - upgradeRequestCount: 12, + upgradeRequestCount: 6, redirects: 0, }, kubernetes: { @@ -118,9 +107,9 @@ function validReport() { }, runtime: { replicas: 2, - creationTimestamp: '2026-08-14T08:00:06.000Z', - recoveryJobUid: '00000000-0000-4000-8000-000000000005', - recoveryCompletedAt: '2026-08-14T08:00:05.000Z', + creationTimestamp: '2026-08-14T08:00:04.000Z', + recoveryJobUid: '00000000-0000-4000-8000-000000000003', + recoveryCompletedAt: '2026-08-14T08:00:03.000Z', nodes: ['worker-a', 'worker-b'], imageIds: [`docker://${control}`], }, @@ -142,7 +131,8 @@ test('offline audit accepts one exact low-sensitive recovery report', () => { test('offline audit rejects broken upgrade, ordering and image relationships', () => { const report = validReport(); report.database.upgradeRevisionCount = 1; - report.ordering.transitionCompletedAt = '2026-08-14T07:59:59.000Z'; + report.ordering.upgradeRecoveryCompletedAt = + '2026-08-14T07:59:59.000Z'; report.runtime.imageIds = [`docker://sha256:${'9'.repeat(64)}`]; report.gates.activePointerJsonUnchanged = false; const codes = validatePluginPackageRecoveryE2ELiveReport(report).findings.map( diff --git a/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs b/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs index 4cb72caa..ed44c8a0 100644 --- a/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs +++ b/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs @@ -3,6 +3,9 @@ const fs = require('node:fs'); const path = require('node:path'); const { test } = require('node:test'); const yaml = require('js-yaml'); +const { + createFixture, +} = require('../../scripts/ql3-plugin-package-recovery-e2e-fixture.cjs'); const root = path.resolve(__dirname, '../..'); const livePath = path.join( @@ -53,6 +56,27 @@ test('fixture uses a real HTTPS and content-addressed OCI Distribution surface', assert.match(live, /requestCount: packageRequests\.length/); }); +test('fixture locks are bound to durable version-three approval dispatches', () => { + const value = createFixture({ + registry: 'registry.fixture.test', + architecture: 'amd64', + createdAtMs: 1_000, + }); + for (const selected of [value.initial, value.upgrade]) { + const proposal = selected.authority.proposalCommand.proposal; + const dispatch = selected.authority.dispatch; + assert.equal(selected.lock.approval.requestVersion, 3); + assert.equal(selected.lock.approval.dispatchId, dispatch.id); + assert.equal(selected.lock.actionDigest, proposal.actionDigest); + assert.equal(selected.lock.planDigest, proposal.previewDigest); + assert.equal(dispatch.action.actionRef, proposal.actionRef); + assert.equal( + selected.authority.consumptionCommand.dispatchId, + dispatch.id, + ); + } +}); + test('gate runs migration, healthy activation and a durable rejected upgrade', () => { assert.match(live, /operations\/base\/migrate-job\.yaml/); assert.match( @@ -69,6 +93,14 @@ test('gate runs migration, healthy activation and a durable rejected upgrade', ( ); assert.match(live, /plugin-package-recovery\/base\/recover-job\.yaml/); assert.match(fixture, /PostgresPluginPackageInstallRepository/); + assert.match(fixture, /PostgresPluginPackageInstallProposalRepository/); + assert.match(fixture, /PostgresApprovalRequestRepository/); + assert.match(fixture, /PostgresApprovedActionExecutionRepository/); + assert.match(fixture, /PostgresProjectPolicyRepository/); + assert.match(fixture, /\.createProposal\(selected\.authority\.proposalCommand\)/); + assert.match(fixture, /\.consume\(selected\.authority\.consumptionCommand\)/); + assert.match(fixture, /repository\.admit\(/); + assert.doesNotMatch(fixture, /pluginPackageInstallCreate/); assert.match( fixture, /PostgresPluginPackagePublisherTrustAuthorityRepository/, @@ -79,12 +111,9 @@ test('gate runs migration, healthy activation and a durable rejected upgrade', ( assert.match(fixture, /publisherTrustStatus/); assert.match(live, /QL3_E2E_POSTGRES_PACKAGE_MANAGER_USER/); assert.match(live, /key: 'package-manager-password'/); - assert.match( - fixture, - /PostgresPluginPackageSecretBindingTransitionRepository/, - ); - assert.match(fixture, /createPluginPackageInstall/); - assert.match(fixture, /pluginPackageInstallCreate/); + assert.match(live, /QL3_E2E_POSTGRES_RUNTIME_USER/); + assert.match(live, /key: 'runtime-password'/); + assert.doesNotMatch(fixture, /commit-transition/); assert.match(live, /initialSeed\.state, 'queued'/); assert.match(live, /upgradeSeed\.state, 'queued'/); assert.match(live, /value\.migrationCount, 65/); @@ -99,27 +128,18 @@ test('gate runs migration, healthy activation and a durable rejected upgrade', ( }); test('deployment controller rejects the upgrade before creating runtime', () => { - const failedStageWait = live.indexOf( - "waitForJob(UPGRADE_STAGE_JOB, 'failed')", - ); - const transitionWait = live.indexOf('waitForJob(TRANSITION_JOB)'); - const rejectionWait = live.indexOf('waitForJob(UPGRADE_REJECTION_JOB)'); + const rejectionWait = live.indexOf('waitForJob(UPGRADE_RECOVERY_JOB)'); const pointerProof = live.indexOf( 'assert.deepEqual(pointerAfterRejection, pointerBeforeUpgrade)', ); const runtimeApply = live.indexOf( 'const runtime = applyRuntimeAfterRecovery(', ); - assert.ok(failedStageWait > 0); - assert.ok(transitionWait > failedStageWait); - assert.ok(rejectionWait > transitionWait); + assert.ok(rejectionWait > 0); assert.ok(pointerProof > rejectionWait); assert.ok(runtimeApply > pointerProof); assert.match(live, /activePointerUnchanged: true/); - assert.match( - live, - /stageFailure\.name,[\s\S]*'ClusterPluginPackageRecoveryRequiredError'/, - ); + assert.doesNotMatch(live, /missingTransitionFailedClosed/); assert.match(live, /qinglong\.io\/plugin-recovery-job-uid/); assert.match(live, /qinglong\.io\/plugin-recovery-completed-at/); assert.match(live, /rollout[\s\S]*status/);