From 6001174e232b72ba12127158fc3618c8257c6138 Mon Sep 17 00:00:00 2001 From: whyour Date: Sun, 23 Aug 2026 14:29:19 +0800 Subject: [PATCH] fix(ci): bind provider evidence to rollout generation --- ...edential-test-kubernetes-live-contract.cjs | 78 +++++++++++++++---- ...dentialTestKubernetesLiveContract.test.cjs | 60 +++++++++++++- 2 files changed, 119 insertions(+), 19 deletions(-) diff --git a/scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs b/scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs index 6e39a349..de79945f 100644 --- a/scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs +++ b/scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs @@ -289,6 +289,34 @@ function providerPods(fixture) { ]).items; } +async function readyProviderPodForGeneration( + fixture, + expectedGeneration, + timeoutMs = 60_000, +) { + assert.match(expectedGeneration, /^[1-9]\d*-[a-z0-9]+(?:-[a-z0-9]+)*$/); + return ( + await waitFor( + `provider generation ${expectedGeneration}`, + timeoutMs, + () => { + const ready = providerPods(fixture).filter(podReady); + const matching = ready.filter( + (pod) => + pod.metadata.annotations?.['qinglong.io/provider-generation'] === + expectedGeneration, + ); + return matching.length === 1 + ? { ready: true, value: matching[0] } + : { + ready: false, + fact: `${matching.length} matching of ${ready.length} Ready provider Pods`, + }; + }, + ) + ).value; +} + function providerObservationKey(pod) { assert.match(pod?.metadata?.uid ?? '', /^[A-Za-z0-9][A-Za-z0-9._-]+$/); const provider = pod?.status?.containerStatuses?.find( @@ -421,6 +449,7 @@ const server=https.createServer({key:fs.readFileSync('/var/run/provider/tls.key' } async function deployProvider(fixture, adminImage, generation, value, nonce) { + const expectedGeneration = `${generation}-${nonce}`; applySecret(fixture, 'ql3-provider-live-authority', 'Opaque', { value, }); @@ -439,7 +468,7 @@ async function deployProvider(fixture, adminImage, generation, value, nonce) { metadata: { labels: { 'app.kubernetes.io/name': PROVIDER_NAME }, annotations: { - 'qinglong.io/provider-generation': `${generation}-${nonce}`, + 'qinglong.io/provider-generation': expectedGeneration, }, }, spec: { @@ -542,10 +571,12 @@ async function deployProvider(fixture, adminImage, generation, value, nonce) { `deployment/${PROVIDER_NAME}`, '--timeout=3m', ]); - const ready = providerPods(fixture).filter(podReady); - assert.equal(ready.length, 1); - assert.match(ready[0].status.podIP, /^\d{1,3}(?:\.\d{1,3}){3}$/); - return ready[0]; + const ready = await readyProviderPodForGeneration( + fixture, + expectedGeneration, + ); + assert.match(ready.status.podIP, /^\d{1,3}(?:\.\d{1,3}){3}$/); + return ready; } async function waitForJob(fixture, name, timeoutMs = 120_000) { @@ -1091,8 +1122,9 @@ async function retryProviderEvidence(read, pause = undefined) { ) { throw error; } - await (pause ?? - (() => new Promise((resolve) => setTimeout(resolve, 500))))(); + await ( + pause ?? (() => new Promise((resolve) => setTimeout(resolve, 500))) + )(); } } throw lastError; @@ -1630,29 +1662,42 @@ async function main() { let observedProviderRequests = 0; let evidenceSequence = 0; const observeProvider = async (pod, sourceNodeName) => { - applyExecutorNetworkPolicy(fixture, pod.status.podIP); + const expectedGeneration = + pod.metadata.annotations?.['qinglong.io/provider-generation']; + assert.match( + expectedGeneration ?? '', + /^[1-9]\d*-[a-z0-9]+(?:-[a-z0-9]+)*$/, + ); + let currentPod; const evidence = await retryProviderEvidence(async () => { + currentPod = await readyProviderPodForGeneration( + fixture, + expectedGeneration, + ); + applyExecutorNetworkPolicy(fixture, currentPod.status.podIP); evidenceSequence += 1; return providerEvidence({ fixture, adminImage, name: `ql3-provider-evidence-${evidenceSequence}`, nodeName: sourceNodeName, - providerPodIp: pod.status.podIP, + providerPodIp: currentPod.status.podIP, }); }); + assert.ok(currentPod); const count = evidence.requestCount; - const observationKey = providerObservationKey(pod); + const observationKey = providerObservationKey(currentPod); const previous = requestObservations.get(observationKey) ?? 0; assert.ok(count >= previous); observedProviderRequests += count - previous; requestObservations.set(observationKey, count); + return currentPod; }; applyExecutorNetworkPolicy(fixture, null); const baseIdentity = await plan('base-denied'); const baseDenied = await execute('base-denied', baseIdentity); - await observeProvider(providerPod, baseDenied.nodeName); + providerPod = await observeProvider(providerPod, baseDenied.nodeName); assert.equal(observedProviderRequests, 0); assert.equal(baseDenied.outcome, 'unreachable'); assert.equal(baseDenied.modelCount, null); @@ -1668,7 +1713,7 @@ async function main() { const exactAllowed = await execute('exact-allowed', allowedIdentity); const exactReplay = await execute('exact-replay', allowedIdentity); assert.equal(exactReplay.status, 'existing'); - await observeProvider(providerPod, exactAllowed.nodeName); + providerPod = await observeProvider(providerPod, exactAllowed.nodeName); const exactCredentialAuditCount = Number( psql( fixture, @@ -1723,7 +1768,7 @@ async function main() { }); const rotatedIdentity = await plan('rotated-material'); const rotatedMaterial = await execute('rotated-material', rotatedIdentity); - await observeProvider(providerPod, rotatedMaterial.nodeName); + providerPod = await observeProvider(providerPod, rotatedMaterial.nodeName); assert.equal(observedProviderRequests, 3); assert.equal(staleMaterial.outcome, 'unreachable'); assert.equal(staleMaterial.modelCount, null); @@ -1741,7 +1786,7 @@ async function main() { assert.notEqual(providerPod.status.podIP, cidrBeforeReplacement); const staleCidrIdentity = await plan('stale-cidr'); const staleCidr = await execute('stale-cidr', staleCidrIdentity); - await observeProvider(providerPod, staleCidr.nodeName); + providerPod = await observeProvider(providerPod, staleCidr.nodeName); assert.equal(observedProviderRequests, 3); assert.equal(staleCidr.outcome, 'unreachable'); assert.equal(staleCidr.modelCount, null); @@ -1754,7 +1799,7 @@ async function main() { }); const refreshedIdentity = await plan('refreshed-cidr'); const refreshedCidr = await execute('refreshed-cidr', refreshedIdentity); - await observeProvider(providerPod, refreshedCidr.nodeName); + providerPod = await observeProvider(providerPod, refreshedCidr.nodeName); assert.equal(observedProviderRequests, 4); assert.equal(refreshedCidr.outcome, 'reachable'); assert.equal(refreshedCidr.modelCount, 2); @@ -1852,7 +1897,7 @@ async function main() { }); const postFailoverIdentity = await plan('post-failover'); const postFailover = await execute('post-failover', postFailoverIdentity); - await observeProvider(providerPod, postFailover.nodeName); + providerPod = await observeProvider(providerPod, postFailover.nodeName); assert.equal(observedProviderRequests, 5); assert.equal(postFailover.outcome, 'reachable'); assert.equal(postFailover.modelCount, 2); @@ -2090,6 +2135,7 @@ module.exports = { executorJob, providerObservationKey, providerServerSource, + readyProviderPodForGeneration, retryProviderEvidence, terminalJobSnapshot, }; diff --git a/test/back/ql3ProviderCredentialTestKubernetesLiveContract.test.cjs b/test/back/ql3ProviderCredentialTestKubernetesLiveContract.test.cjs index 33c4e77d..4fea846d 100644 --- a/test/back/ql3ProviderCredentialTestKubernetesLiveContract.test.cjs +++ b/test/back/ql3ProviderCredentialTestKubernetesLiveContract.test.cjs @@ -14,6 +14,7 @@ const { executorJob, providerObservationKey, providerServerSource, + readyProviderPodForGeneration, retryProviderEvidence, terminalJobSnapshot, } = require('../../scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs'); @@ -341,10 +342,14 @@ test('reads provider evidence from the exact ready Pod with trusted TLS SNI', () 'utf8', ); - assert.match(source, /providerPodIp: pod\.status\.podIP/); + assert.match(source, /currentPod = await readyProviderPodForGeneration/); + assert.match(source, /providerPodIp: currentPod\.status\.podIP/); assert.match(source, /host:process\.argv\[1\]/); assert.match(source, /servername:process\.argv\[3\]/); - assert.match(source, /ca:fs\.readFileSync\('\/var\/run\/provider-ca\/ca\.crt'\)/); + assert.match( + source, + /ca:fs\.readFileSync\('\/var\/run\/provider-ca\/ca\.crt'\)/, + ); assert.doesNotMatch( source, /fetch\('https:\/\/'\+process\.argv\[1\].*\/evidence/, @@ -403,7 +408,12 @@ test('provider fixture uses headless DNS for exact Pod CIDR policy', async () => return { items: [ { - metadata: { uid: 'provider-uid' }, + metadata: { + uid: 'provider-uid', + annotations: { + 'qinglong.io/provider-generation': '1-initial', + }, + }, status: { podIP: '10.42.7.19', conditions: [{ type: 'Ready', status: 'True' }], @@ -420,6 +430,50 @@ test('provider fixture uses headless DNS for exact Pod CIDR policy', async () => assert.equal(service.spec.clusterIP, 'None'); }); +test('binds a rollout to the exact ready provider generation', async () => { + const fixture = { + kubectlJson() { + return { + items: [ + { + metadata: { + uid: 'old-provider', + annotations: { + 'qinglong.io/provider-generation': '1-initial', + }, + }, + status: { + podIP: '10.42.7.18', + conditions: [{ type: 'Ready', status: 'True' }], + }, + }, + { + metadata: { + uid: 'current-provider', + annotations: { + 'qinglong.io/provider-generation': '2-material-rotation', + }, + }, + status: { + podIP: '10.42.7.19', + conditions: [{ type: 'Ready', status: 'True' }], + }, + }, + ], + }; + }, + }; + + const provider = await readyProviderPodForGeneration( + fixture, + '2-material-rotation', + 1_000, + ); + + assert.equal(provider.metadata.uid, 'current-provider'); + assert.equal(provider.status.podIP, '10.42.7.19'); +}); + test('actor writes bounded failure diagnostics to its termination message', () => { const source = fs.readFileSync( path.join(