diff --git a/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs b/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs index 67776fb3..54eef227 100644 --- a/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs +++ b/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs @@ -1098,6 +1098,36 @@ function recoveryResources(fixture, jobName) { ]; } +function waitForPostgresService(timeoutMs = 60_000) { + const deadline = Date.now() + timeoutMs; + let lastFact = 'probe not started'; + while (Date.now() < deadline) { + const probe = kubectl( + [ + '-n', + NAMESPACE, + 'exec', + `pod/${POSTGRES_NAME}`, + '--', + 'pg_isready', + '--host', + POSTGRES_NAME, + '--port', + '5432', + '--username', + 'postgres', + '--dbname', + 'qinglong', + ], + { capture: true, quiet: true, allowFailure: true }, + ); + if (probe.status === 0) return; + lastFact = (probe.stderr || probe.stdout || `status ${probe.status}`).trim(); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1_000); + } + fail(`PostgreSQL Service was not ready after ${timeoutMs}ms: ${lastFact}`); +} + function waitForJob( name, expectedStatus = 'complete', @@ -1795,6 +1825,7 @@ async function main(argv = process.argv.slice(2)) { ], { capture: true, quiet: true }, ); + waitForPostgresService(); kubectl( [ '-n', diff --git a/scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs b/scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs index aaa41ec8..6e39a349 100644 --- a/scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs +++ b/scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs @@ -1074,6 +1074,30 @@ request.on('timeout',()=>request.destroy(Object.assign(new Error('timeout'),{cod return evidence; } +async function retryProviderEvidence(read, pause = undefined) { + assert.equal(typeof read, 'function'); + assert.ok(pause === undefined || typeof pause === 'function'); + let lastError; + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + return await read(); + } catch (error) { + lastError = error; + if ( + !/\b(?:ECONNREFUSED|ECONNRESET|ETIMEDOUT|TIMEOUT)\b/.test( + error instanceof Error ? error.message : String(error), + ) || + attempt === 3 + ) { + throw error; + } + await (pause ?? + (() => new Promise((resolve) => setTimeout(resolve, 500))))(); + } + } + throw lastError; +} + async function waitForExecutorProviderEgress({ fixture, adminImage, @@ -1606,14 +1630,16 @@ async function main() { let observedProviderRequests = 0; let evidenceSequence = 0; const observeProvider = async (pod, sourceNodeName) => { - evidenceSequence += 1; applyExecutorNetworkPolicy(fixture, pod.status.podIP); - const evidence = await providerEvidence({ - fixture, - adminImage, - name: `ql3-provider-evidence-${evidenceSequence}`, - nodeName: sourceNodeName, - providerPodIp: pod.status.podIP, + const evidence = await retryProviderEvidence(async () => { + evidenceSequence += 1; + return providerEvidence({ + fixture, + adminImage, + name: `ql3-provider-evidence-${evidenceSequence}`, + nodeName: sourceNodeName, + providerPodIp: pod.status.podIP, + }); }); const count = evidence.requestCount; const observationKey = providerObservationKey(pod); @@ -2064,5 +2090,6 @@ module.exports = { executorJob, providerObservationKey, providerServerSource, + retryProviderEvidence, terminalJobSnapshot, }; diff --git a/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs b/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs index 426730f2..1733d9af 100644 --- a/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs +++ b/test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs @@ -72,6 +72,16 @@ test('report migration evidence follows the complete PostgreSQL stream', () => { ); }); +test('waits for PostgreSQL Service connectivity before migration', () => { + const readyProbe = live.indexOf('waitForPostgresService();'); + const migration = live.indexOf( + "apply(migrationJob(), 'create reviewed migration Job')", + ); + assert.ok(readyProbe > 0); + assert.ok(migration > readyProbe); + assert.match(live, /'pg_isready',[\s\S]*'--host',[\s\S]*POSTGRES_NAME/); +}); + test('fixture locks are bound to durable version-three approval dispatches', () => { const value = createFixture({ registry: 'registry.fixture.test', diff --git a/test/back/ql3ProviderCredentialTestKubernetesLiveContract.test.cjs b/test/back/ql3ProviderCredentialTestKubernetesLiveContract.test.cjs index 615de2a0..33c4e77d 100644 --- a/test/back/ql3ProviderCredentialTestKubernetesLiveContract.test.cjs +++ b/test/back/ql3ProviderCredentialTestKubernetesLiveContract.test.cjs @@ -14,6 +14,7 @@ const { executorJob, providerObservationKey, providerServerSource, + retryProviderEvidence, terminalJobSnapshot, } = require('../../scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs'); const { @@ -363,6 +364,29 @@ test('starts a fresh provider request baseline after a container restart', () => assert.equal(providerObservationKey(pod), 'provider-uid:1'); }); +test('retries only bounded transient provider evidence failures', async () => { + let attempts = 0; + const evidence = await retryProviderEvidence( + async () => { + attempts += 1; + if (attempts < 3) throw new Error('{"code":"ECONNREFUSED"}'); + return { requestCount: 1 }; + }, + async () => {}, + ); + assert.deepEqual(evidence, { requestCount: 1 }); + assert.equal(attempts, 3); + await assert.rejects( + retryProviderEvidence( + async () => { + throw new Error('invalid evidence schema'); + }, + async () => {}, + ), + /invalid evidence schema/, + ); +}); + test('provider fixture logs only generation and authorization decision', () => { const source = providerServerSource(); assert.match(source, /event:'provider_request',generation,allowed/);