fix(ci): wait for live service readiness

scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs

@@ -1098,6 +1098,36 @@ function recoveryResources(fixture, jobName) {
   ];
 }
 
+function waitForPostgresService(timeoutMs = 60_000) {
+  const deadline = Date.now() + timeoutMs;
+  let lastFact = 'probe not started';
+  while (Date.now() < deadline) {
+    const probe = kubectl(
+      [
+        '-n',
+        NAMESPACE,
+        'exec',
+        `pod/${POSTGRES_NAME}`,
+        '--',
+        'pg_isready',
+        '--host',
+        POSTGRES_NAME,
+        '--port',
+        '5432',
+        '--username',
+        'postgres',
+        '--dbname',
+        'qinglong',
+      ],
+      { capture: true, quiet: true, allowFailure: true },
+    );
+    if (probe.status === 0) return;
+    lastFact = (probe.stderr || probe.stdout || `status ${probe.status}`).trim();
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1_000);
+  }
+  fail(`PostgreSQL Service was not ready after ${timeoutMs}ms: ${lastFact}`);
+}
+
 function waitForJob(
   name,
   expectedStatus = 'complete',
@@ -1795,6 +1825,7 @@ async function main(argv = process.argv.slice(2)) {
       ],
       { capture: true, quiet: true },
     );
+    waitForPostgresService();
     kubectl(
       [
         '-n',

scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs

@@ -1074,6 +1074,30 @@ request.on('timeout',()=>request.destroy(Object.assign(new Error('timeout'),{cod
   return evidence;
 }
 
+async function retryProviderEvidence(read, pause = undefined) {
+  assert.equal(typeof read, 'function');
+  assert.ok(pause === undefined || typeof pause === 'function');
+  let lastError;
+  for (let attempt = 1; attempt <= 3; attempt += 1) {
+    try {
+      return await read();
+    } catch (error) {
+      lastError = error;
+      if (
+        !/\b(?:ECONNREFUSED|ECONNRESET|ETIMEDOUT|TIMEOUT)\b/.test(
+          error instanceof Error ? error.message : String(error),
+        ) ||
+        attempt === 3
+      ) {
+        throw error;
+      }
+      await (pause ??
+        (() => new Promise((resolve) => setTimeout(resolve, 500))))();
+    }
+  }
+  throw lastError;
+}
+
 async function waitForExecutorProviderEgress({
   fixture,
   adminImage,
@@ -1606,14 +1630,16 @@ async function main() {
     let observedProviderRequests = 0;
     let evidenceSequence = 0;
     const observeProvider = async (pod, sourceNodeName) => {
-      evidenceSequence += 1;
       applyExecutorNetworkPolicy(fixture, pod.status.podIP);
-      const evidence = await providerEvidence({
-        fixture,
-        adminImage,
-        name: `ql3-provider-evidence-${evidenceSequence}`,
-        nodeName: sourceNodeName,
-        providerPodIp: pod.status.podIP,
+      const evidence = await retryProviderEvidence(async () => {
+        evidenceSequence += 1;
+        return providerEvidence({
+          fixture,
+          adminImage,
+          name: `ql3-provider-evidence-${evidenceSequence}`,
+          nodeName: sourceNodeName,
+          providerPodIp: pod.status.podIP,
+        });
       });
       const count = evidence.requestCount;
       const observationKey = providerObservationKey(pod);
@@ -2064,5 +2090,6 @@ module.exports = {
   executorJob,
   providerObservationKey,
   providerServerSource,
+  retryProviderEvidence,
   terminalJobSnapshot,
 };

test/back/ql3PluginPackageRecoveryE2ELiveContract.test.cjs

@@ -72,6 +72,16 @@ test('report migration evidence follows the complete PostgreSQL stream', () => {
   );
 });
 
+test('waits for PostgreSQL Service connectivity before migration', () => {
+  const readyProbe = live.indexOf('waitForPostgresService();');
+  const migration = live.indexOf(
+    "apply(migrationJob(), 'create reviewed migration Job')",
+  );
+  assert.ok(readyProbe > 0);
+  assert.ok(migration > readyProbe);
+  assert.match(live, /'pg_isready',[\s\S]*'--host',[\s\S]*POSTGRES_NAME/);
+});
+
 test('fixture locks are bound to durable version-three approval dispatches', () => {
   const value = createFixture({
     registry: 'registry.fixture.test',

test/back/ql3ProviderCredentialTestKubernetesLiveContract.test.cjs

@@ -14,6 +14,7 @@ const {
   executorJob,
   providerObservationKey,
   providerServerSource,
+  retryProviderEvidence,
   terminalJobSnapshot,
 } = require('../../scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs');
 const {
@@ -363,6 +364,29 @@ test('starts a fresh provider request baseline after a container restart', () =>
   assert.equal(providerObservationKey(pod), 'provider-uid:1');
 });
 
+test('retries only bounded transient provider evidence failures', async () => {
+  let attempts = 0;
+  const evidence = await retryProviderEvidence(
+    async () => {
+      attempts += 1;
+      if (attempts < 3) throw new Error('{"code":"ECONNREFUSED"}');
+      return { requestCount: 1 };
+    },
+    async () => {},
+  );
+  assert.deepEqual(evidence, { requestCount: 1 });
+  assert.equal(attempts, 3);
+  await assert.rejects(
+    retryProviderEvidence(
+      async () => {
+        throw new Error('invalid evidence schema');
+      },
+      async () => {},
+    ),
+    /invalid evidence schema/,
+  );
+});
+
 test('provider fixture logs only generation and authorization decision', () => {
   const source = providerServerSource();
   assert.match(source, /event:'provider_request',generation,allowed/);