feat(ql3): add cluster run management live gate

scripts/lib/ql3-management-kubernetes-live-platform.cjs

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const assert = require('node:assert/strict');
+
+function imageIdDigest(image) {
+  assert.match(image.Id, /^sha256:[a-f0-9]{64}$/);
+  return image.Id;
+}
+
+function localManifest(rendered, imageName, localImage) {
+  const occurrences = rendered.split(imageName).length - 1;
+  assert.ok(occurrences >= 1, 'reviewed image reference is missing');
+  return rendered
+    .replaceAll(imageName, localImage)
+    .replaceAll('imagePullPolicy: IfNotPresent', 'imagePullPolicy: Never');
+}
+
+function applySecret(fixture, name, type, stringData) {
+  fixture.apply({
+    apiVersion: 'v1',
+    kind: 'Secret',
+    metadata: {
+      name,
+      namespace: 'qinglong3-system',
+      labels: { 'cnpg.io/reload': 'true' },
+    },
+    immutable: false,
+    type,
+    stringData,
+  });
+}
+
+function psql(fixture, podName, sql) {
+  return fixture.kubectl(
+    [
+      '-n',
+      'qinglong3-system',
+      'exec',
+      podName,
+      '-c',
+      'postgres',
+      '--',
+      'psql',
+      '--username=postgres',
+      '--dbname=qinglong',
+      '--no-psqlrc',
+      '--tuples-only',
+      '--no-align',
+      '--set=ON_ERROR_STOP=1',
+      '--command',
+      sql,
+    ],
+    { capture: true, quiet: true },
+  ).stdout;
+}
+
+function currentPrimaryPod(fixture) {
+  const primaryName = fixture.kubectlJson([
+    '-n',
+    'qinglong3-system',
+    'get',
+    'cluster',
+    'ql3-postgres',
+  ]).status.currentPrimary;
+  assert.match(primaryName || '', /^ql3-postgres-[1-9][0-9]*$/);
+  const pods = fixture.kubectlJson([
+    '-n',
+    'qinglong3-system',
+    'get',
+    'pods',
+    '-l',
+    'cnpg.io/cluster=ql3-postgres',
+  ]).items;
+  const primary = pods.find((pod) => pod.metadata.name === primaryName);
+  assert.ok(primary, 'CloudNativePG primary Pod not found');
+  return primary;
+}
+
+module.exports = {
+  applySecret,
+  currentPrimaryPod,
+  imageIdDigest,
+  localManifest,
+  psql,
+};

scripts/lib/ql3-management-kubernetes-live.cjs

@@ -100,9 +100,7 @@ async function waitForTwoPreserved(options) {
         '-l',
         'app.kubernetes.io/name=' + options.deployment,
       ])
-      .items.filter(
-        (pod) => pod.metadata.deletionTimestamp === undefined,
-      );
+      .items.filter((pod) => pod.metadata.deletionTimestamp === undefined);
     const ready = pods.filter(podReady);
     minimumReady = Math.min(minimumReady, ready.length);
     const replacements = ready.filter(
@@ -117,10 +115,7 @@ async function waitForTwoPreserved(options) {
       : {
           ready: false,
           fact:
-            ready.length +
-            ' ready, ' +
-            replacements.length +
-            ' replacements',
+            ready.length + ' ready, ' + replacements.length + ' replacements',
         };
   });
   assert.ok(
@@ -131,6 +126,17 @@ async function waitForTwoPreserved(options) {
 }
 
 function createManagementClientExecutor(options) {
+  const retryableClientCodes = options.retryableClientCodes ?? [
+    'QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED',
+  ];
+  assert.ok(
+    Array.isArray(retryableClientCodes) &&
+      retryableClientCodes.length >= 1 &&
+      retryableClientCodes.every(
+        (code) => typeof code === 'string' && /^[A-Z0-9_]{1,128}$/.test(code),
+      ),
+    'management client retryable codes are invalid',
+  );
   options.fixture.apply({
     apiVersion: 'v1',
     kind: 'ServiceAccount',
@@ -243,9 +249,14 @@ function createManagementClientExecutor(options) {
                       '--command=/tmp/command.json ' +
                       '--assertion=/tmp/assertion.jwt 2>&1)"',
                     '  status=$?',
-                    '  if [ "$status" -eq 0 ] || { ! printf \'%s\' "$output" | ' +
-                      'grep -q QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED && ' +
-                      '! printf \'%s\' "$output" | grep -q \'"statusCode":503\'; } || ' +
+                    '  if [ "$status" -eq 0 ] || { ' +
+                      retryableClientCodes
+                        .map(
+                          (code) =>
+                            '! printf \'%s\' "$output" | grep -q ' + code,
+                        )
+                        .join(' && ') +
+                      ' && ! printf \'%s\' "$output" | grep -q \'"statusCode":503\'; } || ' +
                       '[ "$attempt" -ge 60 ]; then',
                     '    break',
                     '  fi',
@@ -373,7 +384,10 @@ function createManagementClientExecutor(options) {
       assert.equal(output.event, 'command_completed');
       assert.equal(output.result.operation, definition.command.operation);
       if (expected.resultStatus) {
-        assert.ok(expected.resultStatus.includes(output.result.status));
+        const status = expected.resultField
+          ? output.result[expected.resultField]?.status
+          : output.result.status;
+        assert.ok(expected.resultStatus.includes(status));
       }
     } else {
       assert.equal(
@@ -409,6 +423,36 @@ function createManagementClientExecutor(options) {
   };
 }
 
+function managementHealthStatus(options) {
+  const script = [
+    "const fs=require('node:fs');const https=require('node:https');",
+    "const request=https.request({host:'127.0.0.1',port:Number(process.argv[1]),path:process.argv[2],",
+    'servername:process.argv[3],ca:fs.readFileSync(process.argv[4]),',
+    "minVersion:'TLSv1.3',maxVersion:'TLSv1.3',rejectUnauthorized:true,agent:false},",
+    "(response)=>{response.resume();response.on('end',()=>process.stdout.write(String(response.statusCode)))});",
+    "request.on('error',(error)=>{process.stderr.write(error.message);process.exitCode=1});request.end();",
+  ].join('\n');
+  return Number(
+    options.fixture.kubectl(
+      [
+        '-n',
+        options.namespace,
+        'exec',
+        options.podName,
+        '--',
+        'node',
+        '-e',
+        script,
+        String(options.port),
+        options.route,
+        options.servername,
+        options.caFile,
+      ],
+      { capture: true, quiet: true },
+    ).stdout,
+  );
+}
+
 function podTcpProbe(options) {
   const script = [
     "const net=require('node:net');let finished=false;",
@@ -437,9 +481,7 @@ function podTcpProbe(options) {
 async function clientTcpProbe(options) {
   const labels = {
     'app.kubernetes.io/name': options.appName,
-    ...(options.labelled
-      ? { [options.networkPolicyLabel]: 'true' }
-      : {}),
+    ...(options.labelled ? { [options.networkPolicyLabel]: 'true' } : {}),
   };
   const script = [
     "const fs=require('node:fs');const net=require('node:net');let finished=false;let attempt=0;let socket;",
@@ -497,30 +539,25 @@ async function clientTcpProbe(options) {
       },
     },
   });
-  const observed = await waitFor(
-    options.name + ' completion',
-    180_000,
-    () => {
-      const job = options.fixture.kubectlJson([
-        '-n',
-        options.namespace,
-        'get',
-        'job',
-        options.name,
-      ]);
-      const complete = job.status.conditions?.some(
-        (condition) =>
-          condition.type === 'Complete' && condition.status === 'True',
-      );
-      const failed = job.status.conditions?.some(
-        (condition) =>
-          condition.type === 'Failed' && condition.status === 'True',
-      );
-      return complete || failed
-        ? { ready: true, value: { complete, failed } }
-        : { ready: false, fact: JSON.stringify(job.status ?? {}) };
-    },
-  );
+  const observed = await waitFor(options.name + ' completion', 180_000, () => {
+    const job = options.fixture.kubectlJson([
+      '-n',
+      options.namespace,
+      'get',
+      'job',
+      options.name,
+    ]);
+    const complete = job.status.conditions?.some(
+      (condition) =>
+        condition.type === 'Complete' && condition.status === 'True',
+    );
+    const failed = job.status.conditions?.some(
+      (condition) => condition.type === 'Failed' && condition.status === 'True',
+    );
+    return complete || failed
+      ? { ready: true, value: { complete, failed } }
+      : { ready: false, fact: JSON.stringify(job.status ?? {}) };
+  });
   const probePod = (
     await waitFor(options.name + ' terminal pod', 30_000, () => {
       const pods = options.fixture.kubectlJson([
@@ -541,16 +578,8 @@ async function clientTcpProbe(options) {
   const terminated = probePod.status.containerStatuses[0].state.terminated;
   const observation =
     options.name + ': ' + (terminated.message ?? 'no-message');
-  assert.equal(
-    observed.value.complete,
-    options.expectedConnected,
-    observation,
-  );
-  assert.equal(
-    observed.value.failed,
-    !options.expectedConnected,
-    observation,
-  );
+  assert.equal(observed.value.complete, options.expectedConnected, observation);
+  assert.equal(observed.value.failed, !options.expectedConnected, observation);
   assert.equal(
     terminated.exitCode === 0,
     options.expectedConnected,
@@ -562,14 +591,7 @@ async function clientTcpProbe(options) {
     assert.match(terminated.message ?? '', /^denied:/);
   }
   options.fixture.kubectl(
-    [
-      '-n',
-      options.namespace,
-      'delete',
-      'job',
-      options.name,
-      '--wait=false',
-    ],
+    ['-n', options.namespace, 'delete', 'job', options.name, '--wait=false'],
     { capture: true, quiet: true },
   );
   return options.expectedConnected
@@ -580,6 +602,7 @@ async function clientTcpProbe(options) {
 module.exports = {
   clientTcpProbe,
   createManagementClientExecutor,
+  managementHealthStatus,
   patchManagementGeneration,
   podReady,
   podTcpProbe,

scripts/lib/ql3-management-live-identity.cjs

@@ -57,7 +57,8 @@ function createManagementIdentityCeremony(options) {
     });
   }
 
-  function assertion(key, suffix = crypto.randomUUID()) {
+  function assertionForSubject(key, subject, suffix = crypto.randomUUID()) {
+    assert.match(subject, /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/);
     const now = Math.floor(Date.now() / 1_000);
     const header = Buffer.from(
       JSON.stringify({
@@ -77,6 +78,43 @@ function createManagementIdentityCeremony(options) {
         iss: options.issuer,
         jti: options.jtiPrefix + '-' + suffix,
         ql3_purpose: options.purpose,
+        sub: subject,
+      }),
+    ).toString('base64url');
+    const signed = header + '.' + payload;
+    return (
+      signed +
+      '.' +
+      crypto
+        .sign(null, Buffer.from(signed, 'ascii'), key.privateKey)
+        .toString('base64url')
+    );
+  }
+
+  function assertion(key, suffix = crypto.randomUUID()) {
+    return assertionForSubject(key, options.subject, suffix);
+  }
+
+  function weakAssertion(key, suffix = crypto.randomUUID()) {
+    const now = Math.floor(Date.now() / 1_000);
+    const header = Buffer.from(
+      JSON.stringify({
+        alg: 'EdDSA',
+        kid: key.kid,
+        typ: options.tokenType,
+      }),
+    ).toString('base64url');
+    const payload = Buffer.from(
+      JSON.stringify({
+        acr: 'urn:ql3:password',
+        amr: ['pwd'],
+        aud: options.audience,
+        auth_time: now - 1,
+        exp: now + 290,
+        iat: now,
+        iss: options.issuer,
+        jti: options.jtiPrefix + '-weak-' + suffix,
+        ql3_purpose: options.purpose,
         sub: options.subject,
       }),
     ).toString('base64url');
@@ -90,7 +128,13 @@ function createManagementIdentityCeremony(options) {
     );
   }
 
-  return Object.freeze({ assertion, keyset, reviewedKey });
+  return Object.freeze({
+    assertion,
+    assertionForSubject,
+    keyset,
+    reviewedKey,
+    weakAssertion,
+  });
 }
 
 module.exports = { createManagementIdentityCeremony };

scripts/lib/ql3-run-management-kubernetes-live-scenario.cjs

@@ -0,0 +1,194 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const assert = require('node:assert/strict');
+const crypto = require('node:crypto');
+
+function eventId(ordinal) {
+  assert.ok(Number.isSafeInteger(ordinal) && ordinal >= 1 && ordinal < 1e12);
+  return '41000000-0000-4000-8000-' + String(ordinal).padStart(12, '0');
+}
+
+function retryCommand(projectId, sourceRunId, requestId, mutationId, ordinal) {
+  return Object.freeze({
+    schemaVersion: 1,
+    operation: 'run.retry',
+    request: Object.freeze({
+      projectId,
+      sourceRunId,
+      requestId,
+      auditEventId: eventId(ordinal),
+      failureAuditEventId: eventId(ordinal + 500_000),
+      body: Object.freeze({
+        schema: 'qinglong/run-manual-retry@v1',
+        mutationId,
+        expectedRunVersion: 3,
+        expectedRunStatus: 'failed',
+      }),
+    }),
+  });
+}
+
+function stopCommand(projectId, runId, requestId, mutationId, ordinal) {
+  return Object.freeze({
+    schemaVersion: 1,
+    operation: 'run.stop',
+    request: Object.freeze({
+      projectId,
+      runId,
+      requestId,
+      auditEventId: eventId(ordinal),
+      failureAuditEventId: eventId(ordinal + 500_000),
+      body: Object.freeze({
+        schema: 'qinglong/run-cancellation@v1',
+        mutationId,
+      }),
+    }),
+  });
+}
+
+function sqlString(value) {
+  assert.equal(typeof value, 'string');
+  return "'" + value.replaceAll("'", "''") + "'";
+}
+
+function seedRunManagement(fixture, podName, values, psql) {
+  const nowMs = Date.now();
+  const sourceDigest = 'a'.repeat(64);
+  const taskRevision = `qltd:v1:1:${sourceDigest}`;
+  const taskDigest = 'b'.repeat(64);
+  const planDigest = 'c'.repeat(64);
+  psql(
+    fixture,
+    podName,
+    [
+      'BEGIN;',
+      'INSERT INTO "ql3"."projects" (id, name, slug, status, version, created_at_ms, updated_at_ms)',
+      `VALUES (${sqlString(
+        values.projectId,
+      )}, 'Run Management Live', ${sqlString(
+        values.projectId,
+      )}, 'active', 1, ${nowMs}, ${nowMs});`,
+      'INSERT INTO "ql3"."project_role_bindings" (project_id, subject_type, subject_id, version, state, role, mutation_id, changed_by_type, changed_by_id, created_at_ms)',
+      `VALUES (${sqlString(values.projectId)}, 'user', ${sqlString(
+        values.operatorId,
+      )}, 1, 'active', 'operator', ${sqlString(
+        'binding-' + values.suffix,
+      )}, 'system', 'run-management-live', ${nowMs});`,
+      'INSERT INTO "ql3"."task_definitions" (project_id, task_id, current_revision, created_at_ms, updated_at_ms)',
+      `VALUES (${sqlString(values.projectId)}, ${sqlString(
+        values.taskId,
+      )}, 1, ${nowMs}, ${nowMs});`,
+      'INSERT INTO "ql3"."task_definition_revisions" (project_id, task_id, revision, mutation_id, name, kind, spec_json, labels_json, enabled, content_digest, created_at_ms)',
+      `VALUES (${sqlString(values.projectId)}, ${sqlString(
+        values.taskId,
+      )}, 1, ${sqlString(
+        crypto.randomUUID(),
+      )}::uuid, 'Run Management Live Task', 'command', '{"schema":"qinglong/command@v1","config":{"command":{"kind":"argv","file":"/bin/echo","args":["run-management-live"]}}}'::jsonb, '{}'::jsonb, true, ${sqlString(
+        taskDigest,
+      )}, ${nowMs});`,
+      'INSERT INTO "ql3"."task_execution_revisions" (project_id, task_id, source_revision, task_revision, source_content_digest, executor_type, plan_schema, plan_json, content_digest, created_at_ms)',
+      `VALUES (${sqlString(values.projectId)}, ${sqlString(
+        values.taskId,
+      )}, 1, ${sqlString(taskRevision)}, ${sqlString(
+        sourceDigest,
+      )}, 'remote_worker', 'qinglong/command-execution@v1', '{"file":"/bin/echo","args":["run-management-live"]}'::jsonb, ${sqlString(
+        planDigest,
+      )}, ${nowMs});`,
+      'INSERT INTO "ql3"."runs" (id, project_id, task_id, task_revision, task_name, task_snapshot_ref, trigger_type, execution_origin, execution_owner, status, version, event_sequence, priority, created_at_ms, queued_at_ms, finished_at_ms, error_code, error_summary)',
+      `VALUES (${sqlString(values.sourceRunId)}, ${sqlString(
+        values.projectId,
+      )}, ${sqlString(values.taskId)}, ${sqlString(
+        taskRevision,
+      )}, 'Run Management Live Task', ${sqlString(
+        taskRevision,
+      )}, 'manual', 'manual', 'runtime', 'failed', 3, 3, 0, ${nowMs}, ${nowMs}, ${nowMs}, 'LIVE_SOURCE_FAILURE', 'terminal source for Run management live');`,
+      'INSERT INTO "ql3"."run_attempts" (id, run_id, attempt, status, executor_type, callback_sequence, created_at_ms, finished_at_ms, error_code, error_summary)',
+      `VALUES (${sqlString(values.sourceAttemptId)}, ${sqlString(
+        values.sourceRunId,
+      )}, 1, 'failed', 'remote_worker', 0, ${nowMs}, ${nowMs}, 'LIVE_SOURCE_FAILURE', 'terminal source for Run management live');`,
+      'INSERT INTO "ql3"."run_events" (id, run_id, sequence, type, dedupe_key, actor_type, actor_id, attempt_id, payload, created_at_ms)',
+      `VALUES (${sqlString(crypto.randomUUID())}, ${sqlString(
+        values.sourceRunId,
+      )}, 1, 'run.created', ${sqlString(
+        'run-management-live-created-' + values.suffix,
+      )}, 'user', ${sqlString(values.operatorId)}, ${sqlString(
+        values.sourceAttemptId,
+      )}, '{"status":"created","version":1}'::jsonb, ${nowMs}),`,
+      `(${sqlString(crypto.randomUUID())}, ${sqlString(
+        values.sourceRunId,
+      )}, 2, 'run.queued', ${sqlString(
+        'run-management-live-queued-' + values.suffix,
+      )}, 'user', ${sqlString(values.operatorId)}, ${sqlString(
+        values.sourceAttemptId,
+      )}, '{"from_status":"created","to_status":"queued","version":2}'::jsonb, ${nowMs}),`,
+      `(${sqlString(crypto.randomUUID())}, ${sqlString(
+        values.sourceRunId,
+      )}, 3, 'run.failed', ${sqlString(
+        'run-management-live-failed-' + values.suffix,
+      )}, 'executor', 'run-management-live', ${sqlString(
+        values.sourceAttemptId,
+      )}, '{"from_status":"queued","to_status":"failed","version":3,"error_code":"LIVE_SOURCE_FAILURE"}'::jsonb, ${nowMs});`,
+      'COMMIT;',
+    ].join('\n'),
+  );
+  return Object.freeze({ taskRevision, taskDigest, planDigest });
+}
+
+function durableRunManagementFacts(fixture, podName, values, psql) {
+  return JSON.parse(
+    psql(
+      fixture,
+      podName,
+      [
+        'SELECT json_build_object(',
+        '  \'sourceRunStatus\', (SELECT status FROM "ql3"."runs" WHERE id = ' +
+          sqlString(values.sourceRunId) +
+          '),',
+        '  \'retryRunCount\', (SELECT count(*)::integer FROM "ql3"."runs" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          " AND trigger_type = 'run_manual_retry'),",
+        '  \'retryAttemptCount\', (SELECT count(*)::integer FROM "ql3"."run_attempts" AS attempt JOIN "ql3"."runs" AS run ON run.id = attempt.run_id WHERE run.project_id = ' +
+          sqlString(values.projectId) +
+          " AND run.trigger_type = 'run_manual_retry'),",
+        '  \'retryEventCount\', (SELECT count(*)::integer FROM "ql3"."run_events" AS event JOIN "ql3"."runs" AS run ON run.id = event.run_id WHERE run.project_id = ' +
+          sqlString(values.projectId) +
+          " AND run.trigger_type = 'run_manual_retry' AND event.type IN ('run.created', 'run.queued')),",
+        '  \'stoppedRunCount\', (SELECT count(*)::integer FROM "ql3"."runs" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          " AND trigger_type = 'run_manual_retry' AND cancel_requested_at_ms IS NOT NULL AND cancel_reason = 'user'),",
+        '  \'stopEventCount\', (SELECT count(*)::integer FROM "ql3"."run_events" AS event JOIN "ql3"."runs" AS run ON run.id = event.run_id WHERE run.project_id = ' +
+          sqlString(values.projectId) +
+          " AND event.type = 'run.cancel_requested'),",
+        '  \'allowedAuditCount\', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          " AND operation_id IN ('run.retry', 'run.stop') AND outcome = 'allowed'),",
+        '  \'deniedAuditCount\', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          " AND operation_id IN ('run.retry', 'run.stop') AND outcome = 'denied'),",
+        '  \'weakAuthenticationAuditCount\', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          " AND request_id = 'run-live-weak'),",
+        '  \'duplicateMutationCount\', greatest(0, (SELECT count(*)::integer FROM "ql3"."runs" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          ' AND trigger_type = \'run_manual_retry\') - 1) + greatest(0, (SELECT count(*)::integer FROM "ql3"."run_events" AS event JOIN "ql3"."runs" AS run ON run.id = event.run_id WHERE run.project_id = ' +
+          sqlString(values.projectId) +
+          " AND event.type = 'run.cancel_requested') - 1),",
+        '  \'identityGeneration\', (SELECT generation::integer FROM "ql3"."plugin_package_identity_keyset_ledger" WHERE authority = \'run-management\'),',
+        '  \'migrationCount\', (SELECT count(*)::integer FROM "ql3"."schema_migrations"),',
+        '  \'controlCoreCapability\', (SELECT contract_version::integer FROM "ql3"."schema_capabilities" WHERE contract_name = \'control-core\'),',
+        "  'postgresVersionNumber', current_setting('server_version_num')::integer)",
+      ].join('\n'),
+    ),
+  );
+}
+
+module.exports = {
+  durableRunManagementFacts,
+  eventId,
+  retryCommand,
+  seedRunManagement,
+  sqlString,
+  stopCommand,
+};