fix(ci): make ql3 gates reproducible

packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts

@@ -122,6 +122,20 @@ const MAX_AUTHORIZATION_BYTES = 16 * 1024;
 const MAX_RESPONSE_BYTES = 128 * 1024;
 const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
 
+function tlsPemBlocks(
+  bytes: Buffer,
+  label: 'CERTIFICATE' | 'X509 CRL',
+): Buffer[] {
+  const pattern = new RegExp(
+    `-----BEGIN ${label}-----[\\s\\S]*?-----END ${label}-----`,
+    'g',
+  );
+  const value = bytes.toString('utf8');
+  const matches = value.match(pattern);
+  if (!matches || value.replace(pattern, '').trim() !== '') return [bytes];
+  return matches.map((match) => Buffer.from(`${match}\n`, 'utf8'));
+}
+
 export interface ClusterPluginPackageManagementHttpLimits {
   readonly maxBodyBytes?: number;
   readonly maxConnections?: number;
@@ -742,6 +756,18 @@ export async function startClusterPluginPackageManagementHttp(
   const createRequestId = options.createRequestId ?? randomUUID;
   const clientCertificateRequired =
     options.tls.clientCertificateAuthority !== undefined;
+  const clientTrust = clientCertificateRequired
+    ? {
+        ca: tlsPemBlocks(
+          options.tls.clientCertificateAuthority as Buffer,
+          'CERTIFICATE',
+        ),
+        crl: tlsPemBlocks(
+          options.tls.clientCertificateRevocationList as Buffer,
+          'X509 CRL',
+        ),
+      }
+    : undefined;
   const rateLimiter = new BoundedRateLimiter(limits, now);
   let availability: 'ready' | 'unavailable' | 'stopped' = 'ready';
   let inFlight = 0;
@@ -755,12 +781,7 @@ export async function startClusterPluginPackageManagementHttp(
       minVersion: 'TLSv1.3',
       maxVersion: 'TLSv1.3',
       honorCipherOrder: true,
-      ...(clientCertificateRequired
-        ? {
-            ca: options.tls.clientCertificateAuthority,
-            crl: options.tls.clientCertificateRevocationList,
-          }
-        : {}),
+      ...(clientTrust ?? {}),
       requestCert: clientCertificateRequired,
       // Health probes intentionally remain reachable without a client
       // certificate. Every non-health route checks TLSSocket.authorized before

packages/ql3-cluster-admin/test/fixtures/next-client-ca-cert.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDOTCCAiGgAwIBAgIUfb1aCt8o4WAshKLJue/oY/vt58UwDQYJKoZIhvcNAQEL
+BQAwJDEiMCAGA1UEAwwZUWluZ0xvbmctMy1OZXh0LUNsaWVudC1DQTAeFw0yNjA4
+MjExNzMxMjRaFw0zNjA4MTgxNzMxMjRaMCQxIjAgBgNVBAMMGVFpbmdMb25nLTMt
+TmV4dC1DbGllbnQtQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCV
+KV+ZRNKOR+GrPibLTsgOz5cm42/9TfwbZX0NQ2pVQ6X00BcpRqSlIJ7qtro6yiZ0
+lvvPisbmjJYWFB9OmOqjmZT2lGi/oc2UrpzZCfBTVuplNMMvKye/WbN7lOXp5KM2
+Fzm1+sRRWZyyCGlEtoyiBFS/0nr9JGm060CvIGfRaC8GyxbFogvsDUWt3qwDh2wN
+UHLBePF73HWYTEd2YX22nYbDUPVUInfeUmMdcJ6kTYlyDxwNk3HFh7inJjwC5c6H
+kAb9tWP73llZbvGNvqubXauJrIvCEOSKPrqe2Z36F4qDRQ93JbNN9z35I6/4j1nP
+EWeAEDHmQyxzjENA2hpRAgMBAAGjYzBhMB0GA1UdDgQWBBRJ51y34GhuN+2O6++G
+XbtzteCitjAfBgNVHSMEGDAWgBRJ51y34GhuN+2O6++GXbtzteCitjAPBgNVHRMB
+Af8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEAYHoc
+MjXSLEfYXtzHU7hsjOtITjyK8lofXN4dTHU0ayKQCH9XuUBcSnglW+EvbGJ/zTpk
+eceQpCqAuhGW2gnj7+kjtf/GYq/aVBwyHwZvDUB7GZX7suLJqxm0rZSomO/yB8H2
+Shm9iAJ/gb5WQeBW22XWEMAXivqnqt+h3GwqRsrDDySqEJEmRpl7SJRuAIDauFv6
+OVlEB4r29nNGgx9xUMgY0ZMB5dwG7hCTvllmR4NpGfsKUqScjpOys+MrOY6DHyRD
+57W/HkKVR7TP/GIyvAfWEFX8S/0efY2gNV6nAG7YYbD1i4UW5LWRyoiNGYORGhse
+wCTYki14yDV+dRgCtw==
+-----END CERTIFICATE-----

packages/ql3-cluster-admin/test/fixtures/next-client-cert.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDNTCCAh2gAwIBAgIDMQABMA0GCSqGSIb3DQEBCwUAMCQxIjAgBgNVBAMMGVFp
+bmdMb25nLTMtTmV4dC1DbGllbnQtQ0EwHhcNMjYwODIxMTczMTI0WhcNMzYwODE4
+MTczMTI0WjAfMR0wGwYDVQQDDBRxbDMtbmV4dC10ZXN0LXdvcmtlcjCCASIwDQYJ
+KoZIhvcNAQEBBQADggEPADCCAQoCggEBALQPFNYXn/69p1q0rop2hpecZpNgg4hc
+n4VFYadYXZf8xAkGSXdBrxyVu2k2Z5bcboxSztm/W1qANJUX7sgJrRhTj2iGILX2
+do9BUdx8lstYppWzQjE1LtYrHqQdtF759dTMVQ5T0iPgROB0sTeQdMH/3Srki6E+
+7V/6Crk4IvJNWwQnMm4S20y+AY4d/ebtbkU7yA1md9aDZ+m/fKceO5Mro0caM9t4
+3cY9XI47KwVSOGZB9WiOhubMnH8cAmPxsgyola9lWsshW0NAzRG/d5xdSmYHfJMd
+w5MX8d52xWUlPpMDZGvLRFauYCA+NynbJxl3Keg0eSqgxa/NMBAa4kECAwEAAaN1
+MHMwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB
+BQUHAwIwHQYDVR0OBBYEFM1aBVCbqzCtxpsRliCwk3NyF8POMB8GA1UdIwQYMBaA
+FEnnXLfgaG437Y7r74Zdu3O14KK2MA0GCSqGSIb3DQEBCwUAA4IBAQAJVWhijDA3
+JdMKDag2mxZX5wYCGeWWXIUvoo3kkqV1sSoD2w4i0K0D49UucAFqcKjUZG9iXbuE
+m29AZGsm6HV7h490fzI3HY3VUGfjkMO5ikDxXuO3ePbDYwZb2W3CRiwFLclCGBoC
+RDXh8qPvcBdtT6NSn/dzGJq+2nyntKWTcaO+JjdARYLO8wbQWcqMldSvFICHDikd
+eKiyYd26z6x/cEOdGtWaB2OJ37bNcaVgRcA/bvQ+L3wF5WmAeO+ptNA0FRSXolN8
+bY7dXPHFj6zYQ5paSYazT5LqTO6Ga1X4s4KC018aSyYE0QTqGnML9nxxmNw3S+uE
+wFhYDiTuyhAb
+-----END CERTIFICATE-----

packages/ql3-cluster-admin/test/fixtures/next-client-empty-crl.pem

@@ -0,0 +1,11 @@
+-----BEGIN X509 CRL-----
+MIIBfTBnAgEBMA0GCSqGSIb3DQEBCwUAMCQxIjAgBgNVBAMMGVFpbmdMb25nLTMt
+TmV4dC1DbGllbnQtQ0EXDTI2MDgyMTE3MzEyNFoXDTM2MDgxODE3MzEyNFqgDzAN
+MAsGA1UdFAQEAgIQADANBgkqhkiG9w0BAQsFAAOCAQEARwTg7QvukmIloA61T/rJ
+WKc5izTclZh/2YTmBi25FodlhZISnBquHEsCwpqoerXOSMHUKSKd7RhDOJeiYb3H
+GdXVVjI1GXKGh73kPJfPPM3103aFEps8CrayG2+cWZRhB5sAGP+7Zuoa61xaJz2p
+JQCkZXc5AhcTkz6/L5wckkJR4PHw8o5mlE4Cxmqh25nyo4kWUdbh5yJqOT5XOBWg
+4HPr5wLgY49vtPbFv7zUWbypGQDt/XfY53eVJNSBlGsIfqHzA+DvHofooEi9PVyx
+uXFdmn2Ffpck6gMvgVZ6F7DxUIjtzt8wR5NBG/7I7sOquJABkQG3Etw2X8DsrOkj
+lw==
+-----END X509 CRL-----

packages/ql3-cluster-admin/test/fixtures/next-client-key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC0DxTWF5/+vada
+tK6KdoaXnGaTYIOIXJ+FRWGnWF2X/MQJBkl3Qa8clbtpNmeW3G6MUs7Zv1tagDSV
+F+7ICa0YU49ohiC19naPQVHcfJbLWKaVs0IxNS7WKx6kHbRe+fXUzFUOU9Ij4ETg
+dLE3kHTB/90q5IuhPu1f+gq5OCLyTVsEJzJuEttMvgGOHf3m7W5FO8gNZnfWg2fp
+v3ynHjuTK6NHGjPbeN3GPVyOOysFUjhmQfVojobmzJx/HAJj8bIMqJWvZVrLIVtD
+QM0Rv3ecXUpmB3yTHcOTF/HedsVlJT6TA2Rry0RWrmAgPjcp2ycZdynoNHkqoMWv
+zTAQGuJBAgMBAAECggEACPCuwiqOISHdi11pRSq5aBG17h9eqIigywc04BA9JWzB
+dwk+4JL8VNkCYyj2X8E2I/WAm8MqkAXJlpWHn5sLYyfS+Qef3CkOnfJKVD5DZnlS
+XCB4auUPHEcr0E5BRG34zMYQexOiXXGnOdXRbsbXZUKFAroyTKFJO/i4S5YD9i/X
+mY0nOVHqsJTH601LgqMLtFcpfssHLC2wdLCS+sChrjhlEQU0RlBO5c6gg4bJ3Ohg
+Y769BD0zd5ct8rpDy3RlKlboL8bbMQqf9V6pFJ+z3aHK7OJ3Qp5oEpHFk4QtjGvj
+HkqiA18GFJU5xj3CtmX6+VFecVcVj6qGFMW9099KIQKBgQDyH/Iu+Is89GznfaUR
+qxeW02IJLoJmAbCzKoYnb5lBW/T2RLH6gsB3Ww7JOXKnkiu5xchWHu8ghwJm3N2w
+KGfRvfPqnhcNey5vD9kNLOCIyK68dmlDFgBvujnwhHXCHm5DWumdD9866lOvamVL
+vn6mUUbxPpEU7MDX9Dw0H/+ohQKBgQC+YJuKlMBrU3sS0ekz/QLQuik3twqfMHy+
+8c7WIp+I3EMpg+FvWpAGRZ0p5ej5e3iGzJmpH5gvZghloZjpJfencrtkhflCdquR
+GTzCT30u6ycxrb0NgRMLtNj1cwqG7TrfWuS6xWs27UY1FEpykOmIjA5A8jYflJCl
+kRjiAaQdjQKBgBKsAAM7S5GRz+wyNk6cgR8ygfbvTUMMklHez6Wu54Sw0L1FIuyM
+tMb/kK5X7c2aaiDTbdWmYQ6DGt5P3hC9rPPx0COhuWlivHv4HubzNQWXZFJIV2nD
+ApmbJOi1bldf/WTYV/eyx6ZFd9qpO9KRiDbPQcdM6edRMrX0scfbR935AoGBAJws
+ICzE7jydFP9RV58gVSjpLlrYO0xvuEXjk6uyZ3EcT7REzbBNeIzMmqr+Qeo3UZ4z
+RXKpymxHpWfJ8uiADvM8eWFzThEpzICRVXiJQMY2fcaRId9sHTWjO7/Oibo57evA
+Ypac0OEUMF3msVPecNVlH2RDzgaGF0rbBrCQbBMZAoGAaJ2dLgPgPPziEVEeTeF7
+ZB6fl/DfNTrK+VmjIOgQTL/mcaZaGGV/5eyhMjxsqio4p2LnGFEGhOKUuB3KCSfa
+7vhknKsrJJhvFvNsSbk0ifTVVBfH/BW/EKojRgqj6xJ6KOa8FP3JD+BPU+8M1YDl
+txb4gmmNKWBza7kBqnUAkZw=
+-----END PRIVATE KEY-----

packages/ql3-cluster-admin/test/workerCredentialManagementHttp.test.cjs

@@ -47,13 +47,21 @@ const REVOKED_CLIENT_CRL = resolve(
   __dirname,
   '../../ql3-cluster-control/test/fixtures/mtls/revoked-client-crl.pem',
 );
+const NEXT_CLIENT_CA = resolve(
+  __dirname,
+  'fixtures/next-client-ca-cert.pem',
+);
+const NEXT_CLIENT_CRL = resolve(
+  __dirname,
+  'fixtures/next-client-empty-crl.pem',
+);
 const NEXT_CLIENT_CERT = resolve(
   __dirname,
-  'fixtures/management-service-cert.pem',
+  'fixtures/next-client-cert.pem',
 );
 const NEXT_CLIENT_KEY = resolve(
   __dirname,
-  'fixtures/management-service-key.pem',
+  'fixtures/next-client-key.pem',
 );
 const WORKER_PATH = '/api/v3/worker-credentials/management';
 const CANONICAL_WORKER_PATH = '/api/v3/workers/management';
@@ -171,7 +179,11 @@ async function health(application, path) {
   });
 }
 
-async function startWithClientTrust(certificateAuthority, execute) {
+async function startWithClientTrust(
+  certificateAuthority,
+  certificateRevocationList,
+  execute,
+) {
   return startClusterWorkerCredentialManagementHttp({
     host: '127.0.0.1',
     port: 0,
@@ -179,7 +191,7 @@ async function startWithClientTrust(certificateAuthority, execute) {
       privateKey: Buffer.from(readFileSync(SERVER_KEY)),
       certificate: Buffer.from(readFileSync(SERVER_CERT)),
       clientCertificateAuthority: certificateAuthority,
-      clientCertificateRevocationList: Buffer.from(readFileSync(EMPTY_CRL)),
+      clientCertificateRevocationList: certificateRevocationList,
     },
     identities: identities(),
     transport: { execute },
@@ -402,9 +414,12 @@ test('accepts both client CAs during overlap then rejects the retired CA', async
     stale: false,
   });
   const oldAuthority = Buffer.from(readFileSync(CLIENT_CA));
-  const nextAuthority = Buffer.from(readFileSync(NEXT_CLIENT_CERT));
+  const nextAuthority = Buffer.from(readFileSync(NEXT_CLIENT_CA));
+  const oldRevocationList = Buffer.from(readFileSync(EMPTY_CRL));
+  const nextRevocationList = Buffer.from(readFileSync(NEXT_CLIENT_CRL));
   const overlap = await startWithClientTrust(
     Buffer.concat([oldAuthority, nextAuthority]),
+    Buffer.concat([oldRevocationList, nextRevocationList]),
     execute,
   );
   try {
@@ -423,7 +438,11 @@ test('accepts both client CAs during overlap then rejects the retired CA', async
     await overlap.close();
   }
 
-  const retired = await startWithClientTrust(nextAuthority, execute);
+  const retired = await startWithClientTrust(
+    nextAuthority,
+    nextRevocationList,
+    execute,
+  );
   try {
     assert.equal((await request(retired)).statusCode, 401);
     assert.equal(

packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts

@@ -1,10 +1,4 @@
 // Cluster Copilot resolves an external request key before cancelling its Run.
-import {
-  CopilotFailureDiagnosisCancellationNotFoundError,
-  CopilotFailureDiagnosisCancellationUnavailableError,
-  InvalidCopilotFailureDiagnosisCancellationError,
-  type CopilotFailureDiagnosisCancellationCommand,
-} from '@qinglong/ai/failure-diagnosis-cancellation';
 import {
   CLUSTER_RUN_CANCELLATION_SCHEMA,
   ClusterRunCancellationFenceRejectedError,
@@ -13,6 +7,10 @@ import {
   InvalidClusterRunCancellationError,
   parseClusterRunCancellationRequestBody,
 } from '@qinglong/runtime-core/cluster-run-cancellation';
+import type {
+  SecurityPolicyFence,
+  SecuritySubject,
+} from '@qinglong/runtime-core/security';
 
 import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
 import type {
@@ -33,9 +31,19 @@ export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE =
     projectParameter: 'projectId',
   });
 
+export interface ClusterCopilotFailureDiagnosisCancellationCommand {
+  readonly projectId: string;
+  readonly sourceRunId: string;
+  readonly requestId: string;
+  readonly mutationId: string;
+  readonly eventId: string;
+  readonly subject: Readonly<SecuritySubject>;
+  readonly policyFence: Readonly<SecurityPolicyFence>;
+}
+
 export interface ClusterCopilotFailureDiagnosisCancellationCapability {
   cancel(
-    command: Readonly<CopilotFailureDiagnosisCancellationCommand>,
+    command: Readonly<ClusterCopilotFailureDiagnosisCancellationCommand>,
   ): Promise<unknown>;
 }
 
@@ -54,6 +62,14 @@ const CANCEL_REASONS = new Set([
   'timeout',
 ]);
 
+function hasFailureCode(error: unknown, code: string): boolean {
+  return (
+    error instanceof Error &&
+    'code' in error &&
+    (error as Error & { readonly code?: unknown }).code === code
+  );
+}
+
 function response(
   statusCode: number,
   body: Readonly<Record<string, unknown>>,
@@ -228,7 +244,10 @@ export function createClusterControlCopilotFailureDiagnosisCancellationRoute(
         return response(view.status === 'accepted' ? 202 : 200, view);
       } catch (error) {
         if (
-          error instanceof CopilotFailureDiagnosisCancellationNotFoundError ||
+          hasFailureCode(
+            error,
+            'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_NOT_FOUND',
+          ) ||
           error instanceof ClusterRunCancellationNotFoundError
         ) {
           return response(404, {
@@ -242,9 +261,14 @@ export function createClusterControlCopilotFailureDiagnosisCancellationRoute(
           });
         }
         if (
-          error instanceof InvalidCopilotFailureDiagnosisCancellationError ||
-          error instanceof
-            CopilotFailureDiagnosisCancellationUnavailableError ||
+          hasFailureCode(
+            error,
+            'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_INVALID',
+          ) ||
+          hasFailureCode(
+            error,
+            'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_UNAVAILABLE',
+          ) ||
           error instanceof InvalidClusterRunCancellationError ||
           error instanceof ClusterRunCancellationUnavailableError
         ) {