From 1a4acaa56fcd29c37dcc7c2459475d7127397700 Mon Sep 17 00:00:00 2001 From: whyour Date: Sat, 22 Aug 2026 11:51:17 +0800 Subject: [PATCH] fix(ci): make ql3 gates reproducible --- .github/workflows/ql3-ci.yml | 54 ++++-- .../ql3-cluster-control/tsconfig.default.json | 2 +- .../ql3-local-application/Dockerfile | 14 +- package.json | 5 +- .../pluginPackageManagementHttp.ts | 33 +++- .../test/fixtures/next-client-ca-cert.pem | 20 +++ .../test/fixtures/next-client-cert.pem | 20 +++ .../test/fixtures/next-client-empty-crl.pem | 11 ++ .../test/fixtures/next-client-key.pem | 28 +++ .../workerCredentialManagementHttp.test.cjs | 31 +++- .../failureDiagnosisCancellationRoute.ts | 46 +++-- pnpm-lock.yaml | 24 +-- test/back/ql3CiCleanCheckoutContract.test.cjs | 164 ++++++++++++++++++ 13 files changed, 391 insertions(+), 61 deletions(-) create mode 100644 packages/ql3-cluster-admin/test/fixtures/next-client-ca-cert.pem create mode 100644 packages/ql3-cluster-admin/test/fixtures/next-client-cert.pem create mode 100644 packages/ql3-cluster-admin/test/fixtures/next-client-empty-crl.pem create mode 100644 packages/ql3-cluster-admin/test/fixtures/next-client-key.pem create mode 100644 test/back/ql3CiCleanCheckoutContract.test.cjs diff --git a/.github/workflows/ql3-ci.yml b/.github/workflows/ql3-ci.yml index 24e0874d..1c90fc98 100644 --- a/.github/workflows/ql3-ci.yml +++ b/.github/workflows/ql3-ci.yml @@ -26,16 +26,10 @@ jobs: include: - runner: ubuntu-24.04 arch: x64 - node: '20' - - runner: ubuntu-24.04 - arch: x64 - node: '24' + node: '24.18.0' - runner: ubuntu-24.04-arm arch: arm64 - node: '20' - - runner: ubuntu-24.04-arm - arch: arm64 - node: '24' + node: '24.18.0' steps: - uses: actions/checkout@v6 - uses: pnpm/action-setup@v6 @@ -48,12 +42,18 @@ jobs: cache-dependency-path: pnpm-lock.yaml - name: Verify native runner architecture run: node -e "if (process.arch !== '${{ matrix.arch }}') throw new Error('unexpected architecture ' + process.arch)" - - name: Install dependencies - run: pnpm install --frozen-lockfile + - name: Install dependencies without lifecycle scripts + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Initialize the public test environment + run: cp .env.example .env + - name: Rebuild the reviewed native SQLite binding + run: pnpm rebuild @whyour/sqlite3 + - name: Build backend and QL3 workspace from source + run: | + pnpm build:back + pnpm run build:packages:ql3 - name: Test backend migrations run: pnpm test:back - - name: Build backend - run: pnpm build:back - name: Audit edge dependency and import isolation run: | pnpm audit:edge-imports:ql3 @@ -62,14 +62,12 @@ jobs: - name: Smoke benchmark edge executor run: pnpm benchmark:edge -- --json - name: Prove disabled AI Profile has zero storage or credential reachability - if: matrix.node == '24' run: >- pnpm benchmark:ai-profile -- --json --max-rss-delta-mb=8 --max-disabled-activation-ms=50 - name: Benchmark Node SQLite transaction boundary - if: matrix.node == '24' run: >- pnpm benchmark:db:node-sqlite -- --json @@ -106,6 +104,8 @@ jobs: cache-dependency-path: pnpm-lock.yaml - name: Install workspace dependencies without lifecycle scripts run: pnpm install --frozen-lockfile --ignore-scripts + - name: Build the complete local test fixture workspace + run: pnpm run build:packages:ql3 - name: Build and test Local Owner product closure run: pnpm --filter @qinglong/local-owner-cli test - name: Audit root bridge static import closure @@ -138,6 +138,8 @@ jobs: run: node -e "if (process.arch !== '${{ matrix.arch }}') throw new Error('unexpected architecture ' + process.arch)" - name: Install workspace dependencies without lifecycle scripts run: pnpm install --frozen-lockfile --ignore-scripts + - name: Rebuild the reviewed native SQLite binding + run: pnpm rebuild @whyour/sqlite3 - name: Build benchmark targets once run: | pnpm build:back @@ -536,9 +538,16 @@ jobs: target: runtime steps: - uses: actions/checkout@v6 + - uses: pnpm/action-setup@v6 + with: + version: '8.3.1' - uses: actions/setup-node@v6 with: node-version: '24.18.0' + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - name: Install workspace dependencies without lifecycle scripts + run: pnpm install --frozen-lockfile --ignore-scripts - name: Verify native runner architecture run: node -e "if (process.arch !== '${{ matrix.node_arch }}') throw new Error('unexpected architecture ' + process.arch)" - name: Test exact SBOM and release contract failures @@ -731,11 +740,18 @@ jobs: target: runtime steps: - uses: actions/checkout@v6 + - uses: pnpm/action-setup@v6 + with: + version: '8.3.1' - uses: actions/setup-node@v6 with: node-version: '24.18.0' + cache: pnpm + cache-dependency-path: pnpm-lock.yaml - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + - name: Install workspace dependencies without lifecycle scripts + run: pnpm install --frozen-lockfile --ignore-scripts - name: Test bounded OCI layout audit failures run: node --test test/back/ql3ClusterOciLayoutAudit.test.cjs - name: Build an attested amd64 and arm64 OCI layout @@ -784,6 +800,8 @@ jobs: run: node -e "if (process.arch !== '${{ matrix.arch }}') throw new Error('unexpected architecture ' + process.arch)" - name: Install workspace dependencies without lifecycle scripts run: pnpm install --frozen-lockfile --ignore-scripts + - name: Initialize the public test environment + run: cp .env.example .env - name: Audit profile dependency boundaries run: | pnpm audit:edge-imports:ql3 @@ -1002,6 +1020,8 @@ jobs: run: pnpm install --frozen-lockfile --ignore-scripts - name: Audit exact cluster dependency graph run: pnpm audit:cluster-dependencies:ql3 + - name: Build QL3 workspace once + run: pnpm run build:packages:ql3 - name: Rotate PostgreSQL TLS trust through overlap and rollback run: pnpm test:postgres-tls-rotation:ql3 - name: Fence the old primary, promote the standby and reactivate two controls @@ -1105,6 +1125,8 @@ jobs: echo "f8bede43fe4ee0d478c2355b204a36876b2ae4faac60f2a9452280b293da3b88 cnpg-1.30.0.yaml" | sha256sum --check - name: Install workspace dependencies without lifecycle scripts run: pnpm install --frozen-lockfile --ignore-scripts + - name: Build QL3 workspace once + run: pnpm run build:packages:ql3 - name: Audit the static provider credential test deployment run: pnpm audit:provider-credential-test-deployment:ql3 - name: Prove eight isolated Jobs, material and CIDR rotation, and database failover @@ -1203,9 +1225,9 @@ jobs: QL3_KIND_CLUSTER: ql3-plugin-recovery-e2e-ci run: | install -d -m 0700 "${RUNNER_TEMP}/ql3-plugin-package-recovery-e2e" - pnpm test:plugin-package-recovery-e2e:ql3 -- \ + pnpm test:plugin-package-recovery-e2e:ql3 \ --report="${RUNNER_TEMP}/ql3-plugin-package-recovery-e2e/report.json" - pnpm audit:plugin-package-recovery-e2e:ql3 -- \ + pnpm audit:plugin-package-recovery-e2e:ql3 \ --report="${RUNNER_TEMP}/ql3-plugin-package-recovery-e2e/report.json" - name: Upload Plugin Package recovery E2E evidence if: always() diff --git a/deploy/containers/ql3-cluster-control/tsconfig.default.json b/deploy/containers/ql3-cluster-control/tsconfig.default.json index 68264ba9..bf63b93f 100644 --- a/deploy/containers/ql3-cluster-control/tsconfig.default.json +++ b/deploy/containers/ql3-cluster-control/tsconfig.default.json @@ -4,7 +4,7 @@ "rootDir": "../../../packages/ql3-cluster-control/src", "outDir": "../../../.ql3-image-build/cluster-control-default-dist" }, - "include": ["../../../packages/ql3-cluster-control/src/**/*.ts"], + "include": ["../../../packages/ql3-cluster-control/src/cli.ts"], "exclude": [ "../../../packages/ql3-cluster-control/src/aiCli.ts", "../../../packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts" diff --git a/deploy/containers/ql3-local-application/Dockerfile b/deploy/containers/ql3-local-application/Dockerfile index 4811889c..eab71c4d 100644 --- a/deploy/containers/ql3-local-application/Dockerfile +++ b/deploy/containers/ql3-local-application/Dockerfile @@ -53,20 +53,20 @@ RUN ln -s /opt/qinglong/node_modules node_modules \ /opt/qinglong/node_modules/@qinglong/runtime-core \ && /opt/qinglong/node_modules/.bin/tsc \ -p packages/ql3-runtime-core/tsconfig.json \ - && /opt/qinglong/node_modules/.bin/tsc \ - -p packages/ql3-ai/tsconfig.json \ && /opt/qinglong/node_modules/.bin/tsc \ -p packages/ql3-local-command-file/tsconfig.json \ - && /opt/qinglong/node_modules/.bin/tsc \ - -p packages/ql3-local-sqlite/tsconfig.json \ - && /opt/qinglong/node_modules/.bin/tsc \ - -p packages/ql3-local-admin/tsconfig.json \ && /opt/qinglong/node_modules/.bin/tsc \ -p packages/ql3-local-process/tsconfig.json \ && /opt/qinglong/node_modules/.bin/tsc \ - -p packages/ql3-local-execution/tsconfig.json \ + -p packages/ql3-local-sqlite/tsconfig.json \ + && /opt/qinglong/node_modules/.bin/tsc \ + -p packages/ql3-ai/tsconfig.json \ && /opt/qinglong/node_modules/.bin/tsc \ -p packages/ql3-local-secret/tsconfig.json \ + && /opt/qinglong/node_modules/.bin/tsc \ + -p packages/ql3-local-admin/tsconfig.json \ + && /opt/qinglong/node_modules/.bin/tsc \ + -p packages/ql3-local-execution/tsconfig.json \ && /opt/qinglong/node_modules/.bin/tsc \ -p packages/ql3-local-application/tsconfig.json diff --git a/package.json b/package.json index 4ec21c20..c8a9fb26 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "build:back": "node scripts/ql3-build-back.cjs", "build:packages:ql3": "node scripts/ql3-clean-package-artifacts.cjs && pnpm -r --workspace-concurrency=1 --filter './packages/ql3-*' run build", "test:back": "node --test test/back/*.test.cjs", - "test:packages:ql3": "pnpm run build:packages:ql3 && pnpm -r --workspace-concurrency=1 --filter './packages/ql3-*' exec sh -c 'node --test test/*.test.cjs'", + "test:packages:ql3": "pnpm run build:packages:ql3 && pnpm -r --workspace-concurrency=1 --filter './packages/ql3-*' exec sh -c 'node --test --test-concurrency=1 test/*.test.cjs'", "test:postgres-ha:ql3": "pnpm --filter @qinglong/ai build && pnpm --filter @qinglong/cluster-admin check && pnpm --filter @qinglong/cluster-control check && node scripts/ql3-postgres-ha-contract.cjs", "audit:postgres-ha-evidence:ql3": "node scripts/ql3-postgres-ha-evidence-audit.cjs", "test:automation-postgres-integration:ql3": "pnpm --filter @qinglong/cluster-admin test:automation-integration", @@ -202,7 +202,8 @@ "@whyour/sqlite3>tar": "7.5.20", "@mapbox/node-pre-gyp>tar": "7.5.20", "@kubernetes/client-node>form-data": "4.0.6", - "@kubernetes/client-node>js-yaml": "4.3.0", + "@kubernetes/client-node>js-yaml": "4.3.1", + "socks>ip-address": "10.3.1", "celebrate>lodash": "4.18.1", "express>path-to-regexp": "0.1.13", "faye-websocket>websocket-driver": "0.7.5", diff --git a/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts b/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts index 594235d7..bf09c36a 100644 --- a/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts +++ b/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts @@ -122,6 +122,20 @@ const MAX_AUTHORIZATION_BYTES = 16 * 1024; const MAX_RESPONSE_BYTES = 128 * 1024; const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/; +function tlsPemBlocks( + bytes: Buffer, + label: 'CERTIFICATE' | 'X509 CRL', +): Buffer[] { + const pattern = new RegExp( + `-----BEGIN ${label}-----[\\s\\S]*?-----END ${label}-----`, + 'g', + ); + const value = bytes.toString('utf8'); + const matches = value.match(pattern); + if (!matches || value.replace(pattern, '').trim() !== '') return [bytes]; + return matches.map((match) => Buffer.from(`${match}\n`, 'utf8')); +} + export interface ClusterPluginPackageManagementHttpLimits { readonly maxBodyBytes?: number; readonly maxConnections?: number; @@ -742,6 +756,18 @@ export async function startClusterPluginPackageManagementHttp( const createRequestId = options.createRequestId ?? randomUUID; const clientCertificateRequired = options.tls.clientCertificateAuthority !== undefined; + const clientTrust = clientCertificateRequired + ? { + ca: tlsPemBlocks( + options.tls.clientCertificateAuthority as Buffer, + 'CERTIFICATE', + ), + crl: tlsPemBlocks( + options.tls.clientCertificateRevocationList as Buffer, + 'X509 CRL', + ), + } + : undefined; const rateLimiter = new BoundedRateLimiter(limits, now); let availability: 'ready' | 'unavailable' | 'stopped' = 'ready'; let inFlight = 0; @@ -755,12 +781,7 @@ export async function startClusterPluginPackageManagementHttp( minVersion: 'TLSv1.3', maxVersion: 'TLSv1.3', honorCipherOrder: true, - ...(clientCertificateRequired - ? { - ca: options.tls.clientCertificateAuthority, - crl: options.tls.clientCertificateRevocationList, - } - : {}), + ...(clientTrust ?? {}), requestCert: clientCertificateRequired, // Health probes intentionally remain reachable without a client // certificate. Every non-health route checks TLSSocket.authorized before diff --git a/packages/ql3-cluster-admin/test/fixtures/next-client-ca-cert.pem b/packages/ql3-cluster-admin/test/fixtures/next-client-ca-cert.pem new file mode 100644 index 00000000..ce8a2feb --- /dev/null +++ b/packages/ql3-cluster-admin/test/fixtures/next-client-ca-cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDOTCCAiGgAwIBAgIUfb1aCt8o4WAshKLJue/oY/vt58UwDQYJKoZIhvcNAQEL +BQAwJDEiMCAGA1UEAwwZUWluZ0xvbmctMy1OZXh0LUNsaWVudC1DQTAeFw0yNjA4 +MjExNzMxMjRaFw0zNjA4MTgxNzMxMjRaMCQxIjAgBgNVBAMMGVFpbmdMb25nLTMt +TmV4dC1DbGllbnQtQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCV +KV+ZRNKOR+GrPibLTsgOz5cm42/9TfwbZX0NQ2pVQ6X00BcpRqSlIJ7qtro6yiZ0 +lvvPisbmjJYWFB9OmOqjmZT2lGi/oc2UrpzZCfBTVuplNMMvKye/WbN7lOXp5KM2 +Fzm1+sRRWZyyCGlEtoyiBFS/0nr9JGm060CvIGfRaC8GyxbFogvsDUWt3qwDh2wN +UHLBePF73HWYTEd2YX22nYbDUPVUInfeUmMdcJ6kTYlyDxwNk3HFh7inJjwC5c6H +kAb9tWP73llZbvGNvqubXauJrIvCEOSKPrqe2Z36F4qDRQ93JbNN9z35I6/4j1nP +EWeAEDHmQyxzjENA2hpRAgMBAAGjYzBhMB0GA1UdDgQWBBRJ51y34GhuN+2O6++G +XbtzteCitjAfBgNVHSMEGDAWgBRJ51y34GhuN+2O6++GXbtzteCitjAPBgNVHRMB +Af8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEAYHoc +MjXSLEfYXtzHU7hsjOtITjyK8lofXN4dTHU0ayKQCH9XuUBcSnglW+EvbGJ/zTpk +eceQpCqAuhGW2gnj7+kjtf/GYq/aVBwyHwZvDUB7GZX7suLJqxm0rZSomO/yB8H2 +Shm9iAJ/gb5WQeBW22XWEMAXivqnqt+h3GwqRsrDDySqEJEmRpl7SJRuAIDauFv6 +OVlEB4r29nNGgx9xUMgY0ZMB5dwG7hCTvllmR4NpGfsKUqScjpOys+MrOY6DHyRD +57W/HkKVR7TP/GIyvAfWEFX8S/0efY2gNV6nAG7YYbD1i4UW5LWRyoiNGYORGhse +wCTYki14yDV+dRgCtw== +-----END CERTIFICATE----- diff --git a/packages/ql3-cluster-admin/test/fixtures/next-client-cert.pem b/packages/ql3-cluster-admin/test/fixtures/next-client-cert.pem new file mode 100644 index 00000000..17a29e8e --- /dev/null +++ b/packages/ql3-cluster-admin/test/fixtures/next-client-cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNTCCAh2gAwIBAgIDMQABMA0GCSqGSIb3DQEBCwUAMCQxIjAgBgNVBAMMGVFp +bmdMb25nLTMtTmV4dC1DbGllbnQtQ0EwHhcNMjYwODIxMTczMTI0WhcNMzYwODE4 +MTczMTI0WjAfMR0wGwYDVQQDDBRxbDMtbmV4dC10ZXN0LXdvcmtlcjCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBALQPFNYXn/69p1q0rop2hpecZpNgg4hc +n4VFYadYXZf8xAkGSXdBrxyVu2k2Z5bcboxSztm/W1qANJUX7sgJrRhTj2iGILX2 +do9BUdx8lstYppWzQjE1LtYrHqQdtF759dTMVQ5T0iPgROB0sTeQdMH/3Srki6E+ +7V/6Crk4IvJNWwQnMm4S20y+AY4d/ebtbkU7yA1md9aDZ+m/fKceO5Mro0caM9t4 +3cY9XI47KwVSOGZB9WiOhubMnH8cAmPxsgyola9lWsshW0NAzRG/d5xdSmYHfJMd +w5MX8d52xWUlPpMDZGvLRFauYCA+NynbJxl3Keg0eSqgxa/NMBAa4kECAwEAAaN1 +MHMwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB +BQUHAwIwHQYDVR0OBBYEFM1aBVCbqzCtxpsRliCwk3NyF8POMB8GA1UdIwQYMBaA +FEnnXLfgaG437Y7r74Zdu3O14KK2MA0GCSqGSIb3DQEBCwUAA4IBAQAJVWhijDA3 +JdMKDag2mxZX5wYCGeWWXIUvoo3kkqV1sSoD2w4i0K0D49UucAFqcKjUZG9iXbuE +m29AZGsm6HV7h490fzI3HY3VUGfjkMO5ikDxXuO3ePbDYwZb2W3CRiwFLclCGBoC +RDXh8qPvcBdtT6NSn/dzGJq+2nyntKWTcaO+JjdARYLO8wbQWcqMldSvFICHDikd +eKiyYd26z6x/cEOdGtWaB2OJ37bNcaVgRcA/bvQ+L3wF5WmAeO+ptNA0FRSXolN8 +bY7dXPHFj6zYQ5paSYazT5LqTO6Ga1X4s4KC018aSyYE0QTqGnML9nxxmNw3S+uE +wFhYDiTuyhAb +-----END CERTIFICATE----- diff --git a/packages/ql3-cluster-admin/test/fixtures/next-client-empty-crl.pem b/packages/ql3-cluster-admin/test/fixtures/next-client-empty-crl.pem new file mode 100644 index 00000000..11a0b16c --- /dev/null +++ b/packages/ql3-cluster-admin/test/fixtures/next-client-empty-crl.pem @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBfTBnAgEBMA0GCSqGSIb3DQEBCwUAMCQxIjAgBgNVBAMMGVFpbmdMb25nLTMt +TmV4dC1DbGllbnQtQ0EXDTI2MDgyMTE3MzEyNFoXDTM2MDgxODE3MzEyNFqgDzAN +MAsGA1UdFAQEAgIQADANBgkqhkiG9w0BAQsFAAOCAQEARwTg7QvukmIloA61T/rJ +WKc5izTclZh/2YTmBi25FodlhZISnBquHEsCwpqoerXOSMHUKSKd7RhDOJeiYb3H +GdXVVjI1GXKGh73kPJfPPM3103aFEps8CrayG2+cWZRhB5sAGP+7Zuoa61xaJz2p +JQCkZXc5AhcTkz6/L5wckkJR4PHw8o5mlE4Cxmqh25nyo4kWUdbh5yJqOT5XOBWg +4HPr5wLgY49vtPbFv7zUWbypGQDt/XfY53eVJNSBlGsIfqHzA+DvHofooEi9PVyx +uXFdmn2Ffpck6gMvgVZ6F7DxUIjtzt8wR5NBG/7I7sOquJABkQG3Etw2X8DsrOkj +lw== +-----END X509 CRL----- diff --git a/packages/ql3-cluster-admin/test/fixtures/next-client-key.pem b/packages/ql3-cluster-admin/test/fixtures/next-client-key.pem new file mode 100644 index 00000000..30343c62 --- /dev/null +++ b/packages/ql3-cluster-admin/test/fixtures/next-client-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC0DxTWF5/+vada +tK6KdoaXnGaTYIOIXJ+FRWGnWF2X/MQJBkl3Qa8clbtpNmeW3G6MUs7Zv1tagDSV +F+7ICa0YU49ohiC19naPQVHcfJbLWKaVs0IxNS7WKx6kHbRe+fXUzFUOU9Ij4ETg +dLE3kHTB/90q5IuhPu1f+gq5OCLyTVsEJzJuEttMvgGOHf3m7W5FO8gNZnfWg2fp +v3ynHjuTK6NHGjPbeN3GPVyOOysFUjhmQfVojobmzJx/HAJj8bIMqJWvZVrLIVtD +QM0Rv3ecXUpmB3yTHcOTF/HedsVlJT6TA2Rry0RWrmAgPjcp2ycZdynoNHkqoMWv +zTAQGuJBAgMBAAECggEACPCuwiqOISHdi11pRSq5aBG17h9eqIigywc04BA9JWzB +dwk+4JL8VNkCYyj2X8E2I/WAm8MqkAXJlpWHn5sLYyfS+Qef3CkOnfJKVD5DZnlS +XCB4auUPHEcr0E5BRG34zMYQexOiXXGnOdXRbsbXZUKFAroyTKFJO/i4S5YD9i/X +mY0nOVHqsJTH601LgqMLtFcpfssHLC2wdLCS+sChrjhlEQU0RlBO5c6gg4bJ3Ohg +Y769BD0zd5ct8rpDy3RlKlboL8bbMQqf9V6pFJ+z3aHK7OJ3Qp5oEpHFk4QtjGvj +HkqiA18GFJU5xj3CtmX6+VFecVcVj6qGFMW9099KIQKBgQDyH/Iu+Is89GznfaUR +qxeW02IJLoJmAbCzKoYnb5lBW/T2RLH6gsB3Ww7JOXKnkiu5xchWHu8ghwJm3N2w +KGfRvfPqnhcNey5vD9kNLOCIyK68dmlDFgBvujnwhHXCHm5DWumdD9866lOvamVL +vn6mUUbxPpEU7MDX9Dw0H/+ohQKBgQC+YJuKlMBrU3sS0ekz/QLQuik3twqfMHy+ +8c7WIp+I3EMpg+FvWpAGRZ0p5ej5e3iGzJmpH5gvZghloZjpJfencrtkhflCdquR +GTzCT30u6ycxrb0NgRMLtNj1cwqG7TrfWuS6xWs27UY1FEpykOmIjA5A8jYflJCl +kRjiAaQdjQKBgBKsAAM7S5GRz+wyNk6cgR8ygfbvTUMMklHez6Wu54Sw0L1FIuyM +tMb/kK5X7c2aaiDTbdWmYQ6DGt5P3hC9rPPx0COhuWlivHv4HubzNQWXZFJIV2nD +ApmbJOi1bldf/WTYV/eyx6ZFd9qpO9KRiDbPQcdM6edRMrX0scfbR935AoGBAJws +ICzE7jydFP9RV58gVSjpLlrYO0xvuEXjk6uyZ3EcT7REzbBNeIzMmqr+Qeo3UZ4z +RXKpymxHpWfJ8uiADvM8eWFzThEpzICRVXiJQMY2fcaRId9sHTWjO7/Oibo57evA +Ypac0OEUMF3msVPecNVlH2RDzgaGF0rbBrCQbBMZAoGAaJ2dLgPgPPziEVEeTeF7 +ZB6fl/DfNTrK+VmjIOgQTL/mcaZaGGV/5eyhMjxsqio4p2LnGFEGhOKUuB3KCSfa +7vhknKsrJJhvFvNsSbk0ifTVVBfH/BW/EKojRgqj6xJ6KOa8FP3JD+BPU+8M1YDl +txb4gmmNKWBza7kBqnUAkZw= +-----END PRIVATE KEY----- diff --git a/packages/ql3-cluster-admin/test/workerCredentialManagementHttp.test.cjs b/packages/ql3-cluster-admin/test/workerCredentialManagementHttp.test.cjs index 7d6c6b59..f3f3ecfb 100644 --- a/packages/ql3-cluster-admin/test/workerCredentialManagementHttp.test.cjs +++ b/packages/ql3-cluster-admin/test/workerCredentialManagementHttp.test.cjs @@ -47,13 +47,21 @@ const REVOKED_CLIENT_CRL = resolve( __dirname, '../../ql3-cluster-control/test/fixtures/mtls/revoked-client-crl.pem', ); +const NEXT_CLIENT_CA = resolve( + __dirname, + 'fixtures/next-client-ca-cert.pem', +); +const NEXT_CLIENT_CRL = resolve( + __dirname, + 'fixtures/next-client-empty-crl.pem', +); const NEXT_CLIENT_CERT = resolve( __dirname, - 'fixtures/management-service-cert.pem', + 'fixtures/next-client-cert.pem', ); const NEXT_CLIENT_KEY = resolve( __dirname, - 'fixtures/management-service-key.pem', + 'fixtures/next-client-key.pem', ); const WORKER_PATH = '/api/v3/worker-credentials/management'; const CANONICAL_WORKER_PATH = '/api/v3/workers/management'; @@ -171,7 +179,11 @@ async function health(application, path) { }); } -async function startWithClientTrust(certificateAuthority, execute) { +async function startWithClientTrust( + certificateAuthority, + certificateRevocationList, + execute, +) { return startClusterWorkerCredentialManagementHttp({ host: '127.0.0.1', port: 0, @@ -179,7 +191,7 @@ async function startWithClientTrust(certificateAuthority, execute) { privateKey: Buffer.from(readFileSync(SERVER_KEY)), certificate: Buffer.from(readFileSync(SERVER_CERT)), clientCertificateAuthority: certificateAuthority, - clientCertificateRevocationList: Buffer.from(readFileSync(EMPTY_CRL)), + clientCertificateRevocationList: certificateRevocationList, }, identities: identities(), transport: { execute }, @@ -402,9 +414,12 @@ test('accepts both client CAs during overlap then rejects the retired CA', async stale: false, }); const oldAuthority = Buffer.from(readFileSync(CLIENT_CA)); - const nextAuthority = Buffer.from(readFileSync(NEXT_CLIENT_CERT)); + const nextAuthority = Buffer.from(readFileSync(NEXT_CLIENT_CA)); + const oldRevocationList = Buffer.from(readFileSync(EMPTY_CRL)); + const nextRevocationList = Buffer.from(readFileSync(NEXT_CLIENT_CRL)); const overlap = await startWithClientTrust( Buffer.concat([oldAuthority, nextAuthority]), + Buffer.concat([oldRevocationList, nextRevocationList]), execute, ); try { @@ -423,7 +438,11 @@ test('accepts both client CAs during overlap then rejects the retired CA', async await overlap.close(); } - const retired = await startWithClientTrust(nextAuthority, execute); + const retired = await startWithClientTrust( + nextAuthority, + nextRevocationList, + execute, + ); try { assert.equal((await request(retired)).statusCode, 401); assert.equal( diff --git a/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts b/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts index 05c80d71..07bdb991 100644 --- a/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts +++ b/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts @@ -1,10 +1,4 @@ // Cluster Copilot resolves an external request key before cancelling its Run. -import { - CopilotFailureDiagnosisCancellationNotFoundError, - CopilotFailureDiagnosisCancellationUnavailableError, - InvalidCopilotFailureDiagnosisCancellationError, - type CopilotFailureDiagnosisCancellationCommand, -} from '@qinglong/ai/failure-diagnosis-cancellation'; import { CLUSTER_RUN_CANCELLATION_SCHEMA, ClusterRunCancellationFenceRejectedError, @@ -13,6 +7,10 @@ import { InvalidClusterRunCancellationError, parseClusterRunCancellationRequestBody, } from '@qinglong/runtime-core/cluster-run-cancellation'; +import type { + SecurityPolicyFence, + SecuritySubject, +} from '@qinglong/runtime-core/security'; import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface'; import type { @@ -33,9 +31,19 @@ export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE = projectParameter: 'projectId', }); +export interface ClusterCopilotFailureDiagnosisCancellationCommand { + readonly projectId: string; + readonly sourceRunId: string; + readonly requestId: string; + readonly mutationId: string; + readonly eventId: string; + readonly subject: Readonly; + readonly policyFence: Readonly; +} + export interface ClusterCopilotFailureDiagnosisCancellationCapability { cancel( - command: Readonly, + command: Readonly, ): Promise; } @@ -54,6 +62,14 @@ const CANCEL_REASONS = new Set([ 'timeout', ]); +function hasFailureCode(error: unknown, code: string): boolean { + return ( + error instanceof Error && + 'code' in error && + (error as Error & { readonly code?: unknown }).code === code + ); +} + function response( statusCode: number, body: Readonly>, @@ -228,7 +244,10 @@ export function createClusterControlCopilotFailureDiagnosisCancellationRoute( return response(view.status === 'accepted' ? 202 : 200, view); } catch (error) { if ( - error instanceof CopilotFailureDiagnosisCancellationNotFoundError || + hasFailureCode( + error, + 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_NOT_FOUND', + ) || error instanceof ClusterRunCancellationNotFoundError ) { return response(404, { @@ -242,9 +261,14 @@ export function createClusterControlCopilotFailureDiagnosisCancellationRoute( }); } if ( - error instanceof InvalidCopilotFailureDiagnosisCancellationError || - error instanceof - CopilotFailureDiagnosisCancellationUnavailableError || + hasFailureCode( + error, + 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_INVALID', + ) || + hasFailureCode( + error, + 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_UNAVAILABLE', + ) || error instanceof InvalidClusterRunCancellationError || error instanceof ClusterRunCancellationUnavailableError ) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d1279ae6..c31f1860 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,7 +6,8 @@ overrides: '@whyour/sqlite3>tar': 7.5.20 '@mapbox/node-pre-gyp>tar': 7.5.20 '@kubernetes/client-node>form-data': 4.0.6 - '@kubernetes/client-node>js-yaml': 4.3.0 + '@kubernetes/client-node>js-yaml': 4.3.1 + socks>ip-address: 10.3.1 celebrate>lodash: 4.18.1 express>path-to-regexp: 0.1.13 faye-websocket>websocket-driver: 0.7.5 @@ -397,12 +398,12 @@ importers: packages/ql3-cluster-admin: dependencies: - '@modelcontextprotocol/server': - specifier: 2.0.0 - version: 2.0.0 '@kubernetes/client-node': specifier: 1.4.0 version: 1.4.0 + '@modelcontextprotocol/server': + specifier: 2.0.0 + version: 2.0.0 '@qinglong/ai': specifier: workspace:* version: link:../ql3-ai @@ -3993,7 +3994,7 @@ packages: globals: 13.24.0 ignore: 5.3.2 import-fresh: 3.3.0 - js-yaml: 4.1.0 + js-yaml: 4.3.1 minimatch: 3.1.5 strip-json-comments: 3.1.1 transitivePeerDependencies: @@ -4343,7 +4344,7 @@ packages: form-data: 4.0.6 hpagent: 1.2.0 isomorphic-ws: 5.0.0(ws@8.21.1) - js-yaml: 4.3.0 + js-yaml: 4.3.1 jsonpath-plus: 10.4.0 node-fetch: 2.7.0 openid-client: 6.8.4 @@ -11427,8 +11428,8 @@ packages: loose-envify: 1.4.0 dev: true - /ip-address@10.2.0: - resolution: {integrity: sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==} + /ip-address@10.3.1: + resolution: {integrity: sha512-1e9d3kb97NHJTIJDZW9rKqW2h6+dFa50Dy0fpPSMQp2ADje5gvKsXmdiK6dwY5t76TaTt5+P5N1Y/LoToIxP6g==} engines: {node: '>= 12'} dev: false @@ -11943,12 +11944,11 @@ packages: dependencies: argparse: 2.0.1 - /js-yaml@4.3.0: - resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==} + /js-yaml@4.3.1: + resolution: {integrity: sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==} hasBin: true dependencies: argparse: 2.0.1 - dev: false /jsbi@4.3.2: resolution: {integrity: sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew==} @@ -16263,7 +16263,7 @@ packages: resolution: {integrity: sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} dependencies: - ip-address: 10.2.0 + ip-address: 10.3.1 smart-buffer: 4.2.0 dev: false diff --git a/test/back/ql3CiCleanCheckoutContract.test.cjs b/test/back/ql3CiCleanCheckoutContract.test.cjs new file mode 100644 index 00000000..53f5a9ac --- /dev/null +++ b/test/back/ql3CiCleanCheckoutContract.test.cjs @@ -0,0 +1,164 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { test } = require('node:test'); + +const ROOT = path.resolve(__dirname, '../..'); +const WORKFLOW = fs.readFileSync( + path.join(ROOT, '.github/workflows/ql3-ci.yml'), + 'utf8', +); +const MANIFEST = JSON.parse( + fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'), +); + +function jobSource(name, nextName) { + const source = WORKFLOW.match( + nextName + ? new RegExp(` ${name}:\\n([\\s\\S]*?)\\n ${nextName}:`) + : new RegExp(` ${name}:\\n([\\s\\S]*)$`), + )?.[1]; + assert.ok(source, `${name} job is missing`); + return source; +} + +function assertOrdered(source, values) { + let cursor = -1; + for (const value of values) { + const next = source.indexOf(value, cursor + 1); + assert.notEqual(next, -1, `${value} is missing`); + assert.ok(next > cursor, `${value} is out of order`); + cursor = next; + } +} + +test('pins backend CI to the released Node line and bootstraps a clean checkout', () => { + const backend = jobSource('backend', 'service-manager-bridge'); + assert.doesNotMatch(backend, /node: '20'/); + assert.equal(backend.match(/node: '24\.18\.0'/g)?.length, 2); + assertOrdered(backend, [ + 'pnpm install --frozen-lockfile --ignore-scripts', + 'cp .env.example .env', + 'pnpm rebuild @whyour/sqlite3', + 'pnpm build:back', + 'pnpm run build:packages:ql3', + 'pnpm test:back', + ]); + assert.doesNotMatch(backend, /if: matrix\.node == '24'/); +}); + +test('bootstraps jobs that previously depended on local modules or artifacts', () => { + const serviceManager = jobSource( + 'service-manager-bridge', + 'linux-resource-envelopes', + ); + assertOrdered(serviceManager, [ + 'pnpm install --frozen-lockfile --ignore-scripts', + 'pnpm run build:packages:ql3', + 'pnpm --filter @qinglong/local-owner-cli test', + ]); + + const worker = jobSource('worker-runtime', 'local-profiles'); + assertOrdered(worker, [ + 'pnpm install --frozen-lockfile --ignore-scripts', + 'cp .env.example .env', + 'pnpm audit:edge-imports:ql3', + ]); + + const clusterImage = jobSource( + 'cluster-image', + 'cluster-console-capacity-release-evidence', + ); + const oci = jobSource('image-oci', 'worker-runtime'); + for (const source of [clusterImage, oci]) { + assert.match(source, /pnpm\/action-setup@v6/); + assert.match(source, /pnpm install --frozen-lockfile --ignore-scripts/); + } + + const providerLive = jobSource( + 'cluster-provider-credential-test-kubernetes-live', + 'cluster-plugin-package-kubernetes-live', + ); + assertOrdered(providerLive, [ + 'pnpm install --frozen-lockfile --ignore-scripts', + 'pnpm run build:packages:ql3', + 'pnpm test:provider-credential-test-kubernetes-live:ql3', + ]); +}); + +test('rebuilds the only native legacy binding used by resource evidence', () => { + const resource = jobSource( + 'linux-resource-envelopes', + 'linux-resource-release-evidence', + ); + assertOrdered(resource, [ + 'pnpm install --frozen-lockfile --ignore-scripts', + 'pnpm rebuild @whyour/sqlite3', + 'pnpm build:back', + ]); +}); + +test('does not forward a literal separator into recovery evidence CLIs', () => { + const recovery = jobSource( + 'cluster-plugin-package-recovery-e2e', + ); + assert.doesNotMatch( + recovery, + /pnpm (?:test|audit):plugin-package-recovery-e2e:ql3 -- \\/, + ); + assert.match(recovery, /pnpm test:plugin-package-recovery-e2e:ql3 \\/); + assert.match(recovery, /pnpm audit:plugin-package-recovery-e2e:ql3 \\/); +}); + +test('keeps container TypeScript builds in workspace dependency order', () => { + const localDockerfile = fs.readFileSync( + path.join(ROOT, 'deploy/containers/ql3-local-application/Dockerfile'), + 'utf8', + ); + assertOrdered(localDockerfile, [ + '-p packages/ql3-runtime-core/tsconfig.json', + '-p packages/ql3-local-command-file/tsconfig.json', + '-p packages/ql3-local-process/tsconfig.json', + '-p packages/ql3-local-sqlite/tsconfig.json', + '-p packages/ql3-ai/tsconfig.json', + '-p packages/ql3-local-secret/tsconfig.json', + '-p packages/ql3-local-admin/tsconfig.json', + '-p packages/ql3-local-execution/tsconfig.json', + '-p packages/ql3-local-application/tsconfig.json', + ]); + + const defaultControlConfig = JSON.parse( + fs.readFileSync( + path.join( + ROOT, + 'deploy/containers/ql3-cluster-control/tsconfig.default.json', + ), + 'utf8', + ), + ); + assert.deepEqual(defaultControlConfig.include, [ + '../../../packages/ql3-cluster-control/src/cli.ts', + ]); + assert.deepEqual(defaultControlConfig.exclude, [ + '../../../packages/ql3-cluster-control/src/aiCli.ts', + '../../../packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts', + ]); + assert.doesNotMatch( + fs.readFileSync( + path.join( + ROOT, + 'packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts', + ), + 'utf8', + ), + /@qinglong\/ai/, + ); +}); + +test('pins patched transitive versions for the QL3 importer audit', () => { + assert.equal( + MANIFEST.pnpm.overrides['@kubernetes/client-node>js-yaml'], + '4.3.1', + ); + assert.equal(MANIFEST.pnpm.overrides['socks>ip-address'], '10.3.1'); +});