From c0ab62e64ac39b5b209901daad6c57c5e74931c3 Mon Sep 17 00:00:00 2001 From: whyour Date: Wed, 12 Aug 2026 07:28:43 +0800 Subject: [PATCH] feat(ql3): add strong cluster run management --- .../run-management/base/deployment.yaml | 198 ++++++++ .../run-management/base/kustomization.yaml | 9 + .../run-management/base/network-policy.yaml | 38 ++ .../base/pod-disruption-budget.yaml | 15 + .../run-management/base/service-account.yaml | 10 + .../run-management/base/service.yaml | 19 + .../cloudnative-pg/deployment-patch.yaml | 71 +++ .../cloudnative-pg/kustomization.yaml | 25 + .../cloudnative-pg/network-policy-patch.yaml | 26 + .../run-management/config.example.yaml | 72 +++ .../cloudnative-pg/credentials.example.yaml | 12 + .../cloudnative-pg/database-roles.yaml | 24 + docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 16 + ...esql-cluster-manual-run-retry-authority.md | 2 + ...383-strong-cluster-run-management-plane.md | 74 +++ docs/adr/README.md | 1 + packages/ql3-cluster-admin/package.json | 27 + .../pluginPackageIdentityAssertion.ts | 5 + .../pluginPackageIdentityKeyset.ts | 10 + .../pluginPackageManagementHttp.ts | 37 +- .../src/run-management/runManagement.ts | 273 ++++++++++ .../src/run-management/runManagementCli.ts | 74 +++ .../src/run-management/runManagementClient.ts | 106 ++++ .../run-management/runManagementClientCli.ts | 67 +++ .../src/run-management/runManagementHttp.ts | 24 + .../run-management/runManagementProcess.ts | 467 ++++++++++++++++++ .../run-management/runManagementTransport.ts | 214 ++++++++ .../ql3-cluster-admin/test/bootstrap.test.cjs | 1 + .../test/pluginPackageIdentityKeyset.test.cjs | 61 +++ .../test/pluginPackageRecovery.test.cjs | 1 + .../test/runManagement.test.cjs | 164 ++++++ .../test/runManagementClient.test.cjs | 77 +++ .../test/runManagementHttp.test.cjs | 108 ++++ .../test/runManagementProcess.test.cjs | 72 +++ .../test/runManagementTransport.test.cjs | 141 ++++++ .../test/application.test.cjs | 1 + .../test/bootstrap.test.cjs | 1 + packages/ql3-cluster-postgres/package.json | 5 + .../src/connection/pool.ts | 7 + .../src/entrypoints/runManager.ts | 31 ++ ...inPackageIdentityKeysetLedgerRepository.ts | 6 +- .../src/migration/migrationManifest.ts | 5 + .../src/migrations/index.ts | 2 + .../pg-0056-run-management-boundary.ts | 122 +++++ .../runManualRetryRepository.ts | 51 +- .../src/schema/schemaContract.ts | 19 +- .../src/schema/schemaReadiness.ts | 66 +++ ...ageIdentityKeysetLedgerRepository.test.cjs | 10 +- .../ql3-cluster-postgres/test/pool.test.cjs | 1 + .../postgresqlMigrationDefinitions.test.cjs | 33 ++ .../test/postgresqlSchemaReadiness.test.cjs | 83 +++- .../test/runManualRetryRepository.test.cjs | 35 +- .../ql3-cloudnativepg-deployment-audit.cjs | 11 +- scripts/ql3-cluster-dependency-audit.cjs | 5 + scripts/ql3-postgres-ha-contract.cjs | 63 ++- .../ql3CloudNativePgDeploymentAudit.test.cjs | 1 + test/back/ql3PackageBoundaryAudit.test.cjs | 8 +- test/back/ql3RunManagementDeployment.test.cjs | 85 ++++ 58 files changed, 3087 insertions(+), 105 deletions(-) create mode 100644 deploy/kubernetes/ql3-cluster/operations/run-management/base/deployment.yaml create mode 100644 deploy/kubernetes/ql3-cluster/operations/run-management/base/kustomization.yaml create mode 100644 deploy/kubernetes/ql3-cluster/operations/run-management/base/network-policy.yaml create mode 100644 deploy/kubernetes/ql3-cluster/operations/run-management/base/pod-disruption-budget.yaml create mode 100644 deploy/kubernetes/ql3-cluster/operations/run-management/base/service-account.yaml create mode 100644 deploy/kubernetes/ql3-cluster/operations/run-management/base/service.yaml create mode 100644 deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/deployment-patch.yaml create mode 100644 deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/kustomization.yaml create mode 100644 deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/network-policy-patch.yaml create mode 100644 deploy/kubernetes/ql3-cluster/operations/run-management/config.example.yaml create mode 100644 docs/adr/ADR-0383-strong-cluster-run-management-plane.md create mode 100644 packages/ql3-cluster-admin/src/run-management/runManagement.ts create mode 100644 packages/ql3-cluster-admin/src/run-management/runManagementCli.ts create mode 100644 packages/ql3-cluster-admin/src/run-management/runManagementClient.ts create mode 100644 packages/ql3-cluster-admin/src/run-management/runManagementClientCli.ts create mode 100644 packages/ql3-cluster-admin/src/run-management/runManagementHttp.ts create mode 100644 packages/ql3-cluster-admin/src/run-management/runManagementProcess.ts create mode 100644 packages/ql3-cluster-admin/src/run-management/runManagementTransport.ts create mode 100644 packages/ql3-cluster-admin/test/runManagement.test.cjs create mode 100644 packages/ql3-cluster-admin/test/runManagementClient.test.cjs create mode 100644 packages/ql3-cluster-admin/test/runManagementHttp.test.cjs create mode 100644 packages/ql3-cluster-admin/test/runManagementProcess.test.cjs create mode 100644 packages/ql3-cluster-admin/test/runManagementTransport.test.cjs create mode 100644 packages/ql3-cluster-postgres/src/entrypoints/runManager.ts create mode 100644 packages/ql3-cluster-postgres/src/run-management/pg-0056-run-management-boundary.ts create mode 100644 test/back/ql3RunManagementDeployment.test.cjs diff --git a/deploy/kubernetes/ql3-cluster/operations/run-management/base/deployment.yaml b/deploy/kubernetes/ql3-cluster/operations/run-management/base/deployment.yaml new file mode 100644 index 00000000..4926472a --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/run-management/base/deployment.yaml @@ -0,0 +1,198 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ql3-run-management + namespace: qinglong3-system + labels: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management + app.kubernetes.io/part-of: qinglong3 +spec: + replicas: 2 + minReadySeconds: 10 + revisionHistoryLimit: 3 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management + template: + metadata: + annotations: + qinglong.io/run-management-client-ca-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000 + qinglong.io/run-management-client-crl-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000 + labels: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management + app.kubernetes.io/part-of: qinglong3 + spec: + serviceAccountName: ql3-run-management + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management + containers: + - name: management + image: qinglong3-cluster-admin:3.0.0-alpha.0 + imagePullPolicy: IfNotPresent + command: + - node + - /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/run-management/runManagementCli.js + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + env: + - name: QL3_PROFILE + value: cluster-admin + - name: QL3_RUN_MANAGEMENT_ENABLED + value: 'true' + - name: QL3_RUN_MANAGEMENT_HOST + value: 0.0.0.0 + - name: QL3_RUN_MANAGEMENT_PORT + value: '8448' + - name: QL3_RUN_MANAGEMENT_TLS_CERT_FILE + value: /var/run/secrets/qinglong3/run-management-tls/tls.crt + - name: QL3_RUN_MANAGEMENT_TLS_KEY_FILE + value: /var/run/secrets/qinglong3/run-management-tls/tls.key + - name: QL3_RUN_MANAGEMENT_CLIENT_CA_FILE + value: /var/run/secrets/qinglong3/run-management-tls/ca.crt + - name: QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE + value: /var/run/secrets/qinglong3/run-management-tls/client.crl + - name: QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE + value: /var/run/qinglong3/run-management-identity/keyset.json + - name: QL3_RUN_MANAGEMENT_MAX_BODY_BYTES + value: '32768' + - name: QL3_RUN_MANAGEMENT_MAX_CONNECTIONS + value: '32' + - name: QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS + value: '16' + - name: QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS + value: '10000' + - name: QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS + value: '5000' + - name: QL3_RUN_MANAGEMENT_RATE_WINDOW_MS + value: '60000' + - name: QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT + value: '30' + - name: QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT + value: '300' + - name: QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS + value: '1024' + - name: QL3_POSTGRES_RUN_MANAGER_TLS_MODE + value: verify-full + - name: QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE + value: /var/run/secrets/qinglong3/postgres-run-manager/ca.crt + - name: QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME + value: qinglong3-run-manager + - name: QL3_POSTGRES_RUN_MANAGER_POOL_MAX + value: '2' + - name: QL3_POSTGRES_RUN_MANAGER_URL + valueFrom: + secretKeyRef: + name: ql3-cluster-run-management-database + key: postgres-run-manager-url + - name: QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME + valueFrom: + secretKeyRef: + name: ql3-cluster-run-management-database + key: postgres-tls-servername + ports: + - name: https + containerPort: 8448 + protocol: TCP + startupProbe: + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 2 + timeoutSeconds: 1 + failureThreshold: 30 + readinessProbe: + httpGet: + path: /readyz + port: https + scheme: HTTPS + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 2 + livenessProbe: + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + resources: + requests: + cpu: 50m + memory: 96Mi + limits: + cpu: '1' + memory: 384Mi + volumeMounts: + - name: tmp + mountPath: /tmp + - name: management-tls + mountPath: /var/run/secrets/qinglong3/run-management-tls + readOnly: true + - name: management-identity + mountPath: /var/run/qinglong3/run-management-identity + readOnly: true + - name: postgres-run-manager-ca + mountPath: /var/run/secrets/qinglong3/postgres-run-manager + readOnly: true + volumes: + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 16Mi + - name: management-tls + secret: + secretName: ql3-run-management-tls + defaultMode: 288 + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + - key: ca.crt + path: ca.crt + - key: client.crl + path: client.crl + - name: management-identity + secret: + secretName: ql3-run-management-identity + defaultMode: 292 + items: + - key: keyset.json + path: keyset.json + - name: postgres-run-manager-ca + secret: + secretName: ql3-cluster-run-management-database + defaultMode: 292 + items: + - key: postgres-ca.crt + path: ca.crt diff --git a/deploy/kubernetes/ql3-cluster/operations/run-management/base/kustomization.yaml b/deploy/kubernetes/ql3-cluster/operations/run-management/base/kustomization.yaml new file mode 100644 index 00000000..7221c155 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/run-management/base/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - service-account.yaml + - service.yaml + - deployment.yaml + - pod-disruption-budget.yaml + - network-policy.yaml diff --git a/deploy/kubernetes/ql3-cluster/operations/run-management/base/network-policy.yaml b/deploy/kubernetes/ql3-cluster/operations/run-management/base/network-policy.yaml new file mode 100644 index 00000000..21b4a480 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/run-management/base/network-policy.yaml @@ -0,0 +1,38 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: ql3-run-management + namespace: qinglong3-system + labels: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management + app.kubernetes.io/part-of: qinglong3 +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + qinglong.io/run-management-client: 'true' + ports: + - protocol: TCP + port: 8448 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 diff --git a/deploy/kubernetes/ql3-cluster/operations/run-management/base/pod-disruption-budget.yaml b/deploy/kubernetes/ql3-cluster/operations/run-management/base/pod-disruption-budget.yaml new file mode 100644 index 00000000..9639b3c0 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/run-management/base/pod-disruption-budget.yaml @@ -0,0 +1,15 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: ql3-run-management + namespace: qinglong3-system + labels: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management + app.kubernetes.io/part-of: qinglong3 +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management diff --git a/deploy/kubernetes/ql3-cluster/operations/run-management/base/service-account.yaml b/deploy/kubernetes/ql3-cluster/operations/run-management/base/service-account.yaml new file mode 100644 index 00000000..f1fb634f --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/run-management/base/service-account.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ql3-run-management + namespace: qinglong3-system + labels: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management + app.kubernetes.io/part-of: qinglong3 +automountServiceAccountToken: false diff --git a/deploy/kubernetes/ql3-cluster/operations/run-management/base/service.yaml b/deploy/kubernetes/ql3-cluster/operations/run-management/base/service.yaml new file mode 100644 index 00000000..f4f3d54e --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/run-management/base/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: ql3-run-management + namespace: qinglong3-system + labels: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management + app.kubernetes.io/part-of: qinglong3 +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: ql3-run-management + app.kubernetes.io/component: run-management + ports: + - name: https + port: 8448 + targetPort: https + protocol: TCP diff --git a/deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/deployment-patch.yaml b/deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/deployment-patch.yaml new file mode 100644 index 00000000..9bc4c237 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/deployment-patch.yaml @@ -0,0 +1,71 @@ +- op: replace + path: /spec/template/spec/containers/0/env + value: + - name: QL3_PROFILE + value: cluster-admin + - name: QL3_RUN_MANAGEMENT_ENABLED + value: 'true' + - name: QL3_RUN_MANAGEMENT_HOST + value: 0.0.0.0 + - name: QL3_RUN_MANAGEMENT_PORT + value: '8448' + - name: QL3_RUN_MANAGEMENT_TLS_CERT_FILE + value: /var/run/secrets/qinglong3/run-management-tls/tls.crt + - name: QL3_RUN_MANAGEMENT_TLS_KEY_FILE + value: /var/run/secrets/qinglong3/run-management-tls/tls.key + - name: QL3_RUN_MANAGEMENT_CLIENT_CA_FILE + value: /var/run/secrets/qinglong3/run-management-tls/ca.crt + - name: QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE + value: /var/run/secrets/qinglong3/run-management-tls/client.crl + - name: QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE + value: /var/run/qinglong3/run-management-identity/keyset.json + - name: QL3_RUN_MANAGEMENT_MAX_BODY_BYTES + value: '32768' + - name: QL3_RUN_MANAGEMENT_MAX_CONNECTIONS + value: '32' + - name: QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS + value: '16' + - name: QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS + value: '10000' + - name: QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS + value: '5000' + - name: QL3_RUN_MANAGEMENT_RATE_WINDOW_MS + value: '60000' + - name: QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT + value: '30' + - name: QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT + value: '300' + - name: QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS + value: '1024' + - name: QL3_POSTGRES_RUN_MANAGER_TLS_MODE + value: verify-full + - name: QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE + value: /var/run/secrets/qinglong3/postgres-run-manager/ca.crt + - name: QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME + value: qinglong3-run-manager + - name: QL3_POSTGRES_RUN_MANAGER_POOL_MAX + value: '2' + - name: QL3_POSTGRES_RUN_MANAGER_HOST + value: ql3-postgres-rw.qinglong3-system.svc + - name: QL3_POSTGRES_RUN_MANAGER_PORT + value: '5432' + - name: QL3_POSTGRES_RUN_MANAGER_DATABASE + value: qinglong + - name: QL3_POSTGRES_RUN_MANAGER_USER + valueFrom: + secretKeyRef: + name: ql3-postgres-run-manager-auth + key: username + - name: QL3_POSTGRES_RUN_MANAGER_PASSWORD + valueFrom: + secretKeyRef: + name: ql3-postgres-run-manager-auth + key: password + - name: QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME + value: ql3-postgres-rw.qinglong3-system.svc +- op: replace + path: /spec/template/spec/volumes/3/secret/secretName + value: ql3-postgres-ca +- op: replace + path: /spec/template/spec/volumes/3/secret/items/0/key + value: ca.crt diff --git a/deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/kustomization.yaml b/deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/kustomization.yaml new file mode 100644 index 00000000..da018124 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../base + +patches: + - target: + group: apps + version: v1 + kind: Deployment + labelSelector: app.kubernetes.io/name=ql3-run-management + path: deployment-patch.yaml + - target: + group: networking.k8s.io + version: v1 + kind: NetworkPolicy + name: ql3-run-management + path: network-policy-patch.yaml + +images: + - name: qinglong3-cluster-admin + newName: registry.example.com/qinglong/qinglong3-cluster-admin + # Fail closed until the independently verified release digest is supplied. + digest: sha256:0000000000000000000000000000000000000000000000000000000000000000 diff --git a/deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/network-policy-patch.yaml b/deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/network-policy-patch.yaml new file mode 100644 index 00000000..e0e337ed --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg/network-policy-patch.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: ql3-run-management + namespace: qinglong3-system +spec: + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - podSelector: + matchLabels: + cnpg.io/cluster: ql3-postgres + ports: + - protocol: TCP + port: 5432 diff --git a/deploy/kubernetes/ql3-cluster/operations/run-management/config.example.yaml b/deploy/kubernetes/ql3-cluster/operations/run-management/config.example.yaml new file mode 100644 index 00000000..9841942a --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/run-management/config.example.yaml @@ -0,0 +1,72 @@ +# Example only. Copy into a private overlay and replace every placeholder. +# This file is intentionally excluded from committed Kustomizations. +# Assertions require aud=qinglong3-run-management, +# typ=ql3-run-management+jwt and ql3_purpose=run-management. +apiVersion: v1 +kind: Secret +metadata: + name: ql3-run-management-identity + namespace: qinglong3-system +type: Opaque +stringData: + keyset.json: | + { + "schemaVersion": 1, + "generation": 1, + "issuer": "https://identity.example.test/", + "audience": "qinglong3-run-management", + "keys": [ + { + "alg": "EdDSA", + "crv": "Ed25519", + "kid": "REPLACE_WITH_KEY_ID", + "kty": "OKP", + "use": "sig", + "x": "REPLACE_WITH_ED25519_PUBLIC_JWK_X" + } + ], + "revokedKids": [], + "assuranceMappings": [ + { + "acr": "urn:example:mfa", + "assurance": "multi_factor", + "requiredAmr": ["pwd", "otp"] + }, + { + "acr": "urn:example:hardware", + "assurance": "hardware", + "requiredAmr": ["hwk"] + } + ], + "constraints": { + "maxAssertionBytes": 8192, + "maxLifetimeMs": 300000, + "maxAuthenticationAgeMs": 300000, + "clockSkewMs": 5000 + } + } +--- +apiVersion: v1 +kind: Secret +metadata: + name: ql3-run-management-tls + namespace: qinglong3-system +type: kubernetes.io/tls +stringData: + tls.crt: REPLACE_WITH_SERVER_CERTIFICATE_CHAIN + tls.key: REPLACE_WITH_SERVER_PRIVATE_KEY + ca.crt: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_AUTHORITIES + client.crl: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_REVOCATION_LISTS +--- +# Non-CloudNativePG deployments only. The reviewed overlay uses +# ql3-postgres-run-manager-auth and ql3-postgres-ca instead. +apiVersion: v1 +kind: Secret +metadata: + name: ql3-cluster-run-management-database + namespace: qinglong3-system +type: Opaque +stringData: + postgres-run-manager-url: REPLACE_WITH_RUN_MANAGER_DSN + postgres-tls-servername: REPLACE_WITH_POSTGRES_DNS_NAME + postgres-ca.crt: REPLACE_WITH_POSTGRES_CA_CERTIFICATE diff --git a/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/credentials.example.yaml b/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/credentials.example.yaml index c9bbbf7f..40f17e98 100644 --- a/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/credentials.example.yaml +++ b/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/credentials.example.yaml @@ -122,6 +122,18 @@ stringData: --- apiVersion: v1 kind: Secret +metadata: + name: ql3-postgres-run-manager-auth + namespace: qinglong3-system + labels: + cnpg.io/reload: 'true' +type: kubernetes.io/basic-auth +stringData: + username: ql3_run_manager + password: REPLACE_WITH_SECRET_MANAGER_VALUE +--- +apiVersion: v1 +kind: Secret metadata: name: ql3-postgres-worker-credential-manager-auth namespace: qinglong3-system diff --git a/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/database-roles.yaml b/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/database-roles.yaml index 8c1363f1..fcc8ee59 100644 --- a/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/database-roles.yaml +++ b/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/database-roles.yaml @@ -240,6 +240,30 @@ spec: --- apiVersion: postgresql.cnpg.io/v1 kind: DatabaseRole +metadata: + name: ql3-postgres-run-manager + labels: + app.kubernetes.io/name: ql3-postgres + app.kubernetes.io/component: database-role + app.kubernetes.io/part-of: qinglong3 +spec: + cluster: + name: ql3-postgres + name: ql3_run_manager + comment: QingLong strongly authenticated human Run retry authority + login: true + superuser: false + createdb: false + createrole: false + replication: false + bypassrls: false + connectionLimit: 4 + databaseRoleReclaimPolicy: retain + passwordSecret: + name: ql3-postgres-run-manager-auth +--- +apiVersion: postgresql.cnpg.io/v1 +kind: DatabaseRole metadata: name: ql3-postgres-worker-credential-manager labels: diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index ac96144f..347ac741 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,22 @@ 最新增量证据（2026-08-12）： +- D-295/ADR-0383（已接受） + Cluster 手动 `run.retry` 已从“仅有 PostgreSQL authority”推进到独立强认证产品面：能力内聚在既有 `@qinglong/cluster-admin` + 的 `run-management/` 目录，不新增 workspace package；只有 `QL3_PROFILE=cluster-admin` 且显式启用时才读取 mTLS/CRL、 + purpose-bound OIDC keyset 和创建 PostgreSQL Pool/HTTPS listener。固定 route `/api/v3/runs/management` 同时要求 mTLS 与 + `aud=qinglong3-run-management`、`typ=ql3-run-management+jwt`、`ql3_purpose=run-management` 的五分钟内 + `multi_factor|hardware` User，服务端生成新 Run/Attempt/Event identity；keyset generation 使用独立 durable + `run-management` authority。PostgreSQL capability v55 / migration `pg-0056-run-management-boundary` 引入专用 + `ql3_run_manager` 与最小 `SECURITY DEFINER` Project/RoleBinding lock function，产品进程不再使用宽泛的常驻 runtime role；精确 + replay 改由 immutable created/queued Events 恢复最初创建事实，因此调度进展后仍返回原 durable identity。Kubernetes 部署作为 + `operations/run-management` opt-in overlay 提供两副本/PDB/反亲和/私网 egress，默认 Edge、Standalone 与 Cluster base 均不引用， + 低配设备继续保持零新增进程、连接、timer、listener、watcher、cache 或 sidecar。完整 18-package clean build/test 退出 0； + backend 1,165 pass/2 conditional skip/0 fail；workspace 保持 18 package/1,070 source/1,052 nested，且无 single-source + 或 shallow package。PostgreSQL migration ledger 直属文件仍受 58 hard cap 约束，v55 migration 已归入既有 `run-management` + 领域目录而非放宽阈值。真实 PostgreSQL 18.4 arm64 physical HA 以两个 `ql3_run_manager` Pool 验证 exact concurrent replay、 + 最后 quota slot、独立 keyset ledger、同步 WAL 与 promotion 后事实，共 119 gates、timeline `1→2`；报告 SHA-256 + `6ca8ccfb48841589e10c6484f5c97ce72e24b123f3abb1066a639e63718e64c6`，离线证据审计无 finding。 - D-294/ADR-0382（已接受） Cluster 已补齐共享 `qinglong/run-manual-retry@v1` 的 PostgreSQL 原子 authority，但在可信强认证 transport 完成前保持产品 route 关闭。adapter 只接受五分钟内的 `multi_factor|hardware` User，在单个 `SERIALIZABLE` 事务中取得 Project 行锁，重验 diff --git a/docs/adr/ADR-0382-postgresql-cluster-manual-run-retry-authority.md b/docs/adr/ADR-0382-postgresql-cluster-manual-run-retry-authority.md index fd5ad6f1..b062d800 100644 --- a/docs/adr/ADR-0382-postgresql-cluster-manual-run-retry-authority.md +++ b/docs/adr/ADR-0382-postgresql-cluster-manual-run-retry-authority.md @@ -5,6 +5,8 @@ - 关联 RFC：QL-RFC-0001 D-294 - 前置决策：ADR-0039、ADR-0047、ADR-0119、ADR-0361、ADR-0381 +> 产品装配说明：ADR-0383 已 supersede 本文“复用 `ql3_runtime` 且不新增 role/migration”的决策。本文的共享 retry 语义、原子 repository 与 HA 证据继续有效；生产入口改由强认证的独立 Run Management Plane 和 `ql3_run_manager` 承载。 + ## 上下文 ADR-0381 已冻结手动 retry 的共享语义并完成 Local SQLite/CLI 纵向切片，但 Cluster 尚缺少可在多副本下工作的 PostgreSQL authority。不能把 Local 的进程内状态或单连接假设复制到 Cluster，也不能让每个 `cluster-control` 副本各自维护限流 bucket。 diff --git a/docs/adr/ADR-0383-strong-cluster-run-management-plane.md b/docs/adr/ADR-0383-strong-cluster-run-management-plane.md new file mode 100644 index 00000000..7005933a --- /dev/null +++ b/docs/adr/ADR-0383-strong-cluster-run-management-plane.md @@ -0,0 +1,74 @@ +# ADR-0383：强认证的 Cluster Run Management Plane + +- 状态：Accepted +- 日期：2026-08-12 +- 关联 RFC：QL-RFC-0001 D-295 +- 前置决策：ADR-0039、ADR-0056、ADR-0356、ADR-0364、ADR-0366、ADR-0381、ADR-0382 +- Supersedes：ADR-0382 中“复用 `ql3_runtime` 且不新增 role/migration”的产品装配决策；ADR-0382 的共享语义与 PostgreSQL 原子事务仍有效 + +## 上下文 + +ADR-0382 已证明 PostgreSQL 手动 Run retry 的原子语义与 HA 收敛，但现有 Cluster Control bearer 只能建立 `single_factor` User，不能承载会再次执行外部副作用的人工恢复操作。直接把 repository 接进通用 Cluster HTTP 会混合普通控制面与强人类认证 authority；继续使用 `ql3_runtime` 又会使常驻 runtime 持有本不需要的人工恢复权限。 + +QingLong 同时服务低资源路由设备与多节点集群。新能力不能让 Edge/Standalone 增加进程、连接、timer 或 Cluster 依赖，也不应为了一个内聚领域再制造单文件 workspace package。 + +## 决策 + +### 1. 能力归属既有 Cluster Admin package + +Run management 作为 `@qinglong/cluster-admin` 内的 `run-management/` 领域目录发布 service、transport、HTTPS process、client 与两个 CLI，不新增 workspace package。PostgreSQL adapter 仍归属 `@qinglong/cluster-postgres/run-manager`。根目录只保留协议装配入口，领域源码不得重新平铺。 + +### 2. 独立、显式启用的强认证进程 + +管理端只在 `QL3_PROFILE=cluster-admin` 且 `QL3_RUN_MANAGEMENT_ENABLED=true` 时创建资源；关闭时不读取证书、keyset 或数据库配置，也不创建 Pool、listener、timer、watcher 或 cache。服务使用独立 HTTPS 端口 `8448` 和固定 route `/api/v3/runs/management`，同时要求： + +1. 受信 client CA 与 CRL 校验的 mTLS； +2. purpose-bound OIDC assertion：`aud=qinglong3-run-management`、`typ=ql3-run-management+jwt`、`ql3_purpose=run-management`； +3. 五分钟内的 `multi_factor|hardware` User； +4. 精确的 `qinglong/run-manual-retry@v1` command envelope，不接受调用方指定新 Run/Attempt/Event identity； +5. route admission 与数据库事务内分别重验 identity/Policy/audit fence。 + +Assertion keyset generation 使用 durable PostgreSQL ledger 的独立 `run-management` authority，不能与 Plugin、Worker、automation 或 Approval keyset 互换。client 是 caller-driven 的一次性命令，不引入常驻 agent。 + +### 3. 专用最小权限数据库角色 + +Migration `pg-0056-run-management-boundary` / capability v55 引入 `ql3_run_manager`。该角色只读 Project/RoleBinding/Task/execution revision，只对 Runs、Attempts、Events、security audit 和本 authority 的 identity ledger 取得精确所需权限；它没有 migration、admin、Worker、AI、Approval 或任意表 DELETE 权限，也没有更新既有 Run 的权限。 + +`SECURITY DEFINER` 函数 `ql3.lock_run_management_policy_fence` 只负责锁定 Project 并验证 active owner/admin/operator binding，固定 `search_path`，并仅向 `ql3_runtime` 与 `ql3_run_manager` 授予 EXECUTE。repository 不以 `FOR UPDATE` 读取源/重放 Run，因此不因 PostgreSQL 锁语法扩大 table UPDATE privilege。 + +精确 replay 从 immutable created/queued Events 恢复最初创建事实，而不是依赖被重试 Run 当前仍为 queued；调度进展后相同 mutation 仍返回原始 durable identity。 + +### 4. 部署按 Profile 付费 + +默认 Edge、Standalone 及 Cluster base overlay 都不引用 Run management manifests。需要该能力的集群显式应用 `operations/run-management`；CloudNativePG overlay 使用专用 Secret、primary DNS 与最多两个数据库连接。生产模板为两副本、PDB、反亲和、只读根文件系统、无 ServiceAccount token、受限 ingress 与只到 DNS/PostgreSQL 的 egress。低配单机不承担这些资源。 + +## 验收 + +- service/transport/process/client/HTTPS 的单元与回环测试覆盖强认证、purpose isolation、错误映射、限流、drain 与 exact replay； +- PostgreSQL package 全量测试覆盖 migration checksum、schema/readiness、role privilege 与调度后 replay； +- CloudNativePG 静态审计必须固定十四个角色和非秘密 placeholder；Run management deployment 必须证明 opt-in、mTLS、专用 role 与私网 egress； +- 真实 PostgreSQL 18.4 HA 必须以迁移后角色完成 readiness、事务写入、同步复制与 promotion 后事实核验； +- 完整 workspace package/backend/dependency/package/Profile artifact 门全部通过，GitNexus staged scope 只包含本阶段预期 flow，才允许提交。 + +## 被否决的替代方案 + +1. **接入通用 Cluster Control bearer**：认证强度不足且会扩大普通控制面的恢复执行 authority。 +2. **继续让 `ql3_runtime` 作为产品管理角色**：常驻 runtime 无需持有人工 retry 与身份 keyset 写权限。 +3. **新增 `@qinglong/run-management` package**：没有独立依赖、制品或部署生命周期收益，会重新产生单文件/浅 package。 +4. **每个 Cluster Control Pod 内置管理 listener**：副本与普通 runtime 同扩缩，增加攻击面和低配常驻资源。 +5. **允许调用方提交新 aggregate identity**：会扩大 replay/conflict 表面并削弱 server-authored fact。 + +## 验收证据（2026-08-12） + +- `@qinglong/cluster-postgres`：309 pass、1 个外部数据库条件 skip、0 fail；`@qinglong/cluster-admin`：281 pass、2 个外部集成条件 skip、0 fail；`@qinglong/cluster-control`：230 pass、2 skip、0 fail； +- 完整 18-package clean build/test 全部退出 0；backend 1,167 项中 1,165 pass、2 个环境条件 skip、0 fail； +- workspace 保持 18 个 package，源码为 1,070 个（1,052 个 nested）；`singleSourcePackages=[]`、`shallowSourcePackages=[]`。PostgreSQL ordered migration ledger 直属文件仍为 58，v55 migration 归入既有 `run-management` 领域目录，没有放宽 dense-directory cap； +- CloudNativePG 静态审计固定 14 个最小权限角色；Run Management deployment 静态门证明默认 overlay 不引用、mTLS/OIDC authority 私有、Pool 上限 2、无 ServiceAccount token、只读根和仅 DNS/PostgreSQL egress； +- PostgreSQL 18.4 arm64 physical HA：119 gates、timeline `1→2`，两个独立 `ql3_run_manager` Pool 完成 exact concurrent replay 与最后 quota slot 竞争，`run-management` keyset ledger 经双连接、重启、commit-response-loss 验证；报告 SHA-256 `6ca8ccfb48841589e10c6484f5c97ce72e24b123f3abb1066a639e63718e64c6`，离线审计 `compatible:true`。 + +## 影响 + +- PostgreSQL schema contract 从 v54 升至 v55，生产部署在启用管理面前必须先创建 `ql3_run_manager` 并运行 migration； +- Cluster Admin image 增加两个 opt-in binary，但默认 Profile 不启动它们； +- ADR-0382 的原子 repository 和既有 HA 证据继续成立，产品装配改由本 ADR 的强认证进程与专用角色承载； +- UI 仍需以同一 transport 展示 source/new Run linkage、终态原因和 retry preview，不得绕过该 authority。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 5c335d64..e68eb637 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -386,6 +386,7 @@ | [ADR-0380](./ADR-0380-local-lost-run-retry-single-control-cadence.md) | Local lost Run retry 复用单一 execution-control cadence | Accepted | | [ADR-0381](./ADR-0381-strong-local-manual-run-retry.md) | 强认证的 Local 手动 Run retry | Accepted | | [ADR-0382](./ADR-0382-postgresql-cluster-manual-run-retry-authority.md) | PostgreSQL Cluster 手动 Run retry 原子 Authority | Accepted | +| [ADR-0383](./ADR-0383-strong-cluster-run-management-plane.md) | 强认证的 Cluster Run Management Plane 与专用数据库角色 | Accepted | ## 规则 diff --git a/packages/ql3-cluster-admin/package.json b/packages/ql3-cluster-admin/package.json index 1071edad..9f73502e 100644 --- a/packages/ql3-cluster-admin/package.json +++ b/packages/ql3-cluster-admin/package.json @@ -20,6 +20,31 @@ "require": "./dist/approval-management/approvalDecisionManagement.js", "default": "./dist/approval-management/approvalDecisionManagement.js" }, + "./run-management": { + "types": "./dist/run-management/runManagement.d.ts", + "require": "./dist/run-management/runManagement.js", + "default": "./dist/run-management/runManagement.js" + }, + "./run-management-transport": { + "types": "./dist/run-management/runManagementTransport.d.ts", + "require": "./dist/run-management/runManagementTransport.js", + "default": "./dist/run-management/runManagementTransport.js" + }, + "./run-management-http": { + "types": "./dist/run-management/runManagementHttp.d.ts", + "require": "./dist/run-management/runManagementHttp.js", + "default": "./dist/run-management/runManagementHttp.js" + }, + "./run-management-process": { + "types": "./dist/run-management/runManagementProcess.d.ts", + "require": "./dist/run-management/runManagementProcess.js", + "default": "./dist/run-management/runManagementProcess.js" + }, + "./run-management-client": { + "types": "./dist/run-management/runManagementClient.d.ts", + "require": "./dist/run-management/runManagementClient.js", + "default": "./dist/run-management/runManagementClient.js" + }, "./approval-management": { "types": "./dist/approval-management/approvalManagement.d.ts", "require": "./dist/approval-management/approvalManagement.js", @@ -331,6 +356,8 @@ "ql3-worker-credential-client": "dist/worker-credential/workerCredentialManagementClientCli.js", "ql3-approval-manage": "dist/approval-management/approvalManagementCli.js", "ql3-approval-client": "dist/approval-management/approvalManagementClientCli.js", + "ql3-run-manage": "dist/run-management/runManagementCli.js", + "ql3-run-client": "dist/run-management/runManagementClientCli.js", "ql3-automation-manage": "dist/automation-management/automationManagementCli.js", "ql3-automation-client": "dist/automation-management/automationManagementClientCli.js", "ql3-provider-credential-manage": "dist/model-provider-credential/modelProviderCredentialManagementCli.js", diff --git a/packages/ql3-cluster-admin/src/management-support/pluginPackageIdentityAssertion.ts b/packages/ql3-cluster-admin/src/management-support/pluginPackageIdentityAssertion.ts index 3fc00b1a..e1990372 100644 --- a/packages/ql3-cluster-admin/src/management-support/pluginPackageIdentityAssertion.ts +++ b/packages/ql3-cluster-admin/src/management-support/pluginPackageIdentityAssertion.ts @@ -80,6 +80,11 @@ export const CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PRO purpose: 'model-provider-credential-management', }); +export const CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE = Object.freeze({ + type: 'ql3-run-management+jwt', + purpose: 'run-management', +}); + export interface ClusterPluginPackageIdentityAssertionVerifierOptions { readonly issuer: string; readonly audience: string; diff --git a/packages/ql3-cluster-admin/src/management-support/pluginPackageIdentityKeyset.ts b/packages/ql3-cluster-admin/src/management-support/pluginPackageIdentityKeyset.ts index 2d93e731..d4c788e2 100644 --- a/packages/ql3-cluster-admin/src/management-support/pluginPackageIdentityKeyset.ts +++ b/packages/ql3-cluster-admin/src/management-support/pluginPackageIdentityKeyset.ts @@ -9,6 +9,7 @@ import { CLUSTER_AUTOMATION_MANAGEMENT_IDENTITY_ASSERTION_PROFILE, CLUSTER_APPROVAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE, CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE, + CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE, createClusterPluginPackageIdentityAssertionVerifier, type ClusterManagementIdentityAssertionProfile, type ClusterPluginPackageIdentityAssertionAuthentication, @@ -507,3 +508,12 @@ export function createClusterModelProviderCredentialIdentityKeysetFile( CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE, }); } + +export function createClusterRunIdentityKeysetFile( + options: ClusterWorkerCredentialIdentityKeysetFileOptions, +): Readonly { + return createClusterPluginPackageIdentityKeysetFile({ + ...options, + assertionProfile: CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE, + }); +} diff --git a/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts b/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts index b89b3c3d..32a73986 100644 --- a/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts +++ b/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts @@ -67,6 +67,19 @@ import { ClusterModelProviderCredentialManagementTransportRequestError, ClusterModelProviderCredentialManagementTransportUnavailableError, } from '../model-provider-credential/modelProviderCredentialManagementTransport'; +import { + ClusterRunManagementAuthorizationError, + ClusterRunManagementConflictError, + ClusterRunManagementRateLimitedError, + ClusterRunManagementRequestError, + ClusterRunManagementTargetUnavailableError, + ClusterRunManagementUnavailableError, +} from '../run-management/runManagement'; +import { + ClusterRunManagementTransportAuthenticationError, + ClusterRunManagementTransportRequestError, + ClusterRunManagementTransportUnavailableError, +} from '../run-management/runManagementTransport'; export const CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH = '/api/v3/plugin-packages/management'; @@ -77,18 +90,21 @@ export const CLUSTER_AUTOMATION_MANAGEMENT_PATH = export const CLUSTER_APPROVAL_MANAGEMENT_PATH = '/api/v3/approvals/management'; export const CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH = '/api/v3/provider-credentials/management'; +export const CLUSTER_RUN_MANAGEMENT_PATH = '/api/v3/runs/management'; export type ClusterAuthenticatedManagementPath = | typeof CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH | typeof CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH | typeof CLUSTER_AUTOMATION_MANAGEMENT_PATH | typeof CLUSTER_APPROVAL_MANAGEMENT_PATH - | typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH; + | typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH + | typeof CLUSTER_RUN_MANAGEMENT_PATH; const MANAGEMENT_PATHS = new Set([ CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH, CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH, CLUSTER_AUTOMATION_MANAGEMENT_PATH, CLUSTER_APPROVAL_MANAGEMENT_PATH, CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH, + CLUSTER_RUN_MANAGEMENT_PATH, ]); const DEFAULT_MAX_BODY_BYTES = 64 * 1024; const DEFAULT_MAX_CONNECTIONS = 64; @@ -531,7 +547,8 @@ function responseError(error: unknown): HttpRequestError { error instanceof ClusterApprovalManagementTransportAuthenticationError || error instanceof ClusterModelProviderCredentialManagementTransportAuthenticationError || - error instanceof ClusterModelProviderCredentialManagementAuthenticationError + error instanceof ClusterModelProviderCredentialManagementAuthenticationError || + error instanceof ClusterRunManagementTransportAuthenticationError ) { return new HttpRequestError(401, 'authentication_required'); } @@ -544,6 +561,8 @@ function responseError(error: unknown): HttpRequestError { error instanceof ClusterModelProviderCredentialManagementTransportRequestError || error instanceof ClusterModelProviderCredentialManagementRequestError || + error instanceof ClusterRunManagementTransportRequestError || + error instanceof ClusterRunManagementRequestError || error instanceof PluginPackageManagementRequestError || error instanceof WorkerCredentialManagementRequestError ) { @@ -554,7 +573,8 @@ function responseError(error: unknown): HttpRequestError { error instanceof WorkerCredentialManagementAuthorizationError || error instanceof ClusterAutomationManagementAuthorizationError || error instanceof ClusterApprovalManagementTransportAuthorizationError || - error instanceof ClusterModelProviderCredentialManagementAuthorizationError + error instanceof ClusterModelProviderCredentialManagementAuthorizationError || + error instanceof ClusterRunManagementAuthorizationError ) { return new HttpRequestError(403, 'forbidden'); } @@ -563,7 +583,8 @@ function responseError(error: unknown): HttpRequestError { error instanceof WorkerCredentialManagementConflictError || error instanceof ClusterAutomationManagementConflictError || error instanceof ClusterApprovalManagementTransportConflictError || - error instanceof ClusterModelProviderCredentialManagementConflictError + error instanceof ClusterModelProviderCredentialManagementConflictError || + error instanceof ClusterRunManagementConflictError ) { return new HttpRequestError(409, 'conflict'); } @@ -578,8 +599,12 @@ function responseError(error: unknown): HttpRequestError { ) { return new HttpRequestError(429, 'quota_exceeded', error.retryAfterMs); } + if (error instanceof ClusterRunManagementRateLimitedError) { + return new HttpRequestError(429, 'rate_limited', error.retryAfterMs); + } if ( - error instanceof ClusterApprovalManagementTransportTargetUnavailableError + error instanceof ClusterApprovalManagementTransportTargetUnavailableError || + error instanceof ClusterRunManagementTargetUnavailableError ) { return new HttpRequestError(404, 'not_found'); } @@ -594,6 +619,8 @@ function responseError(error: unknown): HttpRequestError { error instanceof ClusterModelProviderCredentialManagementTransportUnavailableError || error instanceof ClusterModelProviderCredentialManagementUnavailableError || + error instanceof ClusterRunManagementTransportUnavailableError || + error instanceof ClusterRunManagementUnavailableError || error instanceof PluginPackageManagementUnavailableError || error instanceof WorkerCredentialManagementUnavailableError ) { diff --git a/packages/ql3-cluster-admin/src/run-management/runManagement.ts b/packages/ql3-cluster-admin/src/run-management/runManagement.ts new file mode 100644 index 00000000..b4043c91 --- /dev/null +++ b/packages/ql3-cluster-admin/src/run-management/runManagement.ts @@ -0,0 +1,273 @@ +import { randomUUID } from 'node:crypto'; + +import { + PostgresProjectPolicyRepository, + PostgresRunManualRetryRepository, + PostgresSecurityAuditRepository, +} from '@qinglong/cluster-postgres/run-manager'; +import type { PostgresPool } from '@qinglong/runtime-core'; +import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy'; +import { + InvalidRunManualRetryError, + RunManualRetryFenceRejectedError, + RunManualRetryNotFoundError, + RunManualRetryRateLimitedError, + RunManualRetryUnavailableError, + type RunManualRetryResult, + type RunManualRetrySourceStatus, +} from '@qinglong/runtime-core/run-manual-retry'; +import { + normalizeSecurityPrincipal, + type SecurityPolicyFence, + type SecurityPrincipal, +} from '@qinglong/runtime-core/security'; +import { normalizeSecurityAuditRecord } from '@qinglong/runtime-core/security-audit'; + +const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const UUID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +export interface ClusterRunManagementRetryRequest { + readonly projectId: string; + readonly sourceRunId: string; + readonly mutationId: string; + readonly expectedRunVersion: number; + readonly expectedRunStatus: RunManualRetrySourceStatus; + readonly requestId: string; + readonly auditEventId: string; + readonly failureAuditEventId: string; + readonly principal: Readonly; +} + +export interface ClusterRunManagementService { + retry( + request: Readonly, + ): Promise>; +} + +export interface ClusterRunManagementOptions { + readonly pool: PostgresPool; + readonly now?: () => number; + readonly randomUuid?: () => string; +} + +export class ClusterRunManagementConfigurationError extends TypeError { + readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFIGURATION_INVALID'; + constructor() { + super('Cluster Run management configuration is invalid'); + this.name = 'ClusterRunManagementConfigurationError'; + } +} + +export class ClusterRunManagementRequestError extends TypeError { + readonly code = 'CLUSTER_RUN_MANAGEMENT_REQUEST_INVALID'; + constructor() { + super('Cluster Run management request is invalid'); + this.name = 'ClusterRunManagementRequestError'; + } +} + +export class ClusterRunManagementAuthorizationError extends Error { + readonly code = 'CLUSTER_RUN_MANAGEMENT_FORBIDDEN'; + constructor() { + super('Cluster Run management is forbidden'); + this.name = 'ClusterRunManagementAuthorizationError'; + } +} + +export class ClusterRunManagementTargetUnavailableError extends Error { + readonly code = 'CLUSTER_RUN_MANAGEMENT_TARGET_UNAVAILABLE'; + constructor() { + super('Cluster Run management target is unavailable'); + this.name = 'ClusterRunManagementTargetUnavailableError'; + } +} + +export class ClusterRunManagementConflictError extends Error { + readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFLICT'; + constructor() { + super('Cluster Run management conflicts with durable state'); + this.name = 'ClusterRunManagementConflictError'; + } +} + +export class ClusterRunManagementRateLimitedError extends Error { + readonly code = 'CLUSTER_RUN_MANAGEMENT_RATE_LIMITED'; + constructor(readonly retryAfterMs: number) { + super('Cluster Run management rate limit is exhausted'); + this.name = 'ClusterRunManagementRateLimitedError'; + } +} + +export class ClusterRunManagementUnavailableError extends Error { + readonly code = 'CLUSTER_RUN_MANAGEMENT_UNAVAILABLE'; + constructor(options?: ErrorOptions) { + super('Cluster Run management is unavailable', options); + this.name = 'ClusterRunManagementUnavailableError'; + } +} + +function exactRequest( + value: unknown, +): asserts value is Readonly { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + Object.keys(value).sort().join('\0') !== + [ + 'auditEventId', + 'expectedRunStatus', + 'expectedRunVersion', + 'failureAuditEventId', + 'mutationId', + 'principal', + 'projectId', + 'requestId', + 'sourceRunId', + ] + .sort() + .join('\0') + ) { + throw new ClusterRunManagementRequestError(); + } +} + +function validUuid(value: unknown): value is string { + return typeof value === 'string' && UUID_PATTERN.test(value); +} + +function failureReason(error: unknown): string { + if (error instanceof ClusterRunManagementAuthorizationError) { + return 'authorization_rejected'; + } + if (error instanceof RunManualRetryNotFoundError) return 'run_not_found'; + if (error instanceof RunManualRetryRateLimitedError) return 'rate_limited'; + if (error instanceof RunManualRetryFenceRejectedError) return error.reason; + return 'management_unavailable'; +} + +/** Strong OIDC Run management composition over one run-manager Pool. */ +export function createClusterRunManagementService( + options: ClusterRunManagementOptions, +): Readonly { + if ( + !options || + typeof options !== 'object' || + Array.isArray(options) || + Object.keys(options).some( + (key) => key !== 'pool' && key !== 'now' && key !== 'randomUuid', + ) || + !options.pool || + typeof options.pool.query !== 'function' || + typeof options.pool.connect !== 'function' || + (options.now !== undefined && typeof options.now !== 'function') || + (options.randomUuid !== undefined && typeof options.randomUuid !== 'function') + ) { + throw new ClusterRunManagementConfigurationError(); + } + const now = options.now ?? Date.now; + const createId = options.randomUuid ?? randomUUID; + const policy = new ProjectPolicyEngine( + new PostgresProjectPolicyRepository(options.pool), + ); + const retries = new PostgresRunManualRetryRepository(options.pool); + const audit = new PostgresSecurityAuditRepository(options.pool); + + return Object.freeze({ + async retry(requestValue: Readonly) { + exactRequest(requestValue); + const observedAtMs = now(); + let principal: Readonly; + if ( + !Number.isSafeInteger(observedAtMs) || + observedAtMs < 0 || + !IDENTIFIER_PATTERN.test(requestValue.projectId) || + !IDENTIFIER_PATTERN.test(requestValue.sourceRunId) || + !IDENTIFIER_PATTERN.test(requestValue.requestId) || + !validUuid(requestValue.mutationId) || + !validUuid(requestValue.auditEventId) || + !validUuid(requestValue.failureAuditEventId) || + requestValue.auditEventId === requestValue.failureAuditEventId + ) { + throw new ClusterRunManagementRequestError(); + } + try { + principal = normalizeSecurityPrincipal( + requestValue.principal, + observedAtMs, + ); + } catch { + throw new ClusterRunManagementRequestError(); + } + + let fence: Readonly | null = null; + try { + const decision = await policy.authorize( + principal, + requestValue.projectId, + 'run.retry', + ); + fence = decision.fence; + if ( + decision.effect !== 'allow' || + !fence || + fence.bindingVersion === null + ) { + throw new ClusterRunManagementAuthorizationError(); + } + return await retries.retryRun({ + projectId: requestValue.projectId, + sourceRunId: requestValue.sourceRunId, + mutationId: requestValue.mutationId, + expectedRunVersion: requestValue.expectedRunVersion, + expectedRunStatus: requestValue.expectedRunStatus, + runId: createId(), + attemptId: createId(), + createdEventId: createId(), + queuedEventId: createId(), + auditEventId: requestValue.auditEventId, + requestId: requestValue.requestId, + principal, + policyFence: fence, + }); + } catch (error) { + try { + await audit.record( + normalizeSecurityAuditRecord({ + eventId: requestValue.failureAuditEventId, + requestId: requestValue.requestId, + operationId: 'run.retry', + projectId: requestValue.projectId, + subject: principal.subject, + authenticationId: principal.authenticationId, + outcome: 'denied', + reasons: [failureReason(error)], + fence, + occurredAtMs: observedAtMs, + }), + ); + } catch (auditError) { + throw new ClusterRunManagementUnavailableError({ cause: auditError }); + } + if (error instanceof ClusterRunManagementAuthorizationError) throw error; + if (error instanceof InvalidRunManualRetryError) { + throw new ClusterRunManagementRequestError(); + } + if (error instanceof RunManualRetryNotFoundError) { + throw new ClusterRunManagementTargetUnavailableError(); + } + if (error instanceof RunManualRetryFenceRejectedError) { + throw new ClusterRunManagementConflictError(); + } + if (error instanceof RunManualRetryRateLimitedError) { + throw new ClusterRunManagementRateLimitedError(error.retryAfterMs); + } + if (error instanceof RunManualRetryUnavailableError) { + throw new ClusterRunManagementUnavailableError({ cause: error }); + } + throw new ClusterRunManagementUnavailableError({ cause: error }); + } + }, + }); +} diff --git a/packages/ql3-cluster-admin/src/run-management/runManagementCli.ts b/packages/ql3-cluster-admin/src/run-management/runManagementCli.ts new file mode 100644 index 00000000..847388b3 --- /dev/null +++ b/packages/ql3-cluster-admin/src/run-management/runManagementCli.ts @@ -0,0 +1,74 @@ +#!/usr/bin/env node + +import { + startClusterRunManagementProcess, + type ClusterRunManagementProcessRuntime, +} from './runManagementProcess'; + +const USAGE = 'Usage: ql3-run-manage'; + +function fact(error: unknown): Readonly> { + const candidate = error as { readonly name?: unknown; readonly code?: unknown }; + return Object.freeze({ + schemaVersion: 1, + component: 'qinglong3-run-management', + event: 'management_failed', + name: typeof candidate?.name === 'string' ? candidate.name : 'Error', + ...(typeof candidate?.code === 'string' ? { code: candidate.code } : {}), + }); +} + +function emit(value: Readonly>): void { + process.stdout.write(`${JSON.stringify(value)}\n`); +} + +async function run(argv: readonly string[]): Promise { + if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) { + process.stdout.write(`${USAGE}\n`); + return; + } + if (argv.length !== 0) { + process.stderr.write(`${JSON.stringify({ code: 'QL3_RUN_MANAGEMENT_CLI_USAGE_INVALID', message: USAGE })}\n`); + process.exitCode = 64; + return; + } + let runtime: Readonly; + try { + runtime = await startClusterRunManagementProcess({ + environment: process.env, + onError: () => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_unavailable' }), + }); + } catch (error) { + process.stderr.write(`${JSON.stringify(fact(error))}\n`); + process.exitCode = 1; + return; + } + if (runtime.status === 'disabled') { + emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_disabled' }); + return; + } + emit({ + schemaVersion: 1, + component: 'qinglong3-run-management', + event: 'management_started', + address: runtime.address, + identityGeneration: runtime.identity.generation, + databaseContractVersion: runtime.database.contractVersion, + databaseMigrationCount: runtime.database.migrationIds.length, + }); + let stopping: Promise | undefined; + const stop = (): Promise => { + stopping ??= runtime.close().then(() => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_stopped' })); + return stopping; + }; + for (const signal of ['SIGINT', 'SIGTERM'] as const) { + process.once(signal, () => { + void stop().then( + () => { process.exitCode = 0; }, + (error) => { process.stderr.write(`${JSON.stringify(fact(error))}\n`); process.exitCode = 1; }, + ); + }); + } +} + +void run(process.argv.slice(2)); diff --git a/packages/ql3-cluster-admin/src/run-management/runManagementClient.ts b/packages/ql3-cluster-admin/src/run-management/runManagementClient.ts new file mode 100644 index 00000000..8708e489 --- /dev/null +++ b/packages/ql3-cluster-admin/src/run-management/runManagementClient.ts @@ -0,0 +1,106 @@ +import { + RUN_MANUAL_RETRY_SCHEMA, + normalizeRunManualRetryResult, +} from '@qinglong/runtime-core/run-manual-retry'; +import { + ClusterPluginPackageManagementClientRequestError, + executeClusterAuthenticatedManagementClient, + type ClusterAuthenticatedManagementClientResult, + type ClusterPluginPackageManagementClientConnectionOptions, + type ClusterPluginPackageManagementClientPaths, +} from '../management-support/pluginPackageManagementClient'; +import { + normalizeClusterRunManagementCommand, + type ClusterRunManagementCommand, + type ClusterRunManagementTransportResult, +} from './runManagementTransport'; + +const MANAGEMENT_PATH = '/api/v3/runs/management'; + +export type ClusterRunManagementClientPaths = + ClusterPluginPackageManagementClientPaths; +export type ClusterRunManagementClientConnectionOptions = + ClusterPluginPackageManagementClientConnectionOptions; +export type ClusterRunManagementClientResult = + ClusterAuthenticatedManagementClientResult; + +function invalid(): never { + throw new ClusterPluginPackageManagementClientRequestError(); +} + +function exact(value: unknown, keys: readonly string[]): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) invalid(); + const actual = Object.keys(value as object).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + invalid(); + } + return value as Record; +} + +export function validateClusterRunManagementClientResult( + value: unknown, + command: Readonly, +): Readonly { + const envelope = exact(value, ['schemaVersion', 'operation', 'retry']); + if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid(); + const retry = exact(envelope.retry, [ + 'schema', + 'status', + 'projectId', + 'sourceRunId', + 'sourceRunStatus', + 'sourceRunVersion', + 'runId', + 'retryOfRunId', + 'taskId', + 'taskRevision', + 'attemptId', + 'runStatus', + 'runVersion', + 'eventSequence', + 'executorType', + 'executionRevisionDigest', + 'createdAtMs', + ]); + if (retry.schema !== RUN_MANUAL_RETRY_SCHEMA) invalid(); + try { + const { schema: _schema, ...result } = retry; + const normalized = normalizeRunManualRetryResult(result as never); + if ( + normalized.projectId !== command.request.projectId || + normalized.sourceRunId !== command.request.sourceRunId || + normalized.sourceRunVersion !== command.request.body.expectedRunVersion || + normalized.sourceRunStatus !== command.request.body.expectedRunStatus || + normalized.executorType !== 'remote_worker' + ) { + invalid(); + } + } catch { + invalid(); + } + return Object.freeze( + envelope as unknown as ClusterRunManagementTransportResult, + ); +} + +const PROTOCOL = Object.freeze({ + managementPath: MANAGEMENT_PATH, + clientCertificate: 'required' as const, + normalizeCommand: normalizeClusterRunManagementCommand, + validateResult: validateClusterRunManagementClientResult, +}); + +export function executeClusterRunManagementClient( + paths: ClusterRunManagementClientPaths, + connectionOptions?: ClusterRunManagementClientConnectionOptions, +): Promise> { + return executeClusterAuthenticatedManagementClient( + paths, + PROTOCOL, + connectionOptions, + ); +} diff --git a/packages/ql3-cluster-admin/src/run-management/runManagementClientCli.ts b/packages/ql3-cluster-admin/src/run-management/runManagementClientCli.ts new file mode 100644 index 00000000..c12a7525 --- /dev/null +++ b/packages/ql3-cluster-admin/src/run-management/runManagementClientCli.ts @@ -0,0 +1,67 @@ +#!/usr/bin/env node + +import { ClusterPluginPackageManagementClientRemoteError } from '../management-support/pluginPackageManagementClient'; +import { executeClusterRunManagementClient } from './runManagementClient'; + +const USAGE = + 'Usage: ql3-run-client --config=/absolute/client.json --command=/absolute/command.json --assertion=/absolute/assertion.jwt'; + +function argumentsFrom(argv: readonly string[]): Readonly<{ + configFile: string; + commandFile: string; + assertionFile: string; +}> | null { + if (argv.length !== 3) return null; + const values = new Map(); + for (const argument of argv) { + const match = /^--(config|command|assertion)=(\/.+)$/.exec(argument); + if (!match || values.has(match[1]!)) return null; + values.set(match[1]!, match[2]!); + } + if (!values.has('config') || !values.has('command') || !values.has('assertion')) return null; + return Object.freeze({ + configFile: values.get('config')!, + commandFile: values.get('command')!, + assertionFile: values.get('assertion')!, + }); +} + +function failureFact(error: unknown): Readonly> { + const candidate = error as { readonly code?: unknown }; + return Object.freeze({ + schemaVersion: 1, + component: 'qinglong3-run-management-client', + event: 'command_failed', + code: typeof candidate?.code === 'string' ? candidate.code : 'QL3_RUN_MANAGEMENT_CLIENT_FAILED', + ...(error instanceof ClusterPluginPackageManagementClientRemoteError + ? { + statusCode: error.statusCode, + responseCode: error.responseCode, + requestId: error.requestId, + ...(error.retryAfterSeconds === null ? {} : { retryAfterSeconds: error.retryAfterSeconds }), + } + : {}), + }); +} + +async function run(argv: readonly string[]): Promise { + if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) { + process.stdout.write(`${USAGE}\n`); + return; + } + const paths = argumentsFrom(argv); + if (!paths) { + process.stderr.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'usage_invalid', code: 'QL3_RUN_MANAGEMENT_CLIENT_USAGE_INVALID' })}\n`); + process.exitCode = 64; + return; + } + try { + const result = await executeClusterRunManagementClient(paths); + process.stdout.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'command_completed', requestId: result.requestId, result: result.result })}\n`); + } catch (error) { + process.stderr.write(`${JSON.stringify(failureFact(error))}\n`); + process.exitCode = 1; + } +} + +void run(process.argv.slice(2)); diff --git a/packages/ql3-cluster-admin/src/run-management/runManagementHttp.ts b/packages/ql3-cluster-admin/src/run-management/runManagementHttp.ts new file mode 100644 index 00000000..c64885e8 --- /dev/null +++ b/packages/ql3-cluster-admin/src/run-management/runManagementHttp.ts @@ -0,0 +1,24 @@ +import { + CLUSTER_RUN_MANAGEMENT_PATH, + startClusterPluginPackageManagementHttp, + type ClusterPluginPackageManagementHttpApplication, + type StartClusterPluginPackageManagementHttpOptions, +} from '../management-support/pluginPackageManagementHttp'; + +export type ClusterRunManagementHttpApplication = + ClusterPluginPackageManagementHttpApplication; + +export type StartClusterRunManagementHttpOptions = Omit< + StartClusterPluginPackageManagementHttpOptions, + 'managementPath' +>; + +/** Starts the shared bounded OIDC/mTLS HTTPS adapter on the Run-only path. */ +export function startClusterRunManagementHttp( + options: StartClusterRunManagementHttpOptions, +): Promise> { + return startClusterPluginPackageManagementHttp({ + ...options, + managementPath: CLUSTER_RUN_MANAGEMENT_PATH, + }); +} diff --git a/packages/ql3-cluster-admin/src/run-management/runManagementProcess.ts b/packages/ql3-cluster-admin/src/run-management/runManagementProcess.ts new file mode 100644 index 00000000..d606fb1a --- /dev/null +++ b/packages/ql3-cluster-admin/src/run-management/runManagementProcess.ts @@ -0,0 +1,467 @@ +import type { + OpenPostgresDatabase, + PostgresDatabaseResource, +} from '@qinglong/runtime-core'; +import { + PostgresRunManagementIdentityKeysetLedgerRepository, + assertPostgresRunManagerSchemaReady, + createPostgresDatabaseOpener, + isPostgresTlsDnsServername, + loadPostgresCertificateAuthorityFile, + loadPostgresConnectionEnvironment, + type PostgresConnectionOptions, + type PostgresPoolOptions, + type PostgresSchemaReadinessReport, +} from '@qinglong/cluster-postgres/run-manager'; +import { + absoluteManagementEnvironmentFile, + booleanManagementEnvironmentValue, + boundedManagementEnvironmentValue, + integerManagementEnvironmentValue, + readManagementTlsFile, +} from '../management-support/managementProcessSupport'; +import { + createClusterRunIdentityKeysetFile, + type ClusterPluginPackageIdentityKeysetFile, + type ClusterPluginPackageIdentityKeysetSnapshot, +} from '../management-support/pluginPackageIdentityKeyset'; +import { validateClusterManagementClientTrust } from '../worker-credential/management-server/workerCredentialManagementMutualTls'; +import { createClusterRunManagementService } from './runManagement'; +import { + startClusterRunManagementHttp, + type ClusterRunManagementHttpApplication, + type StartClusterRunManagementHttpOptions, +} from './runManagementHttp'; +import { createClusterRunManagementTransport } from './runManagementTransport'; + +const SAFE_HOST = /^[A-Za-z0-9][A-Za-z0-9.:-]{0,254}$/; +const SAFE_APPLICATION_NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,62}$/; + +export type ClusterRunManagementProcessEnvironment = Readonly< + Record +>; + +export type ClusterRunManagementProcessConfig = + | Readonly<{ enabled: false }> + | Readonly<{ + enabled: true; + profile: 'cluster-admin'; + host: string; + port: number; + certificateFile: string; + privateKeyFile: string; + clientCertificateAuthorityFile: string; + clientCertificateRevocationListFile: string; + identityKeysetFile: string; + http: Readonly<{ + maxBodyBytes: number; + maxConnections: number; + maxConcurrentRequests: number; + requestTimeoutMs: number; + drainTimeoutMs: number; + rateWindowMs: number; + peerRequestLimit: number; + globalRequestLimit: number; + maxRateLimitPeers: number; + }>; + database: Readonly<{ + connection: PostgresConnectionOptions; + pool: PostgresPoolOptions; + }>; + }>; + +export type ClusterRunManagementProcessRuntime = + | Readonly<{ status: 'disabled'; close(): Promise }> + | Readonly<{ + status: 'active'; + address: Readonly<{ host: string; port: number }>; + database: PostgresSchemaReadinessReport; + identity: ClusterPluginPackageIdentityKeysetSnapshot; + availabilityStatus(): 'ready' | 'unavailable' | 'stopped'; + close(): Promise; + }>; + +export interface StartClusterRunManagementProcessOptions { + readonly environment: ClusterRunManagementProcessEnvironment; + readonly openDatabase?: OpenPostgresDatabase; + readonly identities?: ClusterPluginPackageIdentityKeysetFile; + readonly assertReady?: ( + pool: PostgresDatabaseResource['pool'], + ) => Promise; + readonly startHttp?: ( + options: StartClusterRunManagementHttpOptions, + ) => Promise>; + readonly now?: () => number; + readonly randomUuid?: () => string; + readonly onError?: (error: unknown) => void; +} + +export class ClusterRunManagementProcessConfigError extends TypeError { + readonly code = 'QL3_RUN_MANAGEMENT_PROCESS_CONFIG_INVALID'; + constructor(message: string) { + super(`Run management process configuration is invalid: ${message}`); + this.name = 'ClusterRunManagementProcessConfigError'; + } +} + +function failure(message: string): ClusterRunManagementProcessConfigError { + return new ClusterRunManagementProcessConfigError(message); +} + +function bounded( + environment: ClusterRunManagementProcessEnvironment, + name: string, + maximumLength: number, + required = false, +): string | undefined { + return boundedManagementEnvironmentValue( + environment, + name, + maximumLength, + failure, + required, + ); +} + +function bool( + environment: ClusterRunManagementProcessEnvironment, + name: string, +): boolean { + return booleanManagementEnvironmentValue(environment, name, failure); +} + +function integer( + environment: ClusterRunManagementProcessEnvironment, + name: string, + fallback: number, + minimum: number, + maximum: number, +): number { + return integerManagementEnvironmentValue( + environment, + name, + fallback, + minimum, + maximum, + failure, + ); +} + +function absolute( + environment: ClusterRunManagementProcessEnvironment, + name: string, +): string { + return absoluteManagementEnvironmentFile(environment, name, failure); +} + +function loadDatabase( + environment: ClusterRunManagementProcessEnvironment, +): Readonly<{ + connection: PostgresConnectionOptions; + pool: PostgresPoolOptions; +}> { + let connection: PostgresConnectionOptions; + try { + connection = loadPostgresConnectionEnvironment(environment, { + connectionString: 'QL3_POSTGRES_RUN_MANAGER_URL', + host: 'QL3_POSTGRES_RUN_MANAGER_HOST', + port: 'QL3_POSTGRES_RUN_MANAGER_PORT', + database: 'QL3_POSTGRES_RUN_MANAGER_DATABASE', + user: 'QL3_POSTGRES_RUN_MANAGER_USER', + password: 'QL3_POSTGRES_RUN_MANAGER_PASSWORD', + }); + } catch (error) { + throw failure( + error instanceof Error + ? error.message + : 'PostgreSQL run manager connection is invalid', + ); + } + const mode = environment.QL3_POSTGRES_RUN_MANAGER_TLS_MODE ?? 'verify-full'; + if (mode !== 'verify-full' && mode !== 'disable') { + throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_MODE must be verify-full or disable'); + } + if ( + mode === 'disable' && + !bool(environment, 'QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE') + ) { + throw failure( + 'disabling run manager PostgreSQL TLS requires QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE=true', + ); + } + const servername = bounded( + environment, + 'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME', + 253, + ); + if (mode === 'verify-full' && !isPostgresTlsDnsServername(servername)) { + throw failure( + 'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME must be an explicit DNS name', + ); + } + const caFile = bounded( + environment, + 'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE', + 4_096, + ); + if (mode === 'disable' && caFile !== undefined) { + throw failure( + 'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE cannot be used when TLS is disabled', + ); + } + let ca: string | undefined; + if (caFile !== undefined) { + try { + ca = loadPostgresCertificateAuthorityFile(caFile); + } catch { + throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE is invalid'); + } + } + const applicationName = + bounded(environment, 'QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME', 63) ?? + 'qinglong3-run-manager'; + if (!SAFE_APPLICATION_NAME.test(applicationName)) { + throw failure('QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME is invalid'); + } + return Object.freeze({ + connection: Object.freeze({ + ...connection, + tls: + mode === 'disable' + ? Object.freeze({ mode: 'disable' as const }) + : Object.freeze({ + mode: 'verify-full' as const, + servername: servername!, + ...(ca === undefined ? {} : { ca }), + }), + }), + pool: Object.freeze({ + applicationName, + maxConnections: integer( + environment, + 'QL3_POSTGRES_RUN_MANAGER_POOL_MAX', + 2, + 1, + 4, + ), + idleTimeoutMs: integer( + environment, + 'QL3_POSTGRES_RUN_MANAGER_IDLE_TIMEOUT_MS', + 10_000, + 1_000, + 60_000, + ), + connectionTimeoutMs: integer( + environment, + 'QL3_POSTGRES_RUN_MANAGER_CONNECTION_TIMEOUT_MS', + 5_000, + 100, + 60_000, + ), + }), + }); +} + +export function loadClusterRunManagementProcessConfig( + environment: ClusterRunManagementProcessEnvironment, +): Readonly { + if (!environment || typeof environment !== 'object') { + throw failure('environment is invalid'); + } + if (!bool(environment, 'QL3_RUN_MANAGEMENT_ENABLED')) { + return Object.freeze({ enabled: false as const }); + } + if (environment.QL3_PROFILE !== 'cluster-admin') { + throw failure('QL3_PROFILE must be cluster-admin when Run management is enabled'); + } + const host = bounded(environment, 'QL3_RUN_MANAGEMENT_HOST', 255) ?? '0.0.0.0'; + if (!SAFE_HOST.test(host)) throw failure('QL3_RUN_MANAGEMENT_HOST is invalid'); + const http = Object.freeze({ + maxBodyBytes: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_BODY_BYTES', 32 * 1024, 1_024, 256 * 1024), + maxConnections: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONNECTIONS', 32, 1, 512), + maxConcurrentRequests: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS', 16, 1, 256), + requestTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS', 10_000, 1_000, 60_000), + drainTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS', 5_000, 100, 60_000), + rateWindowMs: integer(environment, 'QL3_RUN_MANAGEMENT_RATE_WINDOW_MS', 60_000, 1_000, 5 * 60_000), + peerRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT', 30, 1, 10_000), + globalRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT', 300, 1, 100_000), + maxRateLimitPeers: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS', 1_024, 1, 16_384), + }); + if (http.globalRequestLimit < http.peerRequestLimit) { + throw failure('global request limit cannot be below peer request limit'); + } + return Object.freeze({ + enabled: true as const, + profile: 'cluster-admin' as const, + host, + port: integer(environment, 'QL3_RUN_MANAGEMENT_PORT', 8_448, 1, 65_535), + certificateFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_CERT_FILE'), + privateKeyFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_KEY_FILE'), + clientCertificateAuthorityFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CA_FILE'), + clientCertificateRevocationListFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE'), + identityKeysetFile: absolute(environment, 'QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE'), + http, + database: loadDatabase(environment), + }); +} + +export async function startClusterRunManagementProcess( + options: StartClusterRunManagementProcessOptions, +): Promise> { + if ( + !options || + typeof options !== 'object' || + Array.isArray(options) || + Object.keys(options).some( + (key) => + ![ + 'environment', + 'openDatabase', + 'identities', + 'assertReady', + 'startHttp', + 'now', + 'randomUuid', + 'onError', + ].includes(key), + ) || + !options.environment || + typeof options.environment !== 'object' || + (options.openDatabase !== undefined && typeof options.openDatabase !== 'function') || + (options.identities !== undefined && + (typeof options.identities.reload !== 'function' || + typeof options.identities.bind !== 'function')) || + (options.assertReady !== undefined && typeof options.assertReady !== 'function') || + (options.startHttp !== undefined && typeof options.startHttp !== 'function') || + (options.now !== undefined && typeof options.now !== 'function') || + (options.randomUuid !== undefined && typeof options.randomUuid !== 'function') || + (options.onError !== undefined && typeof options.onError !== 'function') + ) { + throw failure('options are invalid'); + } + const config = loadClusterRunManagementProcessConfig(options.environment); + if (!config.enabled) { + return Object.freeze({ status: 'disabled' as const, close: () => Promise.resolve() }); + } + const now = options.now ?? Date.now; + let http: Readonly | undefined; + let database: PostgresDatabaseResource | undefined; + let unavailableError: unknown; + let closePromise: Promise | undefined; + const report = (error: unknown): void => { + try { + options.onError?.(error); + } catch { + // Diagnostics never own availability or cleanup. + } + }; + const openDatabase = + options.openDatabase ?? + createPostgresDatabaseOpener({ + role: 'run-manager', + connection: config.database.connection, + pool: config.database.pool, + onPoolError(error) { + const first = unavailableError === undefined; + unavailableError ??= error; + http?.withdraw(error); + if (first) report(error); + }, + }); + try { + database = await openDatabase(); + const evidence = await (options.assertReady ?? assertPostgresRunManagerSchemaReady)(database.pool); + if (unavailableError !== undefined) throw unavailableError; + const identities = + options.identities ?? + createClusterRunIdentityKeysetFile({ + filePath: config.identityKeysetFile, + now, + ledger: new PostgresRunManagementIdentityKeysetLedgerRepository( + database.pool, + 'run-management', + ), + }); + const identity = await identities.reload(); + const service = createClusterRunManagementService({ + pool: database.pool, + now, + ...(options.randomUuid === undefined ? {} : { randomUuid: options.randomUuid }), + }); + const transport = createClusterRunManagementTransport({ service, now }); + const privateKey = readManagementTlsFile(config.privateKeyFile, true, failure); + try { + const certificate = readManagementTlsFile(config.certificateFile, false, failure); + const clientCertificateAuthority = readManagementTlsFile( + config.clientCertificateAuthorityFile, + false, + failure, + ); + const clientCertificateRevocationList = readManagementTlsFile( + config.clientCertificateRevocationListFile, + false, + failure, + ); + validateClusterManagementClientTrust( + clientCertificateAuthority, + clientCertificateRevocationList, + now(), + failure, + ); + http = await (options.startHttp ?? startClusterRunManagementHttp)({ + host: config.host, + port: config.port, + tls: { + privateKey, + certificate, + clientCertificateAuthority, + clientCertificateRevocationList, + }, + transport, + identities, + limits: config.http, + now, + onError: report, + }); + } finally { + privateKey.fill(0); + } + if (unavailableError !== undefined) http.withdraw(unavailableError); + return Object.freeze({ + status: 'active' as const, + address: http.address, + database: evidence, + identity, + availabilityStatus: () => http?.availabilityStatus() ?? 'stopped', + close(): Promise { + closePromise ??= (async () => { + let primaryError: unknown; + try { + await http?.close(); + } catch (error) { + primaryError = error; + } + try { + await database?.close(); + } catch (error) { + primaryError ??= error; + } + if (primaryError) throw primaryError; + })(); + return closePromise; + }, + }); + } catch (error) { + try { + await http?.close(); + } catch { + // Preserve startup failure. + } + try { + await database?.close(); + } catch { + // Preserve startup failure. + } + throw error; + } +} diff --git a/packages/ql3-cluster-admin/src/run-management/runManagementTransport.ts b/packages/ql3-cluster-admin/src/run-management/runManagementTransport.ts new file mode 100644 index 00000000..68634180 --- /dev/null +++ b/packages/ql3-cluster-admin/src/run-management/runManagementTransport.ts @@ -0,0 +1,214 @@ +import { + createRunManualRetryResponseBody, + parseRunManualRetryRequestBody, + type RunManualRetryResponseBody, +} from '@qinglong/runtime-core/run-manual-retry'; +import { + normalizeSecurityPrincipal, + type SecurityPrincipal, +} from '@qinglong/runtime-core/security'; +import type { ClusterRunManagementService } from './runManagement'; + +const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const UUID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; +const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']); + +export type ClusterRunManagementCommand = Readonly<{ + schemaVersion: 1; + operation: 'run.retry'; + request: Readonly<{ + projectId: string; + sourceRunId: string; + requestId: string; + auditEventId: string; + failureAuditEventId: string; + body: Readonly<{ + schema: 'qinglong/run-manual-retry@v1'; + mutationId: string; + expectedRunVersion: number; + expectedRunStatus: 'failed' | 'cancelled' | 'timed_out'; + }>; + }>; +}>; + +export type ClusterRunManagementTransportResult = Readonly<{ + schemaVersion: 1; + operation: 'run.retry'; + retry: Readonly; +}>; + +export interface ClusterRunManagementAuthentication { + authenticate(): Promise | null>; +} + +export interface ClusterRunManagementTransport { + execute( + command: unknown, + authentication: ClusterRunManagementAuthentication, + ): Promise>; +} + +export class ClusterRunManagementTransportConfigurationError extends TypeError { + readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_CONFIGURATION_INVALID'; + constructor() { + super('Cluster Run management transport configuration is invalid'); + this.name = 'ClusterRunManagementTransportConfigurationError'; + } +} + +export class ClusterRunManagementTransportRequestError extends TypeError { + readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_REQUEST_INVALID'; + constructor() { + super('Cluster Run management transport request is invalid'); + this.name = 'ClusterRunManagementTransportRequestError'; + } +} + +export class ClusterRunManagementTransportAuthenticationError extends Error { + readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_AUTHENTICATION_REQUIRED'; + constructor() { + super('Cluster Run management transport requires a strong User principal'); + this.name = 'ClusterRunManagementTransportAuthenticationError'; + } +} + +export class ClusterRunManagementTransportUnavailableError extends Error { + readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_UNAVAILABLE'; + constructor() { + super('Cluster Run management transport is unavailable'); + this.name = 'ClusterRunManagementTransportUnavailableError'; + } +} + +function invalid(): never { + throw new ClusterRunManagementTransportRequestError(); +} + +function exact(value: unknown, keys: readonly string[]): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) invalid(); + const actual = Object.keys(value as object).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + invalid(); + } + return value as Record; +} + +function identifier(value: unknown): string { + if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) invalid(); + return value; +} + +function uuid(value: unknown): string { + if (typeof value !== 'string' || !UUID_PATTERN.test(value)) invalid(); + return value; +} + +export function normalizeClusterRunManagementCommand( + value: unknown, +): Readonly { + const envelope = exact(value, ['schemaVersion', 'operation', 'request']); + if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid(); + const request = exact(envelope.request, [ + 'projectId', + 'sourceRunId', + 'requestId', + 'auditEventId', + 'failureAuditEventId', + 'body', + ]); + let body: ReturnType; + try { + body = parseRunManualRetryRequestBody(request.body); + } catch { + invalid(); + } + const auditEventId = uuid(request.auditEventId); + const failureAuditEventId = uuid(request.failureAuditEventId); + if (auditEventId === failureAuditEventId) invalid(); + return Object.freeze({ + schemaVersion: 1, + operation: 'run.retry', + request: Object.freeze({ + projectId: identifier(request.projectId), + sourceRunId: identifier(request.sourceRunId), + requestId: identifier(request.requestId), + auditEventId, + failureAuditEventId, + body, + }), + }); +} + +export function createClusterRunManagementTransport(options: Readonly<{ + service: ClusterRunManagementService; + now?: () => number; +}>): Readonly { + if ( + !options || + typeof options !== 'object' || + Array.isArray(options) || + Object.keys(options).some((key) => key !== 'service' && key !== 'now') || + !options.service || + typeof options.service.retry !== 'function' || + (options.now !== undefined && typeof options.now !== 'function') + ) { + throw new ClusterRunManagementTransportConfigurationError(); + } + const now = options.now ?? Date.now; + return Object.freeze({ + async execute( + commandValue: unknown, + authentication: ClusterRunManagementAuthentication, + ) { + const command = normalizeClusterRunManagementCommand(commandValue); + if ( + !authentication || + typeof authentication !== 'object' || + Array.isArray(authentication) || + Object.keys(authentication).length !== 1 || + typeof authentication.authenticate !== 'function' + ) { + throw new ClusterRunManagementTransportConfigurationError(); + } + let candidate: Readonly | null; + try { + candidate = await authentication.authenticate(); + } catch { + throw new ClusterRunManagementTransportUnavailableError(); + } + let principal: Readonly; + try { + principal = normalizeSecurityPrincipal(candidate as SecurityPrincipal, now()); + } catch { + throw new ClusterRunManagementTransportAuthenticationError(); + } + if ( + principal.subject.type !== 'user' || + !STRONG_ASSURANCES.has(principal.assurance) + ) { + throw new ClusterRunManagementTransportAuthenticationError(); + } + const result = await options.service.retry({ + projectId: command.request.projectId, + sourceRunId: command.request.sourceRunId, + mutationId: command.request.body.mutationId, + expectedRunVersion: command.request.body.expectedRunVersion, + expectedRunStatus: command.request.body.expectedRunStatus, + requestId: command.request.requestId, + auditEventId: command.request.auditEventId, + failureAuditEventId: command.request.failureAuditEventId, + principal, + }); + return Object.freeze({ + schemaVersion: 1, + operation: 'run.retry', + retry: createRunManualRetryResponseBody(result), + }); + }, + }); +} diff --git a/packages/ql3-cluster-admin/test/bootstrap.test.cjs b/packages/ql3-cluster-admin/test/bootstrap.test.cjs index e2215410..6f68d350 100644 --- a/packages/ql3-cluster-admin/test/bootstrap.test.cjs +++ b/packages/ql3-cluster-admin/test/bootstrap.test.cjs @@ -163,6 +163,7 @@ function database(serverVersionNum = '160014') { 'enforce_plugin_package_stage_provenance', 'lock_active_plugin_package_project', 'lock_approval_policy_fence', + 'lock_run_management_policy_fence', 'plugin_package_lifecycle_blocking_runs', 'plugin_package_automation_start_allowed', 'plugin_package_run_start_allowed', diff --git a/packages/ql3-cluster-admin/test/pluginPackageIdentityKeyset.test.cjs b/packages/ql3-cluster-admin/test/pluginPackageIdentityKeyset.test.cjs index fa1bf0d9..e5e3f300 100644 --- a/packages/ql3-cluster-admin/test/pluginPackageIdentityKeyset.test.cjs +++ b/packages/ql3-cluster-admin/test/pluginPackageIdentityKeyset.test.cjs @@ -12,6 +12,7 @@ const { createClusterAutomationIdentityKeysetFile, createClusterApprovalIdentityKeysetFile, createClusterModelProviderCredentialIdentityKeysetFile, + createClusterRunIdentityKeysetFile, } = require('@qinglong/cluster-admin/plugin-package-identity-keyset'); const NOW_MS = 1_700_000_000_000; @@ -216,6 +217,38 @@ function providerCredentialAssertion(key, overrides = {}) { ).toString('base64url')}`; } +function runAssertion(key, overrides = {}) { + const header = Buffer.from( + JSON.stringify({ + alg: 'EdDSA', + kid: key.kid, + typ: 'ql3-run-management+jwt', + }), + ).toString('base64url'); + const now = Math.floor(NOW_MS / 1000); + const payload = Buffer.from( + JSON.stringify({ + acr: 'urn:ql3:mfa', + amr: ['pwd', 'otp'], + aud: 'qinglong3-run-management', + auth_time: now - 10, + exp: now + 120, + iat: now, + iss: ISSUER, + jti: `run-assertion-${key.kid}`, + ql3_purpose: 'run-management', + sub: 'run-operator-1', + ...overrides, + }), + ).toString('base64url'); + const signed = `${header}.${payload}`; + return `${signed}.${sign( + null, + Buffer.from(signed, 'ascii'), + key.privateKey, + ).toString('base64url')}`; +} + async function atomicWrite(filePath, document) { const nextPath = `${filePath}.next`; await writeFile(nextPath, `${JSON.stringify(document)}\n`, { mode: 0o644 }); @@ -374,6 +407,34 @@ test('loads a provider credential keyset isolated by type, purpose and audience' }); }); +test('loads a Run keyset isolated from every other management purpose', async () => { + await fixture(async ({ filePath }) => { + const key = reviewedKey('run-identity-key-1'); + await atomicWrite(filePath, { + ...keyset(1, [key]), + audience: 'qinglong3-run-management', + }); + const provider = createClusterRunIdentityKeysetFile({ + filePath, + now: () => NOW_MS, + }); + const principal = await provider.bind(runAssertion(key)).authenticate(); + assert.deepEqual(principal.subject, { + type: 'user', + id: 'run-operator-1', + }); + await assert.rejects(provider.bind(approvalAssertion(key)).authenticate(), { + code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID', + }); + await assert.rejects( + provider + .bind(runAssertion(key, { ql3_purpose: 'approval-management' })) + .authenticate(), + { code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' }, + ); + }); +}); + test('supports overlap rotation then immediately revokes the previous key', async () => { await fixture(async ({ filePath }) => { const first = reviewedKey('issuer-key-1'); diff --git a/packages/ql3-cluster-admin/test/pluginPackageRecovery.test.cjs b/packages/ql3-cluster-admin/test/pluginPackageRecovery.test.cjs index f34a6247..27ccf501 100644 --- a/packages/ql3-cluster-admin/test/pluginPackageRecovery.test.cjs +++ b/packages/ql3-cluster-admin/test/pluginPackageRecovery.test.cjs @@ -197,6 +197,7 @@ function database(serverVersionNum = '160014') { 'plugin_package_tool_start_allowed', 'plugin_package_workflow_admission_snapshot', 'plugin_package_workflow_task_attempt_snapshot', + 'lock_run_management_policy_fence', ].includes(functionName), isOwner: false, })), diff --git a/packages/ql3-cluster-admin/test/runManagement.test.cjs b/packages/ql3-cluster-admin/test/runManagement.test.cjs new file mode 100644 index 00000000..8072fb15 --- /dev/null +++ b/packages/ql3-cluster-admin/test/runManagement.test.cjs @@ -0,0 +1,164 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + ClusterRunManagementAuthorizationError, + createClusterRunManagementService, +} = require('@qinglong/cluster-admin/run-management'); + +const NOW = 1_000_000; +const SOURCE_DIGEST = 'a'.repeat(64); +const EXECUTION_DIGEST = 'b'.repeat(64); +const TASK_REVISION = `qltd:v1:7:${SOURCE_DIGEST}`; +const GENERATED = [ + '019f9500-0000-4000-8000-000000000010', + '019f9500-0000-4000-8000-000000000011', + '019f9500-0000-4000-8000-000000000012', + '019f9500-0000-4000-8000-000000000013', +]; + +function request() { + return { + projectId: 'project-1', + sourceRunId: 'source-run-1', + mutationId: '019f9500-0000-4000-8000-000000000001', + expectedRunVersion: 7, + expectedRunStatus: 'failed', + requestId: 'request-1', + auditEventId: '019f9500-0000-4000-8000-000000000002', + failureAuditEventId: '019f9500-0000-4000-8000-000000000003', + principal: { + subject: { type: 'user', id: 'operator-1' }, + authenticationId: 'oidc:run-management-1', + authenticatedAtMs: 999_000, + expiresAtMs: 1_100_000, + assurance: 'multi_factor', + }, + }; +} + +function policyRow(role = 'operator') { + return { + projectId: 'project-1', + projectName: 'Project 1', + projectSlug: 'project-1', + projectStatus: 'active', + projectVersion: 2, + projectCreatedAtMs: '1', + projectUpdatedAtMs: '2', + bindingProjectId: 'project-1', + bindingSubjectType: 'user', + bindingSubjectId: 'operator-1', + bindingVersion: 3, + bindingState: 'active', + bindingRole: role, + bindingMutationId: 'binding-3', + bindingChangedByType: 'user', + bindingChangedById: 'owner-1', + bindingCreatedAtMs: '3', + }; +} + +function fixture(role = 'operator') { + const calls = []; + const pool = { + async query(sql, params = []) { + const text = sql.replace(/\s+/g, ' ').trim(); + calls.push({ scope: 'pool', sql: text, params }); + if (text.includes('LEFT JOIN LATERAL')) return { rows: [policyRow(role)] }; + if (text.startsWith('INSERT INTO "ql3"."security_audit_events"')) { + return { rows: [], rowCount: 1 }; + } + throw new Error(`unexpected pool query: ${text}`); + }, + async connect() { + return { + async query(sql, params = []) { + const text = sql.replace(/\s+/g, ' ').trim(); + calls.push({ scope: 'client', sql: text, params }); + if ( + text === 'BEGIN ISOLATION LEVEL SERIALIZABLE' || + text === 'COMMIT' || + text === 'ROLLBACK' || + text.startsWith('SELECT set_config') + ) return { rows: [], rowCount: 0 }; + if (text.includes('statement_timestamp()')) { + return { rows: [{ nowMs: NOW }], rowCount: 1 }; + } + if (text.includes('lock_run_management_policy_fence')) { + return { rows: [{ matches: true }], rowCount: 1 }; + } + if (text.includes('idempotency_key = $2')) return { rows: [] }; + if (text.includes('WHERE run.id = $1')) { + return { + rows: [{ + projectId: 'project-1', + taskId: 'task-1', + taskRevision: TASK_REVISION, + taskName: 'Task 1', + taskSnapshotRef: TASK_REVISION, + parentRunId: null, + triggerType: 'task_start', + executionOwner: 'runtime', + inputRef: null, + priority: 1, + runStatus: 'failed', + runVersion: 7, + attemptExecutorType: 'remote_worker', + }], + }; + } + if (text.includes('FROM "ql3"."task_definitions"')) { + return { rows: [{ enabled: true }] }; + } + if (text.includes('task_execution_revisions')) { + return { rows: [{ sourceContentDigest: SOURCE_DIGEST, contentDigest: EXECUTION_DIGEST }] }; + } + if (text.startsWith('SELECT') && text.includes("trigger_type = 'run_manual_retry'")) { + return { rows: [] }; + } + if (text.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 }; + throw new Error(`unexpected client query: ${text}`); + }, + release() {}, + }; + }, + }; + let index = 0; + return { + calls, + service: createClusterRunManagementService({ + pool, + now: () => NOW, + randomUuid: () => GENERATED[index++], + }), + }; +} + +test('authorizes run.retry and keeps all generated aggregate identities server-side', async () => { + const { calls, service } = fixture(); + const result = await service.retry(request()); + assert.equal(result.status, 'accepted'); + assert.equal(result.runId, GENERATED[0]); + assert.equal(result.attemptId, GENERATED[1]); + const runInsert = calls.find(({ sql }) => sql.startsWith('INSERT INTO "ql3"."runs"')); + assert.equal(runInsert.params[0], GENERATED[0]); + assert.equal(runInsert.params.includes(GENERATED[2]), false); + assert.equal( + calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')), + true, + ); +}); + +test('denied policy writes only the caller-supplied failure audit', async () => { + const { calls, service } = fixture('viewer'); + await assert.rejects(service.retry(request()), ClusterRunManagementAuthorizationError); + const audits = calls.filter(({ sql }) => + sql.startsWith('INSERT INTO "ql3"."security_audit_events"'), + ); + assert.equal(audits.length, 1); + assert.equal(audits[0].params[0], request().failureAuditEventId); + assert.equal(calls.some(({ scope }) => scope === 'client'), false); +}); diff --git a/packages/ql3-cluster-admin/test/runManagementClient.test.cjs b/packages/ql3-cluster-admin/test/runManagementClient.test.cjs new file mode 100644 index 00000000..5c9efb75 --- /dev/null +++ b/packages/ql3-cluster-admin/test/runManagementClient.test.cjs @@ -0,0 +1,77 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + validateClusterRunManagementClientResult, +} = require('@qinglong/cluster-admin/run-management-client'); +const { + normalizeClusterRunManagementCommand, +} = require('@qinglong/cluster-admin/run-management-transport'); +const { + ClusterPluginPackageManagementClientRequestError, +} = require('@qinglong/cluster-admin/plugin-package-management-client'); + +const command = normalizeClusterRunManagementCommand({ + schemaVersion: 1, + operation: 'run.retry', + request: { + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'request-1', + auditEventId: '019f9400-0000-4000-8000-000000000001', + failureAuditEventId: '019f9400-0000-4000-8000-000000000002', + body: { + schema: 'qinglong/run-manual-retry@v1', + mutationId: '019f9400-0000-4000-8000-000000000003', + expectedRunVersion: 7, + expectedRunStatus: 'failed', + }, + }, +}); + +function response(overrides = {}) { + return { + schemaVersion: 1, + operation: 'run.retry', + retry: { + schema: 'qinglong/run-manual-retry@v1', + status: 'accepted', + projectId: 'project-1', + sourceRunId: 'source-run-1', + sourceRunStatus: 'failed', + sourceRunVersion: 7, + runId: '019f9400-0000-4000-8000-000000000010', + retryOfRunId: 'source-run-1', + taskId: 'task-1', + taskRevision: `qltd:v1:1:${'a'.repeat(64)}`, + attemptId: '019f9400-0000-4000-8000-000000000011', + runStatus: 'queued', + runVersion: 2, + eventSequence: 2, + executorType: 'remote_worker', + executionRevisionDigest: 'b'.repeat(64), + createdAtMs: 1_000_000, + ...overrides, + }, + }; +} + +test('validates one low-sensitive retry response against the request fence', () => { + assert.deepEqual(validateClusterRunManagementClientResult(response(), command), response()); +}); + +test('rejects response target, execution placement and shape drift', () => { + for (const candidate of [ + response({ projectId: 'project-2' }), + response({ executorType: 'local_process' }), + response({ sourceRunVersion: 8 }), + { ...response(), principal: { type: 'user', id: 'operator-1' } }, + ]) { + assert.throws( + () => validateClusterRunManagementClientResult(candidate, command), + ClusterPluginPackageManagementClientRequestError, + ); + } +}); diff --git a/packages/ql3-cluster-admin/test/runManagementHttp.test.cjs b/packages/ql3-cluster-admin/test/runManagementHttp.test.cjs new file mode 100644 index 00000000..d3bf848e --- /dev/null +++ b/packages/ql3-cluster-admin/test/runManagementHttp.test.cjs @@ -0,0 +1,108 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { request: httpsRequest } = require('node:https'); +const { resolve } = require('node:path'); +const { test } = require('node:test'); + +const { + ClusterRunManagementRateLimitedError, +} = require('@qinglong/cluster-admin/run-management'); +const { + startClusterRunManagementHttp, +} = require('@qinglong/cluster-admin/run-management-http'); + +const SERVER_KEY = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-key.pem'); +const SERVER_CERT = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem'); +const PATH = '/api/v3/runs/management'; + +function post(port, path = PATH) { + const body = Buffer.from(JSON.stringify({ + schemaVersion: 1, + operation: 'run.retry', + request: { + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'request-1', + auditEventId: '019f9600-0000-4000-8000-000000000001', + failureAuditEventId: '019f9600-0000-4000-8000-000000000002', + body: { + schema: 'qinglong/run-manual-retry@v1', + mutationId: '019f9600-0000-4000-8000-000000000003', + expectedRunVersion: 7, + expectedRunStatus: 'failed', + }, + }, + })); + return new Promise((resolvePromise, reject) => { + const outgoing = httpsRequest({ + host: '127.0.0.1', + port, + path, + method: 'POST', + rejectUnauthorized: false, + agent: false, + headers: { + authorization: 'Bearer assertion', + 'content-type': 'application/json', + 'content-length': String(body.length), + }, + }, (incoming) => { + const chunks = []; + incoming.on('data', (chunk) => chunks.push(chunk)); + incoming.once('end', () => { + const bytes = Buffer.concat(chunks); + resolvePromise({ + statusCode: incoming.statusCode, + headers: incoming.headers, + body: bytes.length ? JSON.parse(bytes.toString('utf8')) : null, + }); + }); + }); + outgoing.once('error', reject); + outgoing.end(body); + }); +} + +test('serves only the Run path and maps durable quota to bounded HTTP facts', async () => { + const application = await startClusterRunManagementHttp({ + host: '127.0.0.1', + port: 0, + tls: { + privateKey: Buffer.from(readFileSync(SERVER_KEY)), + certificate: Buffer.from(readFileSync(SERVER_CERT)), + }, + identities: { + async reload() { throw new Error('not used'); }, + bind() { + return { authenticate: async () => ({ + subject: { type: 'user', id: 'operator-1' }, + authenticationId: 'oidc:run-management-1', + authenticatedAtMs: 900, + expiresAtMs: 10_000, + assurance: 'hardware', + }) }; + }, + }, + transport: { + async execute(_command, authentication) { + await authentication.authenticate(); + throw new ClusterRunManagementRateLimitedError(1_500); + }, + }, + now: () => 1_000, + }); + try { + const limited = await post(application.address.port); + assert.equal(limited.statusCode, 429); + assert.equal(limited.headers['retry-after'], '2'); + assert.equal(limited.body.error.code, 'rate_limited'); + assert.match(limited.body.requestId, /^[0-9a-f-]{36}$/); + const absent = await post(application.address.port, '/api/v3/approvals/management'); + assert.equal(absent.statusCode, 404); + assert.equal(absent.body.error.code, 'not_found'); + } finally { + await application.close(); + } +}); diff --git a/packages/ql3-cluster-admin/test/runManagementProcess.test.cjs b/packages/ql3-cluster-admin/test/runManagementProcess.test.cjs new file mode 100644 index 00000000..e6b87e15 --- /dev/null +++ b/packages/ql3-cluster-admin/test/runManagementProcess.test.cjs @@ -0,0 +1,72 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + ClusterRunManagementProcessConfigError, + loadClusterRunManagementProcessConfig, + startClusterRunManagementProcess, +} = require('@qinglong/cluster-admin/run-management-process'); + +function enabled(overrides = {}) { + return { + QL3_RUN_MANAGEMENT_ENABLED: 'true', + QL3_PROFILE: 'cluster-admin', + QL3_RUN_MANAGEMENT_TLS_CERT_FILE: '/run/ql3/run/tls.crt', + QL3_RUN_MANAGEMENT_TLS_KEY_FILE: '/run/ql3/run/tls.key', + QL3_RUN_MANAGEMENT_CLIENT_CA_FILE: '/run/ql3/run/client-ca.crt', + QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE: '/run/ql3/run/client.crl', + QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE: '/run/ql3/run/identity.json', + QL3_POSTGRES_RUN_MANAGER_HOST: 'postgres.qinglong3-system.svc', + QL3_POSTGRES_RUN_MANAGER_DATABASE: 'qinglong3', + QL3_POSTGRES_RUN_MANAGER_USER: 'ql3_run_manager', + QL3_POSTGRES_RUN_MANAGER_PASSWORD: 'secret', + QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: 'postgres.qinglong3-system.svc', + ...overrides, + }; +} + +test('disabled Run manager acquires no PostgreSQL or file authority', async () => { + let opened = false; + const runtime = await startClusterRunManagementProcess({ + environment: {}, + openDatabase: async () => { + opened = true; + throw new Error('must not open'); + }, + }); + assert.equal(runtime.status, 'disabled'); + assert.equal(opened, false); + await runtime.close(); +}); + +test('loads a bounded opt-in Run-only process configuration', () => { + const config = loadClusterRunManagementProcessConfig(enabled()); + assert.equal(config.enabled, true); + assert.equal(config.port, 8448); + assert.equal(config.http.maxConcurrentRequests, 16); + assert.equal(config.database.pool.maxConnections, 2); + assert.equal(config.database.connection.user, 'ql3_run_manager'); + assert.deepEqual(config.database.connection.tls, { + mode: 'verify-full', + servername: 'postgres.qinglong3-system.svc', + }); +}); + +test('rejects profile drift and implicit insecure PostgreSQL', () => { + assert.throws( + () => loadClusterRunManagementProcessConfig(enabled({ QL3_PROFILE: 'cluster-control' })), + ClusterRunManagementProcessConfigError, + ); + assert.throws( + () => + loadClusterRunManagementProcessConfig( + enabled({ + QL3_POSTGRES_RUN_MANAGER_TLS_MODE: 'disable', + QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: undefined, + }), + ), + ClusterRunManagementProcessConfigError, + ); +}); diff --git a/packages/ql3-cluster-admin/test/runManagementTransport.test.cjs b/packages/ql3-cluster-admin/test/runManagementTransport.test.cjs new file mode 100644 index 00000000..0f1221d7 --- /dev/null +++ b/packages/ql3-cluster-admin/test/runManagementTransport.test.cjs @@ -0,0 +1,141 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + ClusterRunManagementTransportAuthenticationError, + ClusterRunManagementTransportRequestError, + createClusterRunManagementTransport, + normalizeClusterRunManagementCommand, +} = require('@qinglong/cluster-admin/run-management-transport'); + +const NOW = 1_000_000; + +function principal(overrides = {}) { + return { + subject: { type: 'user', id: 'operator-1' }, + authenticationId: 'oidc:run-management-1', + authenticatedAtMs: 999_000, + expiresAtMs: 1_100_000, + assurance: 'multi_factor', + ...overrides, + }; +} + +function command(overrides = {}) { + return { + schemaVersion: 1, + operation: 'run.retry', + request: { + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'request-1', + auditEventId: '019f9300-0000-4000-8000-000000000001', + failureAuditEventId: '019f9300-0000-4000-8000-000000000002', + body: { + schema: 'qinglong/run-manual-retry@v1', + mutationId: '019f9300-0000-4000-8000-000000000003', + expectedRunVersion: 7, + expectedRunStatus: 'failed', + }, + ...overrides, + }, + }; +} + +function retryResult() { + return { + status: 'accepted', + projectId: 'project-1', + sourceRunId: 'source-run-1', + sourceRunStatus: 'failed', + sourceRunVersion: 7, + runId: '019f9300-0000-4000-8000-000000000010', + retryOfRunId: 'source-run-1', + taskId: 'task-1', + taskRevision: `qltd:v1:1:${'a'.repeat(64)}`, + attemptId: '019f9300-0000-4000-8000-000000000011', + runStatus: 'queued', + runVersion: 2, + eventSequence: 2, + executorType: 'remote_worker', + executionRevisionDigest: 'b'.repeat(64), + createdAtMs: NOW, + }; +} + +test('routes one exact strong User retry and emits the shared response', async () => { + const calls = []; + const transport = createClusterRunManagementTransport({ + now: () => NOW, + service: { + async retry(request) { + calls.push(request); + return retryResult(); + }, + }, + }); + const result = await transport.execute(command(), { + authenticate: async () => principal(), + }); + assert.equal(calls.length, 1); + assert.equal(calls[0].mutationId, command().request.body.mutationId); + assert.equal(calls[0].principal.assurance, 'multi_factor'); + assert.deepEqual(result, { + schemaVersion: 1, + operation: 'run.retry', + retry: { + schema: 'qinglong/run-manual-retry@v1', + ...retryResult(), + }, + }); +}); + +test('rejects weak or non-User identity before service authority', async () => { + let called = false; + const transport = createClusterRunManagementTransport({ + now: () => NOW, + service: { + async retry() { + called = true; + return retryResult(); + }, + }, + }); + await assert.rejects( + transport.execute(command(), { + authenticate: async () => principal({ assurance: 'single_factor' }), + }), + ClusterRunManagementTransportAuthenticationError, + ); + await assert.rejects( + transport.execute(command(), { + authenticate: async () => + principal({ subject: { type: 'agent', id: 'agent-1' } }), + }), + ClusterRunManagementTransportAuthenticationError, + ); + assert.equal(called, false); +}); + +test('rejects widened commands and ambiguous audit identity', () => { + assert.throws( + () => normalizeClusterRunManagementCommand({ ...command(), principal: principal() }), + ClusterRunManagementTransportRequestError, + ); + assert.throws( + () => + normalizeClusterRunManagementCommand( + command({ failureAuditEventId: command().request.auditEventId }), + ), + ClusterRunManagementTransportRequestError, + ); + assert.throws( + () => + normalizeClusterRunManagementCommand( + command({ body: { ...command().request.body, expectedRunStatus: 'lost' } }), + ), + ClusterRunManagementTransportRequestError, + ); +}); diff --git a/packages/ql3-cluster-control/test/application.test.cjs b/packages/ql3-cluster-control/test/application.test.cjs index d5638263..c5e931b6 100644 --- a/packages/ql3-cluster-control/test/application.test.cjs +++ b/packages/ql3-cluster-control/test/application.test.cjs @@ -334,6 +334,7 @@ function databaseResource(events, options = {}) { 'plugin_package_workflow_task_attempt_snapshot', 'plugin_package_run_start_allowed', 'plugin_package_tool_start_allowed', + 'lock_run_management_policy_fence', ].includes(functionName), isOwner: false, })), diff --git a/packages/ql3-cluster-control/test/bootstrap.test.cjs b/packages/ql3-cluster-control/test/bootstrap.test.cjs index e3bbbb72..46dee65e 100644 --- a/packages/ql3-cluster-control/test/bootstrap.test.cjs +++ b/packages/ql3-cluster-control/test/bootstrap.test.cjs @@ -243,6 +243,7 @@ function databaseResource(events, overrides = {}) { 'plugin_package_workflow_task_attempt_snapshot', 'plugin_package_run_start_allowed', 'plugin_package_tool_start_allowed', + 'lock_run_management_policy_fence', ].includes(functionName), isOwner: false, })), diff --git a/packages/ql3-cluster-postgres/package.json b/packages/ql3-cluster-postgres/package.json index a67aa73c..aae3f4ad 100644 --- a/packages/ql3-cluster-postgres/package.json +++ b/packages/ql3-cluster-postgres/package.json @@ -75,6 +75,11 @@ "require": "./dist/run-management/runManualRetryRepository.js", "default": "./dist/run-management/runManualRetryRepository.js" }, + "./run-manager": { + "types": "./dist/entrypoints/runManager.d.ts", + "require": "./dist/entrypoints/runManager.js", + "default": "./dist/entrypoints/runManager.js" + }, "./approval-manager": { "types": "./dist/approval-management/index.d.ts", "require": "./dist/approval-management/index.js", diff --git a/packages/ql3-cluster-postgres/src/connection/pool.ts b/packages/ql3-cluster-postgres/src/connection/pool.ts index a462a9e6..2fd3f88a 100644 --- a/packages/ql3-cluster-postgres/src/connection/pool.ts +++ b/packages/ql3-cluster-postgres/src/connection/pool.ts @@ -19,6 +19,7 @@ const DEFAULT_AI_CREDENTIAL_TESTER_APPLICATION_NAME = const DEFAULT_AUTOMATION_MANAGER_APPLICATION_NAME = 'qinglong-automation-manager'; const DEFAULT_APPROVAL_MANAGER_APPLICATION_NAME = 'qinglong-approval-manager'; +const DEFAULT_RUN_MANAGER_APPLICATION_NAME = 'qinglong-run-manager'; const DEFAULT_PACKAGE_MANAGER_APPLICATION_NAME = 'qinglong-package-manager'; const DEFAULT_PACKAGE_EXECUTOR_APPLICATION_NAME = 'qinglong-package-executor'; const DEFAULT_WORKER_CREDENTIAL_MANAGER_APPLICATION_NAME = @@ -89,6 +90,7 @@ export type PostgresDatabaseRole = | 'admin' | 'automation-manager' | 'approval-manager' + | 'run-manager' | 'package-manager' | 'package-executor' | 'worker-credential-manager' @@ -255,6 +257,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig { 'admin', 'automation-manager', 'approval-manager', + 'run-manager', 'package-manager', 'package-executor', 'worker-credential-manager', @@ -296,6 +299,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig { const isAdmin = options.role === 'admin'; const isAutomationManager = options.role === 'automation-manager'; const isApprovalManager = options.role === 'approval-manager'; + const isRunManager = options.role === 'run-manager'; const isPackageManager = options.role === 'package-manager'; const isPackageExecutor = options.role === 'package-executor'; const isWorkerCredentialManager = @@ -309,6 +313,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig { isAdmin || isAutomationManager || isApprovalManager || + isRunManager || isPackageManager || isPackageExecutor || isWorkerCredentialManager || @@ -370,6 +375,8 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig { ? DEFAULT_AUTOMATION_MANAGER_APPLICATION_NAME : isApprovalManager ? DEFAULT_APPROVAL_MANAGER_APPLICATION_NAME + : isRunManager + ? DEFAULT_RUN_MANAGER_APPLICATION_NAME : isPackageManager ? DEFAULT_PACKAGE_MANAGER_APPLICATION_NAME : isPackageExecutor diff --git a/packages/ql3-cluster-postgres/src/entrypoints/runManager.ts b/packages/ql3-cluster-postgres/src/entrypoints/runManager.ts new file mode 100644 index 00000000..434ea0b5 --- /dev/null +++ b/packages/ql3-cluster-postgres/src/entrypoints/runManager.ts @@ -0,0 +1,31 @@ +export { PostgresRunManualRetryRepository } from '../run-management/runManualRetryRepository'; +export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository'; +export { PostgresSecurityAuditRepository } from '../security/securityAuditRepository'; +export { + PostgresPluginPackageIdentityKeysetLedgerRepository as PostgresRunManagementIdentityKeysetLedgerRepository, + type ClusterManagementIdentityAuthority, +} from '../management/pluginPackageIdentityKeysetLedgerRepository'; +export { + PgPoolBinding, + createPostgresDatabaseOpener, + isPostgresTlsDnsServername, + type OpenPostgresDatabaseOptions, + type PostgresConnectionOptions, + type PostgresDatabaseRole, + type PostgresPoolOptions, +} from '../connection/pool'; +export { + PostgresConnectionEnvironmentError, + loadPostgresConnectionEnvironment, + type PostgresConnectionEnvironment, + type PostgresConnectionEnvironmentKeys, +} from '../connection/connectionEnvironment'; +export { + loadPostgresCertificateAuthorityFile, + type PostgresCertificateAuthorityFileInspection, +} from '../connection/certificateAuthority'; +export { + PostgresSchemaReadinessError, + assertPostgresRunManagerSchemaReady, + type PostgresSchemaReadinessReport, +} from '../schema/schemaReadiness'; diff --git a/packages/ql3-cluster-postgres/src/management/pluginPackageIdentityKeysetLedgerRepository.ts b/packages/ql3-cluster-postgres/src/management/pluginPackageIdentityKeysetLedgerRepository.ts index e55d6532..6dc23a70 100644 --- a/packages/ql3-cluster-postgres/src/management/pluginPackageIdentityKeysetLedgerRepository.ts +++ b/packages/ql3-cluster-postgres/src/management/pluginPackageIdentityKeysetLedgerRepository.ts @@ -5,7 +5,8 @@ export type ClusterManagementIdentityAuthority = | 'plugin-package-management' | 'worker-credential-management' | 'automation-management' - | 'approval-management'; + | 'approval-management' + | 'run-management'; const DIGEST_PATTERN = /^[A-Za-z0-9_-]{43}$/; const KEY_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/; @@ -189,7 +190,8 @@ export class PostgresPluginPackageIdentityKeysetLedgerRepository authority !== 'plugin-package-management' && authority !== 'worker-credential-management' && authority !== 'automation-management' && - authority !== 'approval-management' + authority !== 'approval-management' && + authority !== 'run-management' ) { throw new TypeError( 'PostgreSQL management identity keyset authority is invalid', diff --git a/packages/ql3-cluster-postgres/src/migration/migrationManifest.ts b/packages/ql3-cluster-postgres/src/migration/migrationManifest.ts index ad2a2d1f..3c039dd5 100644 --- a/packages/ql3-cluster-postgres/src/migration/migrationManifest.ts +++ b/packages/ql3-cluster-postgres/src/migration/migrationManifest.ts @@ -283,5 +283,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest = checksum: 'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30', }), + Object.freeze({ + id: 'pg-0056-run-management-boundary', + checksum: + '7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8', + }), ]), }); diff --git a/packages/ql3-cluster-postgres/src/migrations/index.ts b/packages/ql3-cluster-postgres/src/migrations/index.ts index 73e19b76..b0404562 100644 --- a/packages/ql3-cluster-postgres/src/migrations/index.ts +++ b/packages/ql3-cluster-postgres/src/migrations/index.ts @@ -58,6 +58,7 @@ import { pg0052AutomationManagementIdentityKeysetLedgerMigration } from './pg-00 import { pg0053PluginPackageWorkflowRunListIndexMigration } from './pg-0053-plugin-package-workflow-run-list-index'; import { pg0054ApprovalManagementBoundaryMigration } from './pg-0054-approval-management-boundary'; import { pg0055RunAttemptLogRetentionMigration } from './pg-0055-run-attempt-log-retention'; +import { pg0056RunManagementBoundaryMigration } from '../run-management/pg-0056-run-management-boundary'; export const postgresqlMainMigrationStream: MigrationStreamDefinition = Object.freeze({ @@ -121,5 +122,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition; export const CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS = 60_000; export const CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT = 64; -const ALLOWED_ROLES = new Set([ - 'owner', - 'admin', - 'operator', -]); const CLUSTER_STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']); const TASK_REVISION_PATTERN = /^qltd:v1:([1-9]\d*):([0-9a-f]{64})$/; @@ -128,45 +122,23 @@ async function confirmAuthorization( client: PostgresClient, command: Readonly, ): Promise { - const project = await client.query( + const result = await client.query( ` - SELECT status AS "projectStatus", version AS "projectVersion" - FROM "ql3"."projects" WHERE id = $1 FOR UPDATE - `, - [command.projectId], - ); - if (project.rows.length === 0) throw new RunManualRetryNotFoundError(); - if (project.rows.length !== 1) throw unavailable(); - // Authorized management mutations take the same Project lock. Keeping this - // append-only RoleBinding read lock-free avoids granting UPDATE authority to - // the runtime role merely to use PostgreSQL row-lock syntax. - const binding = await client.query( - ` - SELECT version AS "bindingVersion", state AS "bindingState", - role AS "bindingRole" - FROM "ql3"."project_role_bindings" - WHERE project_id = $1 AND subject_type = $2 AND subject_id = $3 - ORDER BY version DESC LIMIT 1 + SELECT "ql3"."lock_run_management_policy_fence"( + $1::varchar, $2::varchar, $3::varchar, $4::integer, $5::integer + ) AS "matches" `, [ command.projectId, command.principal.subject.type, command.principal.subject.id, + command.policyFence.projectVersion, + command.policyFence.bindingVersion, ], ); - const currentProject = project.rows[0]!; - const currentBinding = binding.rows[0]; if ( - text(currentProject, 'projectStatus') !== 'active' || - integer(currentProject, 'projectVersion') !== - command.policyFence.projectVersion || - !currentBinding || - integer(currentBinding, 'bindingVersion') !== - command.policyFence.bindingVersion || - text(currentBinding, 'bindingState') !== 'active' || - !ALLOWED_ROLES.has( - text(currentBinding, 'bindingRole') as RunManualRetryAllowedRole, - ) + result.rows.length !== 1 || + !postgresRequiredBoolean(result.rows[0]!.matches, unavailable) ) { throw new RunManualRetryFenceRejectedError('authorization_changed'); } @@ -185,8 +157,6 @@ async function findReplay( run.execution_origin AS "executionOrigin", run.execution_owner AS "executionOwner", run.triggered_by AS "triggeredBy", run.request_id AS "requestId", - run.status AS "runStatus", run.version AS "runVersion", - run.event_sequence AS "eventSequence", run.created_at_ms AS "createdAtMs", attempt.id AS "attemptId", attempt.executor_type AS "executorType", created.actor_type AS "createdActorType", @@ -205,7 +175,6 @@ async function findReplay( ON queued.run_id = run.id AND queued.sequence = 2 AND queued.type = 'run.queued' WHERE run.project_id = $1 AND run.idempotency_key = $2 - FOR UPDATE OF run `, [command.projectId, `ql3:run-manual-retry:v1:${command.mutationId}`], ); @@ -227,9 +196,6 @@ function replayResult( text(row, 'executionOwner') !== 'runtime' || text(row, 'triggeredBy') !== command.principal.subject.id || text(row, 'requestId') !== command.mutationId || - text(row, 'runStatus') !== 'queued' || - integer(row, 'runVersion') !== 2 || - integer(row, 'eventSequence') !== 2 || text(row, 'executorType') !== 'remote_worker' || text(row, 'createdActorType') !== command.principal.subject.type || text(row, 'createdActorId') !== command.principal.subject.id || @@ -295,7 +261,6 @@ async function findSource( LIMIT 1 ) AS attempt ON true WHERE run.id = $1 - FOR UPDATE OF run `, [command.sourceRunId], ); diff --git a/packages/ql3-cluster-postgres/src/schema/schemaContract.ts b/packages/ql3-cluster-postgres/src/schema/schemaContract.ts index 915967a2..c0936930 100644 --- a/packages/ql3-cluster-postgres/src/schema/schemaContract.ts +++ b/packages/ql3-cluster-postgres/src/schema/schemaContract.ts @@ -15,13 +15,14 @@ export interface PostgresSchemaContractFunction { export interface PostgresSchemaContract { readonly schema: 'ql3'; readonly contractName: 'control-core'; - readonly contractVersion: 54; - readonly migrationId: 'pg-0055-run-attempt-log-retention'; + readonly contractVersion: 55; + readonly migrationId: 'pg-0056-run-management-boundary'; readonly minimumServerMajor: 16; readonly maximumServerMajor: 18; readonly capabilities: Readonly<{ run_core: 1; run_attempt_log_retention: 1; + run_management_boundary: 1; run_dispatch_lease: 1; run_retry_policy: 1; project_policy: 1; @@ -101,8 +102,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract = Object.freeze({ schema: 'ql3', contractName: 'control-core', - contractVersion: 54, - migrationId: 'pg-0055-run-attempt-log-retention', + contractVersion: 55, + migrationId: 'pg-0056-run-management-boundary', minimumServerMajor: 16, maximumServerMajor: 18, capabilities: Object.freeze({ @@ -143,6 +144,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract = project_tool_definition_snapshot: 1, run_core: 1, run_attempt_log_retention: 1, + run_management_boundary: 1, run_dispatch_lease: 1, run_retry_policy: 1, security_audit: 1, @@ -2323,6 +2325,15 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract = volatility: 'volatile', configuration: Object.freeze(['search_path=pg_catalog, ql3']), }), + Object.freeze({ + name: 'lock_run_management_policy_fence', + identityArguments: + 'character varying, character varying, character varying, integer, integer', + owner: 'ql3_migration', + securityDefiner: true, + volatility: 'volatile', + configuration: Object.freeze(['search_path=pg_catalog, ql3']), + }), Object.freeze({ name: 'commit_plugin_package_task_reconciliation', identityArguments: diff --git a/packages/ql3-cluster-postgres/src/schema/schemaReadiness.ts b/packages/ql3-cluster-postgres/src/schema/schemaReadiness.ts index 78302937..db4d11a9 100644 --- a/packages/ql3-cluster-postgres/src/schema/schemaReadiness.ts +++ b/packages/ql3-cluster-postgres/src/schema/schemaReadiness.ts @@ -19,6 +19,7 @@ export const POSTGRES_SCHEMA_READINESS_ERROR_CODES = [ 'admin_role_invalid', 'automation_manager_role_invalid', 'approval_manager_role_invalid', + 'run_manager_role_invalid', 'package_manager_role_invalid', 'package_executor_role_invalid', 'worker_credential_manager_role_invalid', @@ -1361,6 +1362,37 @@ const REQUIRED_APPROVAL_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze( ), ); +const REQUIRED_RUN_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze( + Object.fromEntries( + postgresqlControlSchemaContract.tables.map(({ name }) => [ + name, + Object.freeze( + name === 'schema_migrations' || + name === 'schema_capabilities' || + name === 'projects' || + name === 'project_role_bindings' || + name === 'task_definitions' || + name === 'task_definition_revisions' || + name === 'task_execution_revisions' + ? { ...NO_TABLE_PRIVILEGES, select: true } + : name === 'runs' || + name === 'run_attempts' || + name === 'run_events' || + name === 'security_audit_events' + ? { ...NO_TABLE_PRIVILEGES, select: true, insert: true } + : name === 'plugin_package_identity_keyset_ledger' + ? { + ...NO_TABLE_PRIVILEGES, + select: true, + insert: true, + update: true, + } + : NO_TABLE_PRIVILEGES, + ), + ]), + ), +); + const REQUIRED_WORKER_CREDENTIAL_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze( Object.fromEntries( @@ -1448,6 +1480,7 @@ const REQUIRED_RUNTIME_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges = enforce_plugin_package_stage_provenance: false, lock_active_plugin_package_project: false, lock_approval_policy_fence: false, + lock_run_management_policy_fence: true, plugin_package_automation_start_allowed: true, plugin_package_workflow_admission_snapshot: true, plugin_package_workflow_task_attempt_snapshot: true, @@ -1464,6 +1497,7 @@ const REQUIRED_PACKAGE_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges = enforce_plugin_package_stage_provenance: false, lock_active_plugin_package_project: false, lock_approval_policy_fence: true, + lock_run_management_policy_fence: false, plugin_package_automation_start_allowed: false, plugin_package_workflow_admission_snapshot: false, plugin_package_workflow_task_attempt_snapshot: false, @@ -1480,6 +1514,7 @@ const REQUIRED_PACKAGE_EXECUTOR_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges enforce_plugin_package_stage_provenance: false, lock_active_plugin_package_project: true, lock_approval_policy_fence: true, + lock_run_management_policy_fence: false, plugin_package_automation_start_allowed: false, plugin_package_workflow_admission_snapshot: false, plugin_package_workflow_task_attempt_snapshot: false, @@ -1500,6 +1535,12 @@ const REQUIRED_APPROVAL_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges lock_approval_policy_fence: true, }); +const REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges = + Object.freeze({ + ...NO_FUNCTION_PRIVILEGES, + lock_run_management_policy_fence: true, + }); + function safeInteger(value: unknown): number | null { if (typeof value === 'number' && Number.isSafeInteger(value)) return value; if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) { @@ -1864,6 +1905,7 @@ async function assertRole( | 'admin_role_invalid' | 'automation_manager_role_invalid' | 'approval_manager_role_invalid' + | 'run_manager_role_invalid' | 'package_manager_role_invalid' | 'package_executor_role_invalid' | 'worker_credential_manager_role_invalid' @@ -2120,6 +2162,30 @@ export async function assertPostgresApprovalManagerSchemaReady( }); } +export async function assertPostgresRunManagerSchemaReady( + queryable: PostgresMigrationQueryable, + contract: PostgresSchemaContract = postgresqlControlSchemaContract, +): Promise { + const server = await readServer(queryable, contract); + const migrationIds = await assertHistory(queryable); + await assertCapability(queryable, contract); + await assertSchemaContract(queryable, contract); + await assertRole( + queryable, + contract, + REQUIRED_RUN_MANAGER_PRIVILEGES, + REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES, + 'run_manager_role_invalid', + ); + return Object.freeze({ + ready: true, + ...server, + contractName: contract.contractName, + contractVersion: contract.contractVersion, + migrationIds, + }); +} + export async function assertPostgresPackageManagerSchemaReady( queryable: PostgresMigrationQueryable, contract: PostgresSchemaContract = postgresqlControlSchemaContract, diff --git a/packages/ql3-cluster-postgres/test/pluginPackageIdentityKeysetLedgerRepository.test.cjs b/packages/ql3-cluster-postgres/test/pluginPackageIdentityKeysetLedgerRepository.test.cjs index 51ed5047..2c482886 100644 --- a/packages/ql3-cluster-postgres/test/pluginPackageIdentityKeysetLedgerRepository.test.cjs +++ b/packages/ql3-cluster-postgres/test/pluginPackageIdentityKeysetLedgerRepository.test.cjs @@ -108,7 +108,7 @@ test('serializes first observation, exact replay and append-only rotation', asyn assert.equal(value.releases(), 3); }); -test('isolates Plugin, Worker, automation and Approval generations by authority key', async () => { +test('isolates Plugin, Worker, automation, Approval and Run generations by authority key', async () => { const value = fixture('worker-credential-management'); await value.repository.observe( snapshot(1, { audience: 'qinglong3-worker-credential-management' }), @@ -131,6 +131,14 @@ test('isolates Plugin, Worker, automation and Approval generations by authority approval.queries.find(({ text }) => text.startsWith('INSERT')).values[0], 'approval-management', ); + const run = fixture('run-management'); + await run.repository.observe( + snapshot(1, { audience: 'qinglong3-run-management' }), + ); + assert.equal( + run.queries.find(({ text }) => text.startsWith('INSERT')).values[0], + 'run-management', + ); assert.throws( () => fixture('worker-credential-executor'), TypeError, diff --git a/packages/ql3-cluster-postgres/test/pool.test.cjs b/packages/ql3-cluster-postgres/test/pool.test.cjs index 391c8814..6ea68e1a 100644 --- a/packages/ql3-cluster-postgres/test/pool.test.cjs +++ b/packages/ql3-cluster-postgres/test/pool.test.cjs @@ -104,6 +104,7 @@ test('enforces role-specific bounded pool sizes', () => { 'ai-credential-tester', 'automation-manager', 'approval-manager', + 'run-manager', 'worker-credential-manager', 'worker-credential-executor', ]) { diff --git a/packages/ql3-cluster-postgres/test/postgresqlMigrationDefinitions.test.cjs b/packages/ql3-cluster-postgres/test/postgresqlMigrationDefinitions.test.cjs index ec78b5e2..7d07cebe 100644 --- a/packages/ql3-cluster-postgres/test/postgresqlMigrationDefinitions.test.cjs +++ b/packages/ql3-cluster-postgres/test/postgresqlMigrationDefinitions.test.cjs @@ -104,6 +104,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async () 'pg-0053-plugin-package-workflow-run-list-index', 'pg-0054-approval-management-boundary', 'pg-0055-run-attempt-log-retention', + 'pg-0056-run-management-boundary', ], ); for (const migration of postgresqlMainMigrationStream.migrations) { @@ -514,6 +515,11 @@ test('freezes every published PostgreSQL migration checksum', () => { checksum: 'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30', }, + { + id: 'pg-0056-run-management-boundary', + checksum: + '7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8', + }, ]; assert.deepEqual( postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({ @@ -1904,3 +1910,30 @@ test('advances capability v54 with durable Cluster log retention authority', asy /migration_id = 'pg-0054-approval-management-boundary'/, ); }); + +test('advances capability v55 with isolated strong Run management authority', async () => { + const migration = migrationById('pg-0056-run-management-boundary'); + const statements = []; + await migration.up({ + async query(statement) { + statements.push(statement); + return { rows: [] }; + }, + }); + const sql = statements.join('\n'); + assert.match(sql, /ql3_run_manager/); + assert.match(sql, /lock_run_management_policy_fence/); + assert.match( + sql, + /GRANT SELECT, INSERT ON "ql3"\."runs", "ql3"\."run_attempts", "ql3"\."run_events", "ql3"\."security_audit_events" TO ql3_run_manager/, + ); + assert.doesNotMatch(sql, /GRANT UPDATE ON "ql3"\."runs"/); + assert.match(sql, /'run-management'/); + assert.match(sql, /contract_version = 55/); + assert.match(sql, /"run_management_boundary":1/); + assert.match(sql, /contract_version = 54/); + assert.match( + sql, + /migration_id = 'pg-0055-run-attempt-log-retention'/, + ); +}); diff --git a/packages/ql3-cluster-postgres/test/postgresqlSchemaReadiness.test.cjs b/packages/ql3-cluster-postgres/test/postgresqlSchemaReadiness.test.cjs index 8736c1fb..404aab5d 100644 --- a/packages/ql3-cluster-postgres/test/postgresqlSchemaReadiness.test.cjs +++ b/packages/ql3-cluster-postgres/test/postgresqlSchemaReadiness.test.cjs @@ -4,6 +4,7 @@ const { PostgresSchemaReadinessError, assertPostgresAdminSchemaReady, assertPostgresApprovalManagerSchemaReady, + assertPostgresRunManagerSchemaReady, assertPostgresAutomationManagerSchemaReady, assertPostgresPackageExecutorSchemaReady, assertPostgresPackageManagerSchemaReady, @@ -474,6 +475,37 @@ function approvalManagerPrivileges() { })); } +function runManagerPrivileges() { + const readable = new Set([ + 'schema_migrations', + 'schema_capabilities', + 'projects', + 'project_role_bindings', + 'task_definitions', + 'task_definition_revisions', + 'task_execution_revisions', + 'runs', + 'run_attempts', + 'run_events', + 'security_audit_events', + 'plugin_package_identity_keyset_ledger', + ]); + return postgresqlControlSchemaContract.tables.map(({ name: tableName }) => ({ + tableName, + selectAllowed: readable.has(tableName), + insertAllowed: [ + 'runs', + 'run_attempts', + 'run_events', + 'security_audit_events', + 'plugin_package_identity_keyset_ledger', + ].includes(tableName), + updateAllowed: tableName === 'plugin_package_identity_keyset_ledger', + deleteAllowed: false, + isOwner: false, + })); +} + function workerCredentialPrivileges(kind) { const manager = kind === 'manager'; const readable = new Set([ @@ -634,6 +666,8 @@ function queryable(overrides = {}) { executeAllowed: overrides.functionMode === 'manager' ? functionName === 'lock_approval_policy_fence' + : overrides.functionMode === 'run-manager' + ? functionName === 'lock_run_management_policy_fence' : overrides.functionMode === 'executor' ? [ 'commit_plugin_package_lifecycle', @@ -651,6 +685,7 @@ function queryable(overrides = {}) { 'plugin_package_workflow_task_attempt_snapshot', 'plugin_package_run_start_allowed', 'plugin_package_tool_start_allowed', + 'lock_run_management_policy_fence', ].includes(functionName), isOwner: false, })), @@ -696,7 +731,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro serverMajor: 16, currentUser: 'ql3_runtime', contractName: 'control-core', - contractVersion: 54, + contractVersion: 55, migrationIds: [ 'pg-0001-schema-capability', 'pg-0002-run-core', @@ -753,6 +788,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro 'pg-0053-plugin-package-workflow-run-list-index', 'pg-0054-approval-management-boundary', 'pg-0055-run-attempt-log-retention', + 'pg-0056-run-management-boundary', ], }); }); @@ -783,10 +819,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async ( }), ); assert.equal(report.currentUser, 'ql3_admin'); - assert.equal(report.contractVersion, 54); + assert.equal(report.contractVersion, 55); assert.equal( report.migrationIds.at(-1), - 'pg-0055-run-attempt-log-retention', + 'pg-0056-run-management-boundary', ); }); @@ -799,10 +835,10 @@ test('accepts the isolated least-privilege automation manager role', async () => }), ); assert.equal(report.currentUser, 'ql3_automation_manager'); - assert.equal(report.contractVersion, 54); + assert.equal(report.contractVersion, 55); assert.equal( report.migrationIds.at(-1), - 'pg-0055-run-attempt-log-retention', + 'pg-0056-run-management-boundary', ); const widened = automationManagerPrivileges(); @@ -831,10 +867,10 @@ test('accepts the isolated least-privilege human Approval manager role', async ( }), ); assert.equal(report.currentUser, 'ql3_approval_manager'); - assert.equal(report.contractVersion, 54); + assert.equal(report.contractVersion, 55); assert.equal( report.migrationIds.at(-1), - 'pg-0055-run-attempt-log-retention', + 'pg-0056-run-management-boundary', ); const widened = approvalManagerPrivileges(); @@ -856,6 +892,35 @@ test('accepts the isolated least-privilege human Approval manager role', async ( ); }); +test('accepts the isolated least-privilege Run manager role', async () => { + const report = await assertPostgresRunManagerSchemaReady( + queryable({ + currentUser: 'ql3_run_manager', + privileges: runManagerPrivileges(), + functionMode: 'run-manager', + }), + ); + assert.equal(report.currentUser, 'ql3_run_manager'); + assert.equal(report.contractVersion, 55); + assert.equal(report.migrationIds.at(-1), 'pg-0056-run-management-boundary'); + + const widened = runManagerPrivileges(); + widened.find(({ tableName }) => tableName === 'runs').updateAllowed = true; + await assert.rejects( + assertPostgresRunManagerSchemaReady( + queryable({ + currentUser: 'ql3_run_manager', + privileges: widened, + functionMode: 'run-manager', + }), + ), + (error) => + error instanceof PostgresSchemaReadinessError && + error.code === 'run_manager_role_invalid' && + error.facts.includes('table-privileges:runs'), + ); +}); + test('accepts isolated Package manager and executor roles', async () => { const manager = await assertPostgresPackageManagerSchemaReady( queryable({ @@ -941,10 +1006,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => { }), ); assert.equal(report.currentUser, 'ql3_worker_ingress'); - assert.equal(report.contractVersion, 54); + assert.equal(report.contractVersion, 55); assert.equal( report.migrationIds.at(-1), - 'pg-0055-run-attempt-log-retention', + 'pg-0056-run-management-boundary', ); }); diff --git a/packages/ql3-cluster-postgres/test/runManualRetryRepository.test.cjs b/packages/ql3-cluster-postgres/test/runManualRetryRepository.test.cjs index b72806c5..358c3cf2 100644 --- a/packages/ql3-cluster-postgres/test/runManualRetryRepository.test.cjs +++ b/packages/ql3-cluster-postgres/test/runManualRetryRepository.test.cjs @@ -138,21 +138,11 @@ function fixture(options = {}) { if (normalized.includes('statement_timestamp()')) { return { rows: [{ nowMs: 1_000_000 }], rowCount: 1 }; } - if (normalized.includes('FROM "ql3"."projects"')) { - const rows = options.projectRows ?? [ - { projectStatus: 'active', projectVersion: 2 }, - ]; - return { rows, rowCount: rows.length }; - } - if (normalized.includes('project_role_bindings')) { - const rows = options.bindingRows ?? [ - { - bindingVersion: 3, - bindingState: 'active', - bindingRole: 'operator', - }, - ]; - return { rows, rowCount: rows.length }; + if (normalized.includes('lock_run_management_policy_fence')) { + return { + rows: [{ matches: options.authorizationMatches ?? true }], + rowCount: 1, + }; } if (normalized.includes('idempotency_key = $2')) { const rows = options.replayRows ?? []; @@ -245,7 +235,11 @@ test('atomically appends a linked queued Run, remote Attempt, events and allowed }); test('returns durable identities for an exact replay without appending again', async () => { - const { calls, repository } = fixture({ replayRows: [replayRow()] }); + const { calls, repository } = fixture({ + replayRows: [ + replayRow({ runStatus: 'running', runVersion: 4, eventSequence: 4 }), + ], + }); const result = await repository.retryRun( command({ runId: '019f9200-0000-4000-8000-000000000102', @@ -257,6 +251,7 @@ test('returns durable identities for an exact replay without appending again', a assert.equal(result.status, 'existing'); assert.equal(result.runId, IDS.runId); assert.equal(result.attemptId, IDS.attemptId); + assert.equal(result.runStatus, 'queued'); assert.equal( calls.some(({ sql }) => sql.startsWith('INSERT INTO')), false, @@ -283,15 +278,11 @@ test('rejects stale authentication and changed authorization inside the transact true, ); assert.equal( - stale.calls.some(({ sql }) => sql.includes('FROM "ql3"."projects"')), + stale.calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')), false, ); - const changed = fixture({ - bindingRows: [ - { bindingVersion: 4, bindingState: 'active', bindingRole: 'operator' }, - ], - }); + const changed = fixture({ authorizationMatches: false }); await assert.rejects( changed.repository.retryRun(command()), (error) => diff --git a/scripts/ql3-cloudnativepg-deployment-audit.cjs b/scripts/ql3-cloudnativepg-deployment-audit.cjs index 6c9da752..535daa59 100644 --- a/scripts/ql3-cloudnativepg-deployment-audit.cjs +++ b/scripts/ql3-cloudnativepg-deployment-audit.cjs @@ -65,6 +65,10 @@ const EXPECTED_ROLES = Object.freeze({ connectionLimit: 32, secret: 'ql3-postgres-runtime-auth', }), + ql3_run_manager: Object.freeze({ + connectionLimit: 4, + secret: 'ql3-postgres-run-manager-auth', + }), ql3_worker_credential_executor: Object.freeze({ connectionLimit: 4, secret: 'ql3-postgres-worker-credential-executor-auth', @@ -258,7 +262,7 @@ function assertRolesAndDatabase(readFile, root, findings) { findings.push( finding( 'QL3_CNPG_DATABASE_ROLE', - 'DatabaseRole resources must match the thirteen fixed least-privilege identities', + 'DatabaseRole resources must match the fourteen fixed least-privilege identities', ), ); break; @@ -272,7 +276,7 @@ function assertRolesAndDatabase(readFile, root, findings) { findings.push( finding( 'QL3_CNPG_DATABASE_ROLE_SET', - 'exactly the thirteen reviewed migration, runtime, AI maintenance, AI credential management/testing, admin, automation, Approval, package and Worker roles are required', + 'exactly the fourteen reviewed migration, runtime, AI maintenance, AI credential management/testing, admin, automation, Approval, Run, package and Worker roles are required', ), ); } @@ -332,6 +336,7 @@ function assertSecretBoundary(readFile, root, findings) { ['ql3-postgres-admin-auth', 'ql3_admin'], ['ql3-postgres-automation-manager-auth', 'ql3_automation_manager'], ['ql3-postgres-approval-manager-auth', 'ql3_approval_manager'], + ['ql3-postgres-run-manager-auth', 'ql3_run_manager'], ['ql3-postgres-worker-ingress-auth', 'ql3_worker_ingress'], ['ql3-postgres-package-manager-auth', 'ql3_package_manager'], ['ql3-postgres-package-executor-auth', 'ql3_package_executor'], @@ -417,7 +422,7 @@ function assertSecretBoundary(readFile, root, findings) { findings.push( finding( 'QL3_CNPG_SECRET_EXAMPLE_SET', - 'the example must describe all thirteen database credentials and the reviewed workload Secrets', + 'the example must describe all fourteen database credentials and the reviewed workload Secrets', ), ); } diff --git a/scripts/ql3-cluster-dependency-audit.cjs b/scripts/ql3-cluster-dependency-audit.cjs index 30373454..e05a8588 100644 --- a/scripts/ql3-cluster-dependency-audit.cjs +++ b/scripts/ql3-cluster-dependency-audit.cjs @@ -2366,6 +2366,11 @@ function auditSourceImports(root, packagePath, findings) { 'src/approval-management/approvalManagementProcess.ts', ].includes(path.relative(packageDirectory, filePath)) && specifier === '@qinglong/cluster-postgres/approval-manager') || + ([ + 'src/run-management/runManagement.ts', + 'src/run-management/runManagementProcess.ts', + ].includes(path.relative(packageDirectory, filePath)) && + specifier === '@qinglong/cluster-postgres/run-manager') || ([ 'src/prompt-output/retention/promptOutputGcCli.ts', 'src/prompt-output/retention/promptOutputGcProcess.ts', diff --git a/scripts/ql3-postgres-ha-contract.cjs b/scripts/ql3-postgres-ha-contract.cjs index d18ae233..f1a73497 100644 --- a/scripts/ql3-postgres-ha-contract.cjs +++ b/scripts/ql3-postgres-ha-contract.cjs @@ -201,6 +201,9 @@ const { const { assertPostgresApprovalManagerSchemaReady, } = require('../packages/ql3-cluster-postgres/dist/approval-management/index.js'); +const { + assertPostgresRunManagerSchemaReady, +} = require('../packages/ql3-cluster-postgres/dist/entrypoints/runManager.js'); const { assertPostgresPackageExecutorSchemaReady, PostgresPluginPackageMaterializedRevisionRepository, @@ -341,6 +344,8 @@ const AUTOMATION_MANAGER_USER = 'ql3_automation_manager'; const AUTOMATION_MANAGER_PASSWORD = 'ql3_automation_manager_test'; const APPROVAL_MANAGER_USER = 'ql3_approval_manager'; const APPROVAL_MANAGER_PASSWORD = 'ql3_approval_manager_test'; +const RUN_MANAGER_USER = 'ql3_run_manager'; +const RUN_MANAGER_PASSWORD = 'ql3_run_manager_test'; const PACKAGE_MANAGER_USER = 'ql3_package_manager'; const PACKAGE_MANAGER_PASSWORD = 'ql3_package_manager_test'; const PACKAGE_EXECUTOR_USER = 'ql3_package_executor'; @@ -3721,6 +3726,9 @@ async function provisionCredentialRoles(database) { await database.pool.query( `CREATE ROLE ${APPROVAL_MANAGER_USER} LOGIN PASSWORD '${APPROVAL_MANAGER_PASSWORD}'`, ); + await database.pool.query( + `CREATE ROLE ${RUN_MANAGER_USER} LOGIN PASSWORD '${RUN_MANAGER_PASSWORD}'`, + ); await database.pool.query( `CREATE ROLE ${PACKAGE_MANAGER_USER} LOGIN PASSWORD '${PACKAGE_MANAGER_PASSWORD}'`, ); @@ -5010,54 +5018,70 @@ async function runIdentityKeysetLedgerMatrix(options) { authority === 'plugin-package-management' || authority === 'worker-credential-management' || authority === 'automation-management' || - authority === 'approval-management', + authority === 'approval-management' || + authority === 'run-management', ); const workerAuthority = authority === 'worker-credential-management'; const automationAuthority = authority === 'automation-management'; const approvalAuthority = authority === 'approval-management'; - const user = approvalAuthority + const runAuthority = authority === 'run-management'; + const user = runAuthority + ? RUN_MANAGER_USER + : approvalAuthority ? APPROVAL_MANAGER_USER : automationAuthority ? AUTOMATION_MANAGER_USER : workerAuthority ? WORKER_CREDENTIAL_MANAGER_USER : PACKAGE_MANAGER_USER; - const password = approvalAuthority + const password = runAuthority + ? RUN_MANAGER_PASSWORD + : approvalAuthority ? APPROVAL_MANAGER_PASSWORD : automationAuthority ? AUTOMATION_MANAGER_PASSWORD : workerAuthority ? WORKER_CREDENTIAL_MANAGER_PASSWORD : PACKAGE_MANAGER_PASSWORD; - const role = approvalAuthority + const role = runAuthority + ? 'run-manager' + : approvalAuthority ? 'approval-manager' : automationAuthority ? 'automation-manager' : workerAuthority ? 'worker-credential-manager' : 'package-manager'; - const applicationPrefix = approvalAuthority + const applicationPrefix = runAuthority + ? 'ql3-ha-run-keyset-ledger' + : approvalAuthority ? 'ql3-ha-approval-keyset-ledger' : automationAuthority ? 'ql3-ha-automation-keyset-ledger' : workerAuthority ? 'ql3-ha-worker-keyset-ledger' : 'ql3-ha-keyset-ledger'; - const keyPrefix = approvalAuthority + const keyPrefix = runAuthority + ? 'ha-run-identity-key' + : approvalAuthority ? 'ha-approval-identity-key' : automationAuthority ? 'ha-automation-identity-key' : workerAuthority ? 'ha-worker-identity-key' : 'ha-identity-key'; - const issuer = approvalAuthority + const issuer = runAuthority + ? 'https://run-identity.ha.example.test/' + : approvalAuthority ? 'https://approval-identity.ha.example.test/' : automationAuthority ? 'https://automation-identity.ha.example.test/' : workerAuthority ? 'https://worker-identity.ha.example.test/' : 'https://identity.ha.example.test/'; - const audience = approvalAuthority + const audience = runAuthority + ? 'qinglong3-run-management' + : approvalAuthority ? 'qinglong3-approval-management' : automationAuthority ? 'qinglong3-automation-management' @@ -9446,13 +9470,13 @@ async function runManualRunRetryHaEvidence(options) { }) ).definition; const firstRuntime = await databaseOpener( - 'runtime', - databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, primaryPort), + 'run-manager', + databaseUrl(RUN_MANAGER_USER, RUN_MANAGER_PASSWORD, primaryPort), 'ql3-ha-manual-run-retry-a', )(); const secondRuntime = await databaseOpener( - 'runtime', - databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, primaryPort), + 'run-manager', + databaseUrl(RUN_MANAGER_USER, RUN_MANAGER_PASSWORD, primaryPort), 'ql3-ha-manual-run-retry-b', )(); try { @@ -9707,6 +9731,7 @@ async function main(argv = process.argv.slice(2)) { let workerCredentialIdentityKeysetLedger; let automationIdentityKeysetLedger; let approvalIdentityKeysetLedger; + let runManagementIdentityKeysetLedger; let automationManagementInspection; let pluginPackageLifecycle; let pluginPackageQuarantine; @@ -9933,6 +9958,11 @@ async function main(argv = process.argv.slice(2)) { ), 'ql3-ha-approval-manager-readiness', )(); + const runManagerReadinessDatabase = await databaseOpener( + 'run-manager', + databaseUrl(RUN_MANAGER_USER, RUN_MANAGER_PASSWORD, primaryPort), + 'ql3-ha-run-manager-readiness', + )(); const packageManagerReadinessDatabase = await databaseOpener( 'package-manager', databaseUrl(PACKAGE_MANAGER_USER, PACKAGE_MANAGER_PASSWORD, primaryPort), @@ -9974,6 +10004,9 @@ async function main(argv = process.argv.slice(2)) { await assertPostgresApprovalManagerSchemaReady( approvalManagerReadinessDatabase.pool, ); + await assertPostgresRunManagerSchemaReady( + runManagerReadinessDatabase.pool, + ); await assertPostgresPackageManagerSchemaReady( packageManagerReadinessDatabase.pool, ); @@ -9992,6 +10025,7 @@ async function main(argv = process.argv.slice(2)) { adminReadinessDatabase.close(), automationManagerReadinessDatabase.close(), approvalManagerReadinessDatabase.close(), + runManagerReadinessDatabase.close(), packageManagerReadinessDatabase.close(), packageExecutorReadinessDatabase.close(), workerManagerReadinessDatabase.close(), @@ -10031,6 +10065,10 @@ async function main(argv = process.argv.slice(2)) { primaryPort, authority: 'approval-management', }); + runManagementIdentityKeysetLedger = await runIdentityKeysetLedgerMatrix({ + primaryPort, + authority: 'run-management', + }); timeline.push({ state: 'durable_identity_keyset_ledger_verified', atMs: Number((performance.now() - startedAt).toFixed(3)), @@ -11968,6 +12006,7 @@ async function main(argv = process.argv.slice(2)) { workerCredentialIdentityKeysetLedger, automationIdentityKeysetLedger, approvalIdentityKeysetLedger, + runManagementIdentityKeysetLedger, automationManagementInspection, pluginPackageLifecycle: pluginPackageLifecycle.report, pluginPackageQuarantine: pluginPackageQuarantine.report, diff --git a/test/back/ql3CloudNativePgDeploymentAudit.test.cjs b/test/back/ql3CloudNativePgDeploymentAudit.test.cjs index 47c6a1de..36b57917 100644 --- a/test/back/ql3CloudNativePgDeploymentAudit.test.cjs +++ b/test/back/ql3CloudNativePgDeploymentAudit.test.cjs @@ -32,6 +32,7 @@ test('accepts the locked CloudNativePG HA and authority profile', () => { 'ql3_migration', 'ql3_package_executor', 'ql3_package_manager', + 'ql3_run_manager', 'ql3_runtime', 'ql3_worker_credential_executor', 'ql3_worker_credential_manager', diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index e94c6624..0d9cecf1 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -340,10 +340,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: clusterAdmin.rootSourceFileRoles, }, { - sourceFiles: 87, + sourceFiles: 94, rootSourceFiles: 1, rootSourceLines: 61, - nestedSourceFiles: 86, + nestedSourceFiles: 93, rootSourceFileRoles: { 'modelInvocationMigrationCli.ts': 'binary_entry', }, @@ -421,10 +421,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: clusterPostgres.rootSourceFileRoles, }, { - sourceFiles: 150, + sourceFiles: 152, rootSourceFiles: 1, rootSourceLines: 125, - nestedSourceFiles: 149, + nestedSourceFiles: 151, rootSourceFileRoles: { 'index.ts': 'public_export' }, }, ); diff --git a/test/back/ql3RunManagementDeployment.test.cjs b/test/back/ql3RunManagementDeployment.test.cjs new file mode 100644 index 00000000..fa44a30d --- /dev/null +++ b/test/back/ql3RunManagementDeployment.test.cjs @@ -0,0 +1,85 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { test } = require('node:test'); +const yaml = require('js-yaml'); + +const ROOT = path.resolve(__dirname, '../..'); + +function document(relativePath) { + return yaml.load(fs.readFileSync(path.join(ROOT, relativePath), 'utf8')); +} + +test('keeps strong Run management opt-in, private and least-resource', () => { + const directory = 'deploy/kubernetes/ql3-cluster/operations/run-management/base'; + const deployment = document(`${directory}/deployment.yaml`); + const service = document(`${directory}/service.yaml`); + const policy = document(`${directory}/network-policy.yaml`); + const kustomization = document(`${directory}/kustomization.yaml`); + const container = deployment.spec.template.spec.containers[0]; + const environment = new Map(container.env.map((entry) => [entry.name, entry])); + + assert.equal(deployment.metadata.name, 'ql3-run-management'); + assert.equal(deployment.spec.replicas, 2); + assert.equal(deployment.spec.template.spec.automountServiceAccountToken, false); + assert.equal(container.securityContext.readOnlyRootFilesystem, true); + assert.equal(container.securityContext.allowPrivilegeEscalation, false); + assert.deepEqual(container.securityContext.capabilities.drop, ['ALL']); + assert.deepEqual(container.command, [ + 'node', + '/opt/qinglong/node_modules/@qinglong/cluster-admin/dist/run-management/runManagementCli.js', + ]); + assert.equal(environment.get('QL3_RUN_MANAGEMENT_ENABLED').value, 'true'); + assert.equal(environment.get('QL3_RUN_MANAGEMENT_PORT').value, '8448'); + assert.equal(environment.get('QL3_POSTGRES_RUN_MANAGER_POOL_MAX').value, '2'); + assert.equal(container.resources.requests.memory, '96Mi'); + assert.equal(service.spec.type, 'ClusterIP'); + assert.equal(service.spec.ports[0].port, 8448); + assert.equal(policy.spec.ingress[0].from[0].podSelector.matchLabels['qinglong.io/run-management-client'], 'true'); + assert.equal(policy.spec.egress.length, 1); + assert.deepEqual(kustomization.resources, [ + 'service-account.yaml', + 'service.yaml', + 'deployment.yaml', + 'pod-disruption-budget.yaml', + 'network-policy.yaml', + ]); + + for (const defaultPath of [ + 'deploy/kubernetes/ql3-cluster/base/kustomization.yaml', + 'deploy/kubernetes/ql3-cluster/overlays/cloudnative-pg/kustomization.yaml', + ]) { + assert.doesNotMatch(fs.readFileSync(path.join(ROOT, defaultPath), 'utf8'), /run-management/); + } +}); + +test('binds the CloudNativePG overlay only to the dedicated Run manager role', () => { + const directory = 'deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg'; + const patch = yaml.load(fs.readFileSync(path.join(ROOT, directory, 'deployment-patch.yaml'), 'utf8')); + const environment = new Map(patch[0].value.map((entry) => [entry.name, entry])); + const policy = document(`${directory}/network-policy-patch.yaml`); + const manifest = JSON.parse( + fs.readFileSync(path.join(ROOT, 'packages/ql3-cluster-admin/package.json'), 'utf8'), + ); + + assert.equal(environment.has('QL3_POSTGRES_RUN_MANAGER_URL'), false); + assert.equal( + environment.get('QL3_POSTGRES_RUN_MANAGER_USER').valueFrom.secretKeyRef.name, + 'ql3-postgres-run-manager-auth', + ); + assert.equal( + environment.get('QL3_POSTGRES_RUN_MANAGER_PASSWORD').valueFrom.secretKeyRef.name, + 'ql3-postgres-run-manager-auth', + ); + assert.equal(environment.get('QL3_POSTGRES_RUN_MANAGER_HOST').value, 'ql3-postgres-rw.qinglong3-system.svc'); + assert.equal(policy.spec.egress.length, 2); + assert.deepEqual(policy.spec.egress[1].to[0].podSelector.matchLabels, { + 'cnpg.io/cluster': 'ql3-postgres', + }); + assert.equal(manifest.bin['ql3-run-manage'], 'dist/run-management/runManagementCli.js'); + assert.equal(manifest.bin['ql3-run-client'], 'dist/run-management/runManagementClientCli.js'); + assert.equal( + manifest.exports['./run-management-process'].require, + './dist/run-management/runManagementProcess.js', + ); +});