feat(ql3): add strong cluster run management

2026-09-22 19:29:13 +08:00 · 2026-08-12 07:28:43 +08:00
parent e38b143dbb
commit c0ab62e64a
58 changed files with 3087 additions and 105 deletions
@@ -0,0 +1,198 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ql3-run-management
+  namespace: qinglong3-system
+  labels:
+    app.kubernetes.io/name: ql3-run-management
+    app.kubernetes.io/component: run-management
+    app.kubernetes.io/part-of: qinglong3
+spec:
+  replicas: 2
+  minReadySeconds: 10
+  revisionHistoryLimit: 3
+  progressDeadlineSeconds: 600
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ql3-run-management
+      app.kubernetes.io/component: run-management
+  template:
+    metadata:
+      annotations:
+        qinglong.io/run-management-client-ca-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
+        qinglong.io/run-management-client-crl-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
+      labels:
+        app.kubernetes.io/name: ql3-run-management
+        app.kubernetes.io/component: run-management
+        app.kubernetes.io/part-of: qinglong3
+    spec:
+      serviceAccountName: ql3-run-management
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 30
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        runAsGroup: 10001
+        fsGroup: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - topologyKey: kubernetes.io/hostname
+              labelSelector:
+                matchLabels:
+                  app.kubernetes.io/name: ql3-run-management
+                  app.kubernetes.io/component: run-management
+      containers:
+        - name: management
+          image: qinglong3-cluster-admin:3.0.0-alpha.0
+          imagePullPolicy: IfNotPresent
+          command:
+            - node
+            - /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/run-management/runManagementCli.js
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          env:
+            - name: QL3_PROFILE
+              value: cluster-admin
+            - name: QL3_RUN_MANAGEMENT_ENABLED
+              value: 'true'
+            - name: QL3_RUN_MANAGEMENT_HOST
+              value: 0.0.0.0
+            - name: QL3_RUN_MANAGEMENT_PORT
+              value: '8448'
+            - name: QL3_RUN_MANAGEMENT_TLS_CERT_FILE
+              value: /var/run/secrets/qinglong3/run-management-tls/tls.crt
+            - name: QL3_RUN_MANAGEMENT_TLS_KEY_FILE
+              value: /var/run/secrets/qinglong3/run-management-tls/tls.key
+            - name: QL3_RUN_MANAGEMENT_CLIENT_CA_FILE
+              value: /var/run/secrets/qinglong3/run-management-tls/ca.crt
+            - name: QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE
+              value: /var/run/secrets/qinglong3/run-management-tls/client.crl
+            - name: QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE
+              value: /var/run/qinglong3/run-management-identity/keyset.json
+            - name: QL3_RUN_MANAGEMENT_MAX_BODY_BYTES
+              value: '32768'
+            - name: QL3_RUN_MANAGEMENT_MAX_CONNECTIONS
+              value: '32'
+            - name: QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS
+              value: '16'
+            - name: QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS
+              value: '10000'
+            - name: QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS
+              value: '5000'
+            - name: QL3_RUN_MANAGEMENT_RATE_WINDOW_MS
+              value: '60000'
+            - name: QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT
+              value: '30'
+            - name: QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT
+              value: '300'
+            - name: QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS
+              value: '1024'
+            - name: QL3_POSTGRES_RUN_MANAGER_TLS_MODE
+              value: verify-full
+            - name: QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE
+              value: /var/run/secrets/qinglong3/postgres-run-manager/ca.crt
+            - name: QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME
+              value: qinglong3-run-manager
+            - name: QL3_POSTGRES_RUN_MANAGER_POOL_MAX
+              value: '2'
+            - name: QL3_POSTGRES_RUN_MANAGER_URL
+              valueFrom:
+                secretKeyRef:
+                  name: ql3-cluster-run-management-database
+                  key: postgres-run-manager-url
+            - name: QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: ql3-cluster-run-management-database
+                  key: postgres-tls-servername
+          ports:
+            - name: https
+              containerPort: 8448
+              protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /livez
+              port: https
+              scheme: HTTPS
+            periodSeconds: 2
+            timeoutSeconds: 1
+            failureThreshold: 30
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: https
+              scheme: HTTPS
+            periodSeconds: 5
+            timeoutSeconds: 2
+            failureThreshold: 2
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: https
+              scheme: HTTPS
+            periodSeconds: 10
+            timeoutSeconds: 2
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: 50m
+              memory: 96Mi
+            limits:
+              cpu: '1'
+              memory: 384Mi
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+            - name: management-tls
+              mountPath: /var/run/secrets/qinglong3/run-management-tls
+              readOnly: true
+            - name: management-identity
+              mountPath: /var/run/qinglong3/run-management-identity
+              readOnly: true
+            - name: postgres-run-manager-ca
+              mountPath: /var/run/secrets/qinglong3/postgres-run-manager
+              readOnly: true
+      volumes:
+        - name: tmp
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi
+        - name: management-tls
+          secret:
+            secretName: ql3-run-management-tls
+            defaultMode: 288
+            items:
+              - key: tls.crt
+                path: tls.crt
+              - key: tls.key
+                path: tls.key
+              - key: ca.crt
+                path: ca.crt
+              - key: client.crl
+                path: client.crl
+        - name: management-identity
+          secret:
+            secretName: ql3-run-management-identity
+            defaultMode: 292
+            items:
+              - key: keyset.json
+                path: keyset.json
+        - name: postgres-run-manager-ca
+          secret:
+            secretName: ql3-cluster-run-management-database
+            defaultMode: 292
+            items:
+              - key: postgres-ca.crt
+                path: ca.crt
@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - service-account.yaml
+  - service.yaml
+  - deployment.yaml
+  - pod-disruption-budget.yaml
+  - network-policy.yaml
@@ -0,0 +1,38 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ql3-run-management
+  namespace: qinglong3-system
+  labels:
+    app.kubernetes.io/name: ql3-run-management
+    app.kubernetes.io/component: run-management
+    app.kubernetes.io/part-of: qinglong3
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: ql3-run-management
+      app.kubernetes.io/component: run-management
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              qinglong.io/run-management-client: 'true'
+      ports:
+        - protocol: TCP
+          port: 8448
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
@@ -0,0 +1,15 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: ql3-run-management
+  namespace: qinglong3-system
+  labels:
+    app.kubernetes.io/name: ql3-run-management
+    app.kubernetes.io/component: run-management
+    app.kubernetes.io/part-of: qinglong3
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ql3-run-management
+      app.kubernetes.io/component: run-management
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ql3-run-management
+  namespace: qinglong3-system
+  labels:
+    app.kubernetes.io/name: ql3-run-management
+    app.kubernetes.io/component: run-management
+    app.kubernetes.io/part-of: qinglong3
+automountServiceAccountToken: false
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: ql3-run-management
+  namespace: qinglong3-system
+  labels:
+    app.kubernetes.io/name: ql3-run-management
+    app.kubernetes.io/component: run-management
+    app.kubernetes.io/part-of: qinglong3
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: ql3-run-management
+    app.kubernetes.io/component: run-management
+  ports:
+    - name: https
+      port: 8448
+      targetPort: https
+      protocol: TCP
@@ -0,0 +1,71 @@
+- op: replace
+  path: /spec/template/spec/containers/0/env
+  value:
+    - name: QL3_PROFILE
+      value: cluster-admin
+    - name: QL3_RUN_MANAGEMENT_ENABLED
+      value: 'true'
+    - name: QL3_RUN_MANAGEMENT_HOST
+      value: 0.0.0.0
+    - name: QL3_RUN_MANAGEMENT_PORT
+      value: '8448'
+    - name: QL3_RUN_MANAGEMENT_TLS_CERT_FILE
+      value: /var/run/secrets/qinglong3/run-management-tls/tls.crt
+    - name: QL3_RUN_MANAGEMENT_TLS_KEY_FILE
+      value: /var/run/secrets/qinglong3/run-management-tls/tls.key
+    - name: QL3_RUN_MANAGEMENT_CLIENT_CA_FILE
+      value: /var/run/secrets/qinglong3/run-management-tls/ca.crt
+    - name: QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE
+      value: /var/run/secrets/qinglong3/run-management-tls/client.crl
+    - name: QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE
+      value: /var/run/qinglong3/run-management-identity/keyset.json
+    - name: QL3_RUN_MANAGEMENT_MAX_BODY_BYTES
+      value: '32768'
+    - name: QL3_RUN_MANAGEMENT_MAX_CONNECTIONS
+      value: '32'
+    - name: QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS
+      value: '16'
+    - name: QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS
+      value: '10000'
+    - name: QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS
+      value: '5000'
+    - name: QL3_RUN_MANAGEMENT_RATE_WINDOW_MS
+      value: '60000'
+    - name: QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT
+      value: '30'
+    - name: QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT
+      value: '300'
+    - name: QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS
+      value: '1024'
+    - name: QL3_POSTGRES_RUN_MANAGER_TLS_MODE
+      value: verify-full
+    - name: QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE
+      value: /var/run/secrets/qinglong3/postgres-run-manager/ca.crt
+    - name: QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME
+      value: qinglong3-run-manager
+    - name: QL3_POSTGRES_RUN_MANAGER_POOL_MAX
+      value: '2'
+    - name: QL3_POSTGRES_RUN_MANAGER_HOST
+      value: ql3-postgres-rw.qinglong3-system.svc
+    - name: QL3_POSTGRES_RUN_MANAGER_PORT
+      value: '5432'
+    - name: QL3_POSTGRES_RUN_MANAGER_DATABASE
+      value: qinglong
+    - name: QL3_POSTGRES_RUN_MANAGER_USER
+      valueFrom:
+        secretKeyRef:
+          name: ql3-postgres-run-manager-auth
+          key: username
+    - name: QL3_POSTGRES_RUN_MANAGER_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: ql3-postgres-run-manager-auth
+          key: password
+    - name: QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME
+      value: ql3-postgres-rw.qinglong3-system.svc
+- op: replace
+  path: /spec/template/spec/volumes/3/secret/secretName
+  value: ql3-postgres-ca
+- op: replace
+  path: /spec/template/spec/volumes/3/secret/items/0/key
+  value: ca.crt
@@ -0,0 +1,25 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../base
+
+patches:
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      labelSelector: app.kubernetes.io/name=ql3-run-management
+    path: deployment-patch.yaml
+  - target:
+      group: networking.k8s.io
+      version: v1
+      kind: NetworkPolicy
+      name: ql3-run-management
+    path: network-policy-patch.yaml
+
+images:
+  - name: qinglong3-cluster-admin
+    newName: registry.example.com/qinglong/qinglong3-cluster-admin
+    # Fail closed until the independently verified release digest is supplied.
+    digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,26 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ql3-run-management
+  namespace: qinglong3-system
+spec:
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+    - to:
+        - podSelector:
+            matchLabels:
+              cnpg.io/cluster: ql3-postgres
+      ports:
+        - protocol: TCP
+          port: 5432
@@ -0,0 +1,72 @@
+# Example only. Copy into a private overlay and replace every placeholder.
+# This file is intentionally excluded from committed Kustomizations.
+# Assertions require aud=qinglong3-run-management,
+# typ=ql3-run-management+jwt and ql3_purpose=run-management.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ql3-run-management-identity
+  namespace: qinglong3-system
+type: Opaque
+stringData:
+  keyset.json: |
+    {
+      "schemaVersion": 1,
+      "generation": 1,
+      "issuer": "https://identity.example.test/",
+      "audience": "qinglong3-run-management",
+      "keys": [
+        {
+          "alg": "EdDSA",
+          "crv": "Ed25519",
+          "kid": "REPLACE_WITH_KEY_ID",
+          "kty": "OKP",
+          "use": "sig",
+          "x": "REPLACE_WITH_ED25519_PUBLIC_JWK_X"
+        }
+      ],
+      "revokedKids": [],
+      "assuranceMappings": [
+        {
+          "acr": "urn:example:mfa",
+          "assurance": "multi_factor",
+          "requiredAmr": ["pwd", "otp"]
+        },
+        {
+          "acr": "urn:example:hardware",
+          "assurance": "hardware",
+          "requiredAmr": ["hwk"]
+        }
+      ],
+      "constraints": {
+        "maxAssertionBytes": 8192,
+        "maxLifetimeMs": 300000,
+        "maxAuthenticationAgeMs": 300000,
+        "clockSkewMs": 5000
+      }
+    }
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ql3-run-management-tls
+  namespace: qinglong3-system
+type: kubernetes.io/tls
+stringData:
+  tls.crt: REPLACE_WITH_SERVER_CERTIFICATE_CHAIN
+  tls.key: REPLACE_WITH_SERVER_PRIVATE_KEY
+  ca.crt: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_AUTHORITIES
+  client.crl: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_REVOCATION_LISTS
+---
+# Non-CloudNativePG deployments only. The reviewed overlay uses
+# ql3-postgres-run-manager-auth and ql3-postgres-ca instead.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ql3-cluster-run-management-database
+  namespace: qinglong3-system
+type: Opaque
+stringData:
+  postgres-run-manager-url: REPLACE_WITH_RUN_MANAGER_DSN
+  postgres-tls-servername: REPLACE_WITH_POSTGRES_DNS_NAME
+  postgres-ca.crt: REPLACE_WITH_POSTGRES_CA_CERTIFICATE
@@ -122,6 +122,18 @@ stringData:
 ---
 apiVersion: v1
 kind: Secret
+metadata:
+  name: ql3-postgres-run-manager-auth
+  namespace: qinglong3-system
+  labels:
+    cnpg.io/reload: 'true'
+type: kubernetes.io/basic-auth
+stringData:
+  username: ql3_run_manager
+  password: REPLACE_WITH_SECRET_MANAGER_VALUE
+---
+apiVersion: v1
+kind: Secret
 metadata:
  name: ql3-postgres-worker-credential-manager-auth
  namespace: qinglong3-system
@@ -240,6 +240,30 @@ spec:
 ---
 apiVersion: postgresql.cnpg.io/v1
 kind: DatabaseRole
+metadata:
+  name: ql3-postgres-run-manager
+  labels:
+    app.kubernetes.io/name: ql3-postgres
+    app.kubernetes.io/component: database-role
+    app.kubernetes.io/part-of: qinglong3
+spec:
+  cluster:
+    name: ql3-postgres
+  name: ql3_run_manager
+  comment: QingLong strongly authenticated human Run retry authority
+  login: true
+  superuser: false
+  createdb: false
+  createrole: false
+  replication: false
+  bypassrls: false
+  connectionLimit: 4
+  databaseRoleReclaimPolicy: retain
+  passwordSecret:
+    name: ql3-postgres-run-manager-auth
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: DatabaseRole
 metadata:
  name: ql3-postgres-worker-credential-manager
  labels: