mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add strong cluster run management
This commit is contained in:
@@ -0,0 +1,198 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ql3-run-management
|
||||
namespace: qinglong3-system
|
||||
labels:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
app.kubernetes.io/part-of: qinglong3
|
||||
spec:
|
||||
replicas: 2
|
||||
minReadySeconds: 10
|
||||
revisionHistoryLimit: 3
|
||||
progressDeadlineSeconds: 600
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 0
|
||||
maxSurge: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
qinglong.io/run-management-client-ca-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
|
||||
qinglong.io/run-management-client-crl-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
|
||||
labels:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
app.kubernetes.io/part-of: qinglong3
|
||||
spec:
|
||||
serviceAccountName: ql3-run-management
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 30
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- topologyKey: kubernetes.io/hostname
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
containers:
|
||||
- name: management
|
||||
image: qinglong3-cluster-admin:3.0.0-alpha.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- node
|
||||
- /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/run-management/runManagementCli.js
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
env:
|
||||
- name: QL3_PROFILE
|
||||
value: cluster-admin
|
||||
- name: QL3_RUN_MANAGEMENT_ENABLED
|
||||
value: 'true'
|
||||
- name: QL3_RUN_MANAGEMENT_HOST
|
||||
value: 0.0.0.0
|
||||
- name: QL3_RUN_MANAGEMENT_PORT
|
||||
value: '8448'
|
||||
- name: QL3_RUN_MANAGEMENT_TLS_CERT_FILE
|
||||
value: /var/run/secrets/qinglong3/run-management-tls/tls.crt
|
||||
- name: QL3_RUN_MANAGEMENT_TLS_KEY_FILE
|
||||
value: /var/run/secrets/qinglong3/run-management-tls/tls.key
|
||||
- name: QL3_RUN_MANAGEMENT_CLIENT_CA_FILE
|
||||
value: /var/run/secrets/qinglong3/run-management-tls/ca.crt
|
||||
- name: QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE
|
||||
value: /var/run/secrets/qinglong3/run-management-tls/client.crl
|
||||
- name: QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE
|
||||
value: /var/run/qinglong3/run-management-identity/keyset.json
|
||||
- name: QL3_RUN_MANAGEMENT_MAX_BODY_BYTES
|
||||
value: '32768'
|
||||
- name: QL3_RUN_MANAGEMENT_MAX_CONNECTIONS
|
||||
value: '32'
|
||||
- name: QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS
|
||||
value: '16'
|
||||
- name: QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS
|
||||
value: '10000'
|
||||
- name: QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS
|
||||
value: '5000'
|
||||
- name: QL3_RUN_MANAGEMENT_RATE_WINDOW_MS
|
||||
value: '60000'
|
||||
- name: QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT
|
||||
value: '30'
|
||||
- name: QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT
|
||||
value: '300'
|
||||
- name: QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS
|
||||
value: '1024'
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_TLS_MODE
|
||||
value: verify-full
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE
|
||||
value: /var/run/secrets/qinglong3/postgres-run-manager/ca.crt
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME
|
||||
value: qinglong3-run-manager
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_POOL_MAX
|
||||
value: '2'
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ql3-cluster-run-management-database
|
||||
key: postgres-run-manager-url
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ql3-cluster-run-management-database
|
||||
key: postgres-tls-servername
|
||||
ports:
|
||||
- name: https
|
||||
containerPort: 8448
|
||||
protocol: TCP
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /livez
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
periodSeconds: 2
|
||||
timeoutSeconds: 1
|
||||
failureThreshold: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 2
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /livez
|
||||
port: https
|
||||
scheme: HTTPS
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 96Mi
|
||||
limits:
|
||||
cpu: '1'
|
||||
memory: 384Mi
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
- name: management-tls
|
||||
mountPath: /var/run/secrets/qinglong3/run-management-tls
|
||||
readOnly: true
|
||||
- name: management-identity
|
||||
mountPath: /var/run/qinglong3/run-management-identity
|
||||
readOnly: true
|
||||
- name: postgres-run-manager-ca
|
||||
mountPath: /var/run/secrets/qinglong3/postgres-run-manager
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
- name: management-tls
|
||||
secret:
|
||||
secretName: ql3-run-management-tls
|
||||
defaultMode: 288
|
||||
items:
|
||||
- key: tls.crt
|
||||
path: tls.crt
|
||||
- key: tls.key
|
||||
path: tls.key
|
||||
- key: ca.crt
|
||||
path: ca.crt
|
||||
- key: client.crl
|
||||
path: client.crl
|
||||
- name: management-identity
|
||||
secret:
|
||||
secretName: ql3-run-management-identity
|
||||
defaultMode: 292
|
||||
items:
|
||||
- key: keyset.json
|
||||
path: keyset.json
|
||||
- name: postgres-run-manager-ca
|
||||
secret:
|
||||
secretName: ql3-cluster-run-management-database
|
||||
defaultMode: 292
|
||||
items:
|
||||
- key: postgres-ca.crt
|
||||
path: ca.crt
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- service-account.yaml
|
||||
- service.yaml
|
||||
- deployment.yaml
|
||||
- pod-disruption-budget.yaml
|
||||
- network-policy.yaml
|
||||
@@ -0,0 +1,38 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: ql3-run-management
|
||||
namespace: qinglong3-system
|
||||
labels:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
app.kubernetes.io/part-of: qinglong3
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
qinglong.io/run-management-client: 'true'
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8448
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: ql3-run-management
|
||||
namespace: qinglong3-system
|
||||
labels:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
app.kubernetes.io/part-of: qinglong3
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: ql3-run-management
|
||||
namespace: qinglong3-system
|
||||
labels:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
app.kubernetes.io/part-of: qinglong3
|
||||
automountServiceAccountToken: false
|
||||
@@ -0,0 +1,19 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ql3-run-management
|
||||
namespace: qinglong3-system
|
||||
labels:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
app.kubernetes.io/part-of: qinglong3
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: ql3-run-management
|
||||
app.kubernetes.io/component: run-management
|
||||
ports:
|
||||
- name: https
|
||||
port: 8448
|
||||
targetPort: https
|
||||
protocol: TCP
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
- op: replace
|
||||
path: /spec/template/spec/containers/0/env
|
||||
value:
|
||||
- name: QL3_PROFILE
|
||||
value: cluster-admin
|
||||
- name: QL3_RUN_MANAGEMENT_ENABLED
|
||||
value: 'true'
|
||||
- name: QL3_RUN_MANAGEMENT_HOST
|
||||
value: 0.0.0.0
|
||||
- name: QL3_RUN_MANAGEMENT_PORT
|
||||
value: '8448'
|
||||
- name: QL3_RUN_MANAGEMENT_TLS_CERT_FILE
|
||||
value: /var/run/secrets/qinglong3/run-management-tls/tls.crt
|
||||
- name: QL3_RUN_MANAGEMENT_TLS_KEY_FILE
|
||||
value: /var/run/secrets/qinglong3/run-management-tls/tls.key
|
||||
- name: QL3_RUN_MANAGEMENT_CLIENT_CA_FILE
|
||||
value: /var/run/secrets/qinglong3/run-management-tls/ca.crt
|
||||
- name: QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE
|
||||
value: /var/run/secrets/qinglong3/run-management-tls/client.crl
|
||||
- name: QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE
|
||||
value: /var/run/qinglong3/run-management-identity/keyset.json
|
||||
- name: QL3_RUN_MANAGEMENT_MAX_BODY_BYTES
|
||||
value: '32768'
|
||||
- name: QL3_RUN_MANAGEMENT_MAX_CONNECTIONS
|
||||
value: '32'
|
||||
- name: QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS
|
||||
value: '16'
|
||||
- name: QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS
|
||||
value: '10000'
|
||||
- name: QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS
|
||||
value: '5000'
|
||||
- name: QL3_RUN_MANAGEMENT_RATE_WINDOW_MS
|
||||
value: '60000'
|
||||
- name: QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT
|
||||
value: '30'
|
||||
- name: QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT
|
||||
value: '300'
|
||||
- name: QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS
|
||||
value: '1024'
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_TLS_MODE
|
||||
value: verify-full
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE
|
||||
value: /var/run/secrets/qinglong3/postgres-run-manager/ca.crt
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME
|
||||
value: qinglong3-run-manager
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_POOL_MAX
|
||||
value: '2'
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_HOST
|
||||
value: ql3-postgres-rw.qinglong3-system.svc
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_PORT
|
||||
value: '5432'
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_DATABASE
|
||||
value: qinglong
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ql3-postgres-run-manager-auth
|
||||
key: username
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ql3-postgres-run-manager-auth
|
||||
key: password
|
||||
- name: QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME
|
||||
value: ql3-postgres-rw.qinglong3-system.svc
|
||||
- op: replace
|
||||
path: /spec/template/spec/volumes/3/secret/secretName
|
||||
value: ql3-postgres-ca
|
||||
- op: replace
|
||||
path: /spec/template/spec/volumes/3/secret/items/0/key
|
||||
value: ca.crt
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- ../base
|
||||
|
||||
patches:
|
||||
- target:
|
||||
group: apps
|
||||
version: v1
|
||||
kind: Deployment
|
||||
labelSelector: app.kubernetes.io/name=ql3-run-management
|
||||
path: deployment-patch.yaml
|
||||
- target:
|
||||
group: networking.k8s.io
|
||||
version: v1
|
||||
kind: NetworkPolicy
|
||||
name: ql3-run-management
|
||||
path: network-policy-patch.yaml
|
||||
|
||||
images:
|
||||
- name: qinglong3-cluster-admin
|
||||
newName: registry.example.com/qinglong/qinglong3-cluster-admin
|
||||
# Fail closed until the independently verified release digest is supplied.
|
||||
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: ql3-run-management
|
||||
namespace: qinglong3-system
|
||||
spec:
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: 53
|
||||
- protocol: TCP
|
||||
port: 53
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
cnpg.io/cluster: ql3-postgres
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5432
|
||||
@@ -0,0 +1,72 @@
|
||||
# Example only. Copy into a private overlay and replace every placeholder.
|
||||
# This file is intentionally excluded from committed Kustomizations.
|
||||
# Assertions require aud=qinglong3-run-management,
|
||||
# typ=ql3-run-management+jwt and ql3_purpose=run-management.
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ql3-run-management-identity
|
||||
namespace: qinglong3-system
|
||||
type: Opaque
|
||||
stringData:
|
||||
keyset.json: |
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generation": 1,
|
||||
"issuer": "https://identity.example.test/",
|
||||
"audience": "qinglong3-run-management",
|
||||
"keys": [
|
||||
{
|
||||
"alg": "EdDSA",
|
||||
"crv": "Ed25519",
|
||||
"kid": "REPLACE_WITH_KEY_ID",
|
||||
"kty": "OKP",
|
||||
"use": "sig",
|
||||
"x": "REPLACE_WITH_ED25519_PUBLIC_JWK_X"
|
||||
}
|
||||
],
|
||||
"revokedKids": [],
|
||||
"assuranceMappings": [
|
||||
{
|
||||
"acr": "urn:example:mfa",
|
||||
"assurance": "multi_factor",
|
||||
"requiredAmr": ["pwd", "otp"]
|
||||
},
|
||||
{
|
||||
"acr": "urn:example:hardware",
|
||||
"assurance": "hardware",
|
||||
"requiredAmr": ["hwk"]
|
||||
}
|
||||
],
|
||||
"constraints": {
|
||||
"maxAssertionBytes": 8192,
|
||||
"maxLifetimeMs": 300000,
|
||||
"maxAuthenticationAgeMs": 300000,
|
||||
"clockSkewMs": 5000
|
||||
}
|
||||
}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ql3-run-management-tls
|
||||
namespace: qinglong3-system
|
||||
type: kubernetes.io/tls
|
||||
stringData:
|
||||
tls.crt: REPLACE_WITH_SERVER_CERTIFICATE_CHAIN
|
||||
tls.key: REPLACE_WITH_SERVER_PRIVATE_KEY
|
||||
ca.crt: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_AUTHORITIES
|
||||
client.crl: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_REVOCATION_LISTS
|
||||
---
|
||||
# Non-CloudNativePG deployments only. The reviewed overlay uses
|
||||
# ql3-postgres-run-manager-auth and ql3-postgres-ca instead.
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ql3-cluster-run-management-database
|
||||
namespace: qinglong3-system
|
||||
type: Opaque
|
||||
stringData:
|
||||
postgres-run-manager-url: REPLACE_WITH_RUN_MANAGER_DSN
|
||||
postgres-tls-servername: REPLACE_WITH_POSTGRES_DNS_NAME
|
||||
postgres-ca.crt: REPLACE_WITH_POSTGRES_CA_CERTIFICATE
|
||||
@@ -122,6 +122,18 @@ stringData:
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ql3-postgres-run-manager-auth
|
||||
namespace: qinglong3-system
|
||||
labels:
|
||||
cnpg.io/reload: 'true'
|
||||
type: kubernetes.io/basic-auth
|
||||
stringData:
|
||||
username: ql3_run_manager
|
||||
password: REPLACE_WITH_SECRET_MANAGER_VALUE
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ql3-postgres-worker-credential-manager-auth
|
||||
namespace: qinglong3-system
|
||||
|
||||
@@ -240,6 +240,30 @@ spec:
|
||||
---
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: DatabaseRole
|
||||
metadata:
|
||||
name: ql3-postgres-run-manager
|
||||
labels:
|
||||
app.kubernetes.io/name: ql3-postgres
|
||||
app.kubernetes.io/component: database-role
|
||||
app.kubernetes.io/part-of: qinglong3
|
||||
spec:
|
||||
cluster:
|
||||
name: ql3-postgres
|
||||
name: ql3_run_manager
|
||||
comment: QingLong strongly authenticated human Run retry authority
|
||||
login: true
|
||||
superuser: false
|
||||
createdb: false
|
||||
createrole: false
|
||||
replication: false
|
||||
bypassrls: false
|
||||
connectionLimit: 4
|
||||
databaseRoleReclaimPolicy: retain
|
||||
passwordSecret:
|
||||
name: ql3-postgres-run-manager-auth
|
||||
---
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: DatabaseRole
|
||||
metadata:
|
||||
name: ql3-postgres-worker-credential-manager
|
||||
labels:
|
||||
|
||||
@@ -11,6 +11,22 @@
|
||||
|
||||
最新增量证据(2026-08-12):
|
||||
|
||||
- D-295/ADR-0383(已接受)
|
||||
Cluster 手动 `run.retry` 已从“仅有 PostgreSQL authority”推进到独立强认证产品面:能力内聚在既有 `@qinglong/cluster-admin`
|
||||
的 `run-management/` 目录,不新增 workspace package;只有 `QL3_PROFILE=cluster-admin` 且显式启用时才读取 mTLS/CRL、
|
||||
purpose-bound OIDC keyset 和创建 PostgreSQL Pool/HTTPS listener。固定 route `/api/v3/runs/management` 同时要求 mTLS 与
|
||||
`aud=qinglong3-run-management`、`typ=ql3-run-management+jwt`、`ql3_purpose=run-management` 的五分钟内
|
||||
`multi_factor|hardware` User,服务端生成新 Run/Attempt/Event identity;keyset generation 使用独立 durable
|
||||
`run-management` authority。PostgreSQL capability v55 / migration `pg-0056-run-management-boundary` 引入专用
|
||||
`ql3_run_manager` 与最小 `SECURITY DEFINER` Project/RoleBinding lock function,产品进程不再使用宽泛的常驻 runtime role;精确
|
||||
replay 改由 immutable created/queued Events 恢复最初创建事实,因此调度进展后仍返回原 durable identity。Kubernetes 部署作为
|
||||
`operations/run-management` opt-in overlay 提供两副本/PDB/反亲和/私网 egress,默认 Edge、Standalone 与 Cluster base 均不引用,
|
||||
低配设备继续保持零新增进程、连接、timer、listener、watcher、cache 或 sidecar。完整 18-package clean build/test 退出 0;
|
||||
backend 1,165 pass/2 conditional skip/0 fail;workspace 保持 18 package/1,070 source/1,052 nested,且无 single-source
|
||||
或 shallow package。PostgreSQL migration ledger 直属文件仍受 58 hard cap 约束,v55 migration 已归入既有 `run-management`
|
||||
领域目录而非放宽阈值。真实 PostgreSQL 18.4 arm64 physical HA 以两个 `ql3_run_manager` Pool 验证 exact concurrent replay、
|
||||
最后 quota slot、独立 keyset ledger、同步 WAL 与 promotion 后事实,共 119 gates、timeline `1→2`;报告 SHA-256
|
||||
`6ca8ccfb48841589e10c6484f5c97ce72e24b123f3abb1066a639e63718e64c6`,离线证据审计无 finding。
|
||||
- D-294/ADR-0382(已接受)
|
||||
Cluster 已补齐共享 `qinglong/run-manual-retry@v1` 的 PostgreSQL 原子 authority,但在可信强认证 transport 完成前保持产品 route
|
||||
关闭。adapter 只接受五分钟内的 `multi_factor|hardware` User,在单个 `SERIALIZABLE` 事务中取得 Project 行锁,重验
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
- 关联 RFC:QL-RFC-0001 D-294
|
||||
- 前置决策:ADR-0039、ADR-0047、ADR-0119、ADR-0361、ADR-0381
|
||||
|
||||
> 产品装配说明:ADR-0383 已 supersede 本文“复用 `ql3_runtime` 且不新增 role/migration”的决策。本文的共享 retry 语义、原子 repository 与 HA 证据继续有效;生产入口改由强认证的独立 Run Management Plane 和 `ql3_run_manager` 承载。
|
||||
|
||||
## 上下文
|
||||
|
||||
ADR-0381 已冻结手动 retry 的共享语义并完成 Local SQLite/CLI 纵向切片,但 Cluster 尚缺少可在多副本下工作的 PostgreSQL authority。不能把 Local 的进程内状态或单连接假设复制到 Cluster,也不能让每个 `cluster-control` 副本各自维护限流 bucket。
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
# ADR-0383:强认证的 Cluster Run Management Plane
|
||||
|
||||
- 状态:Accepted
|
||||
- 日期:2026-08-12
|
||||
- 关联 RFC:QL-RFC-0001 D-295
|
||||
- 前置决策:ADR-0039、ADR-0056、ADR-0356、ADR-0364、ADR-0366、ADR-0381、ADR-0382
|
||||
- Supersedes:ADR-0382 中“复用 `ql3_runtime` 且不新增 role/migration”的产品装配决策;ADR-0382 的共享语义与 PostgreSQL 原子事务仍有效
|
||||
|
||||
## 上下文
|
||||
|
||||
ADR-0382 已证明 PostgreSQL 手动 Run retry 的原子语义与 HA 收敛,但现有 Cluster Control bearer 只能建立 `single_factor` User,不能承载会再次执行外部副作用的人工恢复操作。直接把 repository 接进通用 Cluster HTTP 会混合普通控制面与强人类认证 authority;继续使用 `ql3_runtime` 又会使常驻 runtime 持有本不需要的人工恢复权限。
|
||||
|
||||
QingLong 同时服务低资源路由设备与多节点集群。新能力不能让 Edge/Standalone 增加进程、连接、timer 或 Cluster 依赖,也不应为了一个内聚领域再制造单文件 workspace package。
|
||||
|
||||
## 决策
|
||||
|
||||
### 1. 能力归属既有 Cluster Admin package
|
||||
|
||||
Run management 作为 `@qinglong/cluster-admin` 内的 `run-management/` 领域目录发布 service、transport、HTTPS process、client 与两个 CLI,不新增 workspace package。PostgreSQL adapter 仍归属 `@qinglong/cluster-postgres/run-manager`。根目录只保留协议装配入口,领域源码不得重新平铺。
|
||||
|
||||
### 2. 独立、显式启用的强认证进程
|
||||
|
||||
管理端只在 `QL3_PROFILE=cluster-admin` 且 `QL3_RUN_MANAGEMENT_ENABLED=true` 时创建资源;关闭时不读取证书、keyset 或数据库配置,也不创建 Pool、listener、timer、watcher 或 cache。服务使用独立 HTTPS 端口 `8448` 和固定 route `/api/v3/runs/management`,同时要求:
|
||||
|
||||
1. 受信 client CA 与 CRL 校验的 mTLS;
|
||||
2. purpose-bound OIDC assertion:`aud=qinglong3-run-management`、`typ=ql3-run-management+jwt`、`ql3_purpose=run-management`;
|
||||
3. 五分钟内的 `multi_factor|hardware` User;
|
||||
4. 精确的 `qinglong/run-manual-retry@v1` command envelope,不接受调用方指定新 Run/Attempt/Event identity;
|
||||
5. route admission 与数据库事务内分别重验 identity/Policy/audit fence。
|
||||
|
||||
Assertion keyset generation 使用 durable PostgreSQL ledger 的独立 `run-management` authority,不能与 Plugin、Worker、automation 或 Approval keyset 互换。client 是 caller-driven 的一次性命令,不引入常驻 agent。
|
||||
|
||||
### 3. 专用最小权限数据库角色
|
||||
|
||||
Migration `pg-0056-run-management-boundary` / capability v55 引入 `ql3_run_manager`。该角色只读 Project/RoleBinding/Task/execution revision,只对 Runs、Attempts、Events、security audit 和本 authority 的 identity ledger 取得精确所需权限;它没有 migration、admin、Worker、AI、Approval 或任意表 DELETE 权限,也没有更新既有 Run 的权限。
|
||||
|
||||
`SECURITY DEFINER` 函数 `ql3.lock_run_management_policy_fence` 只负责锁定 Project 并验证 active owner/admin/operator binding,固定 `search_path`,并仅向 `ql3_runtime` 与 `ql3_run_manager` 授予 EXECUTE。repository 不以 `FOR UPDATE` 读取源/重放 Run,因此不因 PostgreSQL 锁语法扩大 table UPDATE privilege。
|
||||
|
||||
精确 replay 从 immutable created/queued Events 恢复最初创建事实,而不是依赖被重试 Run 当前仍为 queued;调度进展后相同 mutation 仍返回原始 durable identity。
|
||||
|
||||
### 4. 部署按 Profile 付费
|
||||
|
||||
默认 Edge、Standalone 及 Cluster base overlay 都不引用 Run management manifests。需要该能力的集群显式应用 `operations/run-management`;CloudNativePG overlay 使用专用 Secret、primary DNS 与最多两个数据库连接。生产模板为两副本、PDB、反亲和、只读根文件系统、无 ServiceAccount token、受限 ingress 与只到 DNS/PostgreSQL 的 egress。低配单机不承担这些资源。
|
||||
|
||||
## 验收
|
||||
|
||||
- service/transport/process/client/HTTPS 的单元与回环测试覆盖强认证、purpose isolation、错误映射、限流、drain 与 exact replay;
|
||||
- PostgreSQL package 全量测试覆盖 migration checksum、schema/readiness、role privilege 与调度后 replay;
|
||||
- CloudNativePG 静态审计必须固定十四个角色和非秘密 placeholder;Run management deployment 必须证明 opt-in、mTLS、专用 role 与私网 egress;
|
||||
- 真实 PostgreSQL 18.4 HA 必须以迁移后角色完成 readiness、事务写入、同步复制与 promotion 后事实核验;
|
||||
- 完整 workspace package/backend/dependency/package/Profile artifact 门全部通过,GitNexus staged scope 只包含本阶段预期 flow,才允许提交。
|
||||
|
||||
## 被否决的替代方案
|
||||
|
||||
1. **接入通用 Cluster Control bearer**:认证强度不足且会扩大普通控制面的恢复执行 authority。
|
||||
2. **继续让 `ql3_runtime` 作为产品管理角色**:常驻 runtime 无需持有人工 retry 与身份 keyset 写权限。
|
||||
3. **新增 `@qinglong/run-management` package**:没有独立依赖、制品或部署生命周期收益,会重新产生单文件/浅 package。
|
||||
4. **每个 Cluster Control Pod 内置管理 listener**:副本与普通 runtime 同扩缩,增加攻击面和低配常驻资源。
|
||||
5. **允许调用方提交新 aggregate identity**:会扩大 replay/conflict 表面并削弱 server-authored fact。
|
||||
|
||||
## 验收证据(2026-08-12)
|
||||
|
||||
- `@qinglong/cluster-postgres`:309 pass、1 个外部数据库条件 skip、0 fail;`@qinglong/cluster-admin`:281 pass、2 个外部集成条件 skip、0 fail;`@qinglong/cluster-control`:230 pass、2 skip、0 fail;
|
||||
- 完整 18-package clean build/test 全部退出 0;backend 1,167 项中 1,165 pass、2 个环境条件 skip、0 fail;
|
||||
- workspace 保持 18 个 package,源码为 1,070 个(1,052 个 nested);`singleSourcePackages=[]`、`shallowSourcePackages=[]`。PostgreSQL ordered migration ledger 直属文件仍为 58,v55 migration 归入既有 `run-management` 领域目录,没有放宽 dense-directory cap;
|
||||
- CloudNativePG 静态审计固定 14 个最小权限角色;Run Management deployment 静态门证明默认 overlay 不引用、mTLS/OIDC authority 私有、Pool 上限 2、无 ServiceAccount token、只读根和仅 DNS/PostgreSQL egress;
|
||||
- PostgreSQL 18.4 arm64 physical HA:119 gates、timeline `1→2`,两个独立 `ql3_run_manager` Pool 完成 exact concurrent replay 与最后 quota slot 竞争,`run-management` keyset ledger 经双连接、重启、commit-response-loss 验证;报告 SHA-256 `6ca8ccfb48841589e10c6484f5c97ce72e24b123f3abb1066a639e63718e64c6`,离线审计 `compatible:true`。
|
||||
|
||||
## 影响
|
||||
|
||||
- PostgreSQL schema contract 从 v54 升至 v55,生产部署在启用管理面前必须先创建 `ql3_run_manager` 并运行 migration;
|
||||
- Cluster Admin image 增加两个 opt-in binary,但默认 Profile 不启动它们;
|
||||
- ADR-0382 的原子 repository 和既有 HA 证据继续成立,产品装配改由本 ADR 的强认证进程与专用角色承载;
|
||||
- UI 仍需以同一 transport 展示 source/new Run linkage、终态原因和 retry preview,不得绕过该 authority。
|
||||
@@ -386,6 +386,7 @@
|
||||
| [ADR-0380](./ADR-0380-local-lost-run-retry-single-control-cadence.md) | Local lost Run retry 复用单一 execution-control cadence | Accepted |
|
||||
| [ADR-0381](./ADR-0381-strong-local-manual-run-retry.md) | 强认证的 Local 手动 Run retry | Accepted |
|
||||
| [ADR-0382](./ADR-0382-postgresql-cluster-manual-run-retry-authority.md) | PostgreSQL Cluster 手动 Run retry 原子 Authority | Accepted |
|
||||
| [ADR-0383](./ADR-0383-strong-cluster-run-management-plane.md) | 强认证的 Cluster Run Management Plane 与专用数据库角色 | Accepted |
|
||||
|
||||
## 规则
|
||||
|
||||
|
||||
@@ -20,6 +20,31 @@
|
||||
"require": "./dist/approval-management/approvalDecisionManagement.js",
|
||||
"default": "./dist/approval-management/approvalDecisionManagement.js"
|
||||
},
|
||||
"./run-management": {
|
||||
"types": "./dist/run-management/runManagement.d.ts",
|
||||
"require": "./dist/run-management/runManagement.js",
|
||||
"default": "./dist/run-management/runManagement.js"
|
||||
},
|
||||
"./run-management-transport": {
|
||||
"types": "./dist/run-management/runManagementTransport.d.ts",
|
||||
"require": "./dist/run-management/runManagementTransport.js",
|
||||
"default": "./dist/run-management/runManagementTransport.js"
|
||||
},
|
||||
"./run-management-http": {
|
||||
"types": "./dist/run-management/runManagementHttp.d.ts",
|
||||
"require": "./dist/run-management/runManagementHttp.js",
|
||||
"default": "./dist/run-management/runManagementHttp.js"
|
||||
},
|
||||
"./run-management-process": {
|
||||
"types": "./dist/run-management/runManagementProcess.d.ts",
|
||||
"require": "./dist/run-management/runManagementProcess.js",
|
||||
"default": "./dist/run-management/runManagementProcess.js"
|
||||
},
|
||||
"./run-management-client": {
|
||||
"types": "./dist/run-management/runManagementClient.d.ts",
|
||||
"require": "./dist/run-management/runManagementClient.js",
|
||||
"default": "./dist/run-management/runManagementClient.js"
|
||||
},
|
||||
"./approval-management": {
|
||||
"types": "./dist/approval-management/approvalManagement.d.ts",
|
||||
"require": "./dist/approval-management/approvalManagement.js",
|
||||
@@ -331,6 +356,8 @@
|
||||
"ql3-worker-credential-client": "dist/worker-credential/workerCredentialManagementClientCli.js",
|
||||
"ql3-approval-manage": "dist/approval-management/approvalManagementCli.js",
|
||||
"ql3-approval-client": "dist/approval-management/approvalManagementClientCli.js",
|
||||
"ql3-run-manage": "dist/run-management/runManagementCli.js",
|
||||
"ql3-run-client": "dist/run-management/runManagementClientCli.js",
|
||||
"ql3-automation-manage": "dist/automation-management/automationManagementCli.js",
|
||||
"ql3-automation-client": "dist/automation-management/automationManagementClientCli.js",
|
||||
"ql3-provider-credential-manage": "dist/model-provider-credential/modelProviderCredentialManagementCli.js",
|
||||
|
||||
@@ -80,6 +80,11 @@ export const CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PRO
|
||||
purpose: 'model-provider-credential-management',
|
||||
});
|
||||
|
||||
export const CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE = Object.freeze({
|
||||
type: 'ql3-run-management+jwt',
|
||||
purpose: 'run-management',
|
||||
});
|
||||
|
||||
export interface ClusterPluginPackageIdentityAssertionVerifierOptions {
|
||||
readonly issuer: string;
|
||||
readonly audience: string;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
CLUSTER_AUTOMATION_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
CLUSTER_APPROVAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
createClusterPluginPackageIdentityAssertionVerifier,
|
||||
type ClusterManagementIdentityAssertionProfile,
|
||||
type ClusterPluginPackageIdentityAssertionAuthentication,
|
||||
@@ -507,3 +508,12 @@ export function createClusterModelProviderCredentialIdentityKeysetFile(
|
||||
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterRunIdentityKeysetFile(
|
||||
options: ClusterWorkerCredentialIdentityKeysetFileOptions,
|
||||
): Readonly<ClusterPluginPackageIdentityKeysetFile> {
|
||||
return createClusterPluginPackageIdentityKeysetFile({
|
||||
...options,
|
||||
assertionProfile: CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -67,6 +67,19 @@ import {
|
||||
ClusterModelProviderCredentialManagementTransportRequestError,
|
||||
ClusterModelProviderCredentialManagementTransportUnavailableError,
|
||||
} from '../model-provider-credential/modelProviderCredentialManagementTransport';
|
||||
import {
|
||||
ClusterRunManagementAuthorizationError,
|
||||
ClusterRunManagementConflictError,
|
||||
ClusterRunManagementRateLimitedError,
|
||||
ClusterRunManagementRequestError,
|
||||
ClusterRunManagementTargetUnavailableError,
|
||||
ClusterRunManagementUnavailableError,
|
||||
} from '../run-management/runManagement';
|
||||
import {
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
ClusterRunManagementTransportRequestError,
|
||||
ClusterRunManagementTransportUnavailableError,
|
||||
} from '../run-management/runManagementTransport';
|
||||
|
||||
export const CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH =
|
||||
'/api/v3/plugin-packages/management';
|
||||
@@ -77,18 +90,21 @@ export const CLUSTER_AUTOMATION_MANAGEMENT_PATH =
|
||||
export const CLUSTER_APPROVAL_MANAGEMENT_PATH = '/api/v3/approvals/management';
|
||||
export const CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH =
|
||||
'/api/v3/provider-credentials/management';
|
||||
export const CLUSTER_RUN_MANAGEMENT_PATH = '/api/v3/runs/management';
|
||||
export type ClusterAuthenticatedManagementPath =
|
||||
| typeof CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_AUTOMATION_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_APPROVAL_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH;
|
||||
| typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH
|
||||
| typeof CLUSTER_RUN_MANAGEMENT_PATH;
|
||||
const MANAGEMENT_PATHS = new Set<ClusterAuthenticatedManagementPath>([
|
||||
CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH,
|
||||
CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH,
|
||||
CLUSTER_AUTOMATION_MANAGEMENT_PATH,
|
||||
CLUSTER_APPROVAL_MANAGEMENT_PATH,
|
||||
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH,
|
||||
CLUSTER_RUN_MANAGEMENT_PATH,
|
||||
]);
|
||||
const DEFAULT_MAX_BODY_BYTES = 64 * 1024;
|
||||
const DEFAULT_MAX_CONNECTIONS = 64;
|
||||
@@ -531,7 +547,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof ClusterApprovalManagementTransportAuthenticationError ||
|
||||
error instanceof
|
||||
ClusterModelProviderCredentialManagementTransportAuthenticationError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthenticationError
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthenticationError ||
|
||||
error instanceof ClusterRunManagementTransportAuthenticationError
|
||||
) {
|
||||
return new HttpRequestError(401, 'authentication_required');
|
||||
}
|
||||
@@ -544,6 +561,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof
|
||||
ClusterModelProviderCredentialManagementTransportRequestError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementRequestError ||
|
||||
error instanceof ClusterRunManagementTransportRequestError ||
|
||||
error instanceof ClusterRunManagementRequestError ||
|
||||
error instanceof PluginPackageManagementRequestError ||
|
||||
error instanceof WorkerCredentialManagementRequestError
|
||||
) {
|
||||
@@ -554,7 +573,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof WorkerCredentialManagementAuthorizationError ||
|
||||
error instanceof ClusterAutomationManagementAuthorizationError ||
|
||||
error instanceof ClusterApprovalManagementTransportAuthorizationError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthorizationError
|
||||
error instanceof ClusterModelProviderCredentialManagementAuthorizationError ||
|
||||
error instanceof ClusterRunManagementAuthorizationError
|
||||
) {
|
||||
return new HttpRequestError(403, 'forbidden');
|
||||
}
|
||||
@@ -563,7 +583,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof WorkerCredentialManagementConflictError ||
|
||||
error instanceof ClusterAutomationManagementConflictError ||
|
||||
error instanceof ClusterApprovalManagementTransportConflictError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementConflictError
|
||||
error instanceof ClusterModelProviderCredentialManagementConflictError ||
|
||||
error instanceof ClusterRunManagementConflictError
|
||||
) {
|
||||
return new HttpRequestError(409, 'conflict');
|
||||
}
|
||||
@@ -578,8 +599,12 @@ function responseError(error: unknown): HttpRequestError {
|
||||
) {
|
||||
return new HttpRequestError(429, 'quota_exceeded', error.retryAfterMs);
|
||||
}
|
||||
if (error instanceof ClusterRunManagementRateLimitedError) {
|
||||
return new HttpRequestError(429, 'rate_limited', error.retryAfterMs);
|
||||
}
|
||||
if (
|
||||
error instanceof ClusterApprovalManagementTransportTargetUnavailableError
|
||||
error instanceof ClusterApprovalManagementTransportTargetUnavailableError ||
|
||||
error instanceof ClusterRunManagementTargetUnavailableError
|
||||
) {
|
||||
return new HttpRequestError(404, 'not_found');
|
||||
}
|
||||
@@ -594,6 +619,8 @@ function responseError(error: unknown): HttpRequestError {
|
||||
error instanceof
|
||||
ClusterModelProviderCredentialManagementTransportUnavailableError ||
|
||||
error instanceof ClusterModelProviderCredentialManagementUnavailableError ||
|
||||
error instanceof ClusterRunManagementTransportUnavailableError ||
|
||||
error instanceof ClusterRunManagementUnavailableError ||
|
||||
error instanceof PluginPackageManagementUnavailableError ||
|
||||
error instanceof WorkerCredentialManagementUnavailableError
|
||||
) {
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import {
|
||||
PostgresProjectPolicyRepository,
|
||||
PostgresRunManualRetryRepository,
|
||||
PostgresSecurityAuditRepository,
|
||||
} from '@qinglong/cluster-postgres/run-manager';
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
|
||||
import {
|
||||
InvalidRunManualRetryError,
|
||||
RunManualRetryFenceRejectedError,
|
||||
RunManualRetryNotFoundError,
|
||||
RunManualRetryRateLimitedError,
|
||||
RunManualRetryUnavailableError,
|
||||
type RunManualRetryResult,
|
||||
type RunManualRetrySourceStatus,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPolicyFence,
|
||||
type SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import { normalizeSecurityAuditRecord } from '@qinglong/runtime-core/security-audit';
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
||||
export interface ClusterRunManagementRetryRequest {
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly mutationId: string;
|
||||
readonly expectedRunVersion: number;
|
||||
readonly expectedRunStatus: RunManualRetrySourceStatus;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly failureAuditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementService {
|
||||
retry(
|
||||
request: Readonly<ClusterRunManagementRetryRequest>,
|
||||
): Promise<Readonly<RunManualRetryResult>>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementOptions {
|
||||
readonly pool: PostgresPool;
|
||||
readonly now?: () => number;
|
||||
readonly randomUuid?: () => string;
|
||||
}
|
||||
|
||||
export class ClusterRunManagementConfigurationError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFIGURATION_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management configuration is invalid');
|
||||
this.name = 'ClusterRunManagementConfigurationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementRequestError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_REQUEST_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management request is invalid');
|
||||
this.name = 'ClusterRunManagementRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementAuthorizationError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_FORBIDDEN';
|
||||
constructor() {
|
||||
super('Cluster Run management is forbidden');
|
||||
this.name = 'ClusterRunManagementAuthorizationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTargetUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TARGET_UNAVAILABLE';
|
||||
constructor() {
|
||||
super('Cluster Run management target is unavailable');
|
||||
this.name = 'ClusterRunManagementTargetUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementConflictError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFLICT';
|
||||
constructor() {
|
||||
super('Cluster Run management conflicts with durable state');
|
||||
this.name = 'ClusterRunManagementConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementRateLimitedError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_RATE_LIMITED';
|
||||
constructor(readonly retryAfterMs: number) {
|
||||
super('Cluster Run management rate limit is exhausted');
|
||||
this.name = 'ClusterRunManagementRateLimitedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_UNAVAILABLE';
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Cluster Run management is unavailable', options);
|
||||
this.name = 'ClusterRunManagementUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function exactRequest(
|
||||
value: unknown,
|
||||
): asserts value is Readonly<ClusterRunManagementRetryRequest> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).sort().join('\0') !==
|
||||
[
|
||||
'auditEventId',
|
||||
'expectedRunStatus',
|
||||
'expectedRunVersion',
|
||||
'failureAuditEventId',
|
||||
'mutationId',
|
||||
'principal',
|
||||
'projectId',
|
||||
'requestId',
|
||||
'sourceRunId',
|
||||
]
|
||||
.sort()
|
||||
.join('\0')
|
||||
) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
}
|
||||
|
||||
function validUuid(value: unknown): value is string {
|
||||
return typeof value === 'string' && UUID_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function failureReason(error: unknown): string {
|
||||
if (error instanceof ClusterRunManagementAuthorizationError) {
|
||||
return 'authorization_rejected';
|
||||
}
|
||||
if (error instanceof RunManualRetryNotFoundError) return 'run_not_found';
|
||||
if (error instanceof RunManualRetryRateLimitedError) return 'rate_limited';
|
||||
if (error instanceof RunManualRetryFenceRejectedError) return error.reason;
|
||||
return 'management_unavailable';
|
||||
}
|
||||
|
||||
/** Strong OIDC Run management composition over one run-manager Pool. */
|
||||
export function createClusterRunManagementService(
|
||||
options: ClusterRunManagementOptions,
|
||||
): Readonly<ClusterRunManagementService> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some(
|
||||
(key) => key !== 'pool' && key !== 'now' && key !== 'randomUuid',
|
||||
) ||
|
||||
!options.pool ||
|
||||
typeof options.pool.query !== 'function' ||
|
||||
typeof options.pool.connect !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||
(options.randomUuid !== undefined && typeof options.randomUuid !== 'function')
|
||||
) {
|
||||
throw new ClusterRunManagementConfigurationError();
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
const createId = options.randomUuid ?? randomUUID;
|
||||
const policy = new ProjectPolicyEngine(
|
||||
new PostgresProjectPolicyRepository(options.pool),
|
||||
);
|
||||
const retries = new PostgresRunManualRetryRepository(options.pool);
|
||||
const audit = new PostgresSecurityAuditRepository(options.pool);
|
||||
|
||||
return Object.freeze({
|
||||
async retry(requestValue: Readonly<ClusterRunManagementRetryRequest>) {
|
||||
exactRequest(requestValue);
|
||||
const observedAtMs = now();
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
if (
|
||||
!Number.isSafeInteger(observedAtMs) ||
|
||||
observedAtMs < 0 ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.projectId) ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.sourceRunId) ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.requestId) ||
|
||||
!validUuid(requestValue.mutationId) ||
|
||||
!validUuid(requestValue.auditEventId) ||
|
||||
!validUuid(requestValue.failureAuditEventId) ||
|
||||
requestValue.auditEventId === requestValue.failureAuditEventId
|
||||
) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(
|
||||
requestValue.principal,
|
||||
observedAtMs,
|
||||
);
|
||||
} catch {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
|
||||
let fence: Readonly<SecurityPolicyFence> | null = null;
|
||||
try {
|
||||
const decision = await policy.authorize(
|
||||
principal,
|
||||
requestValue.projectId,
|
||||
'run.retry',
|
||||
);
|
||||
fence = decision.fence;
|
||||
if (
|
||||
decision.effect !== 'allow' ||
|
||||
!fence ||
|
||||
fence.bindingVersion === null
|
||||
) {
|
||||
throw new ClusterRunManagementAuthorizationError();
|
||||
}
|
||||
return await retries.retryRun({
|
||||
projectId: requestValue.projectId,
|
||||
sourceRunId: requestValue.sourceRunId,
|
||||
mutationId: requestValue.mutationId,
|
||||
expectedRunVersion: requestValue.expectedRunVersion,
|
||||
expectedRunStatus: requestValue.expectedRunStatus,
|
||||
runId: createId(),
|
||||
attemptId: createId(),
|
||||
createdEventId: createId(),
|
||||
queuedEventId: createId(),
|
||||
auditEventId: requestValue.auditEventId,
|
||||
requestId: requestValue.requestId,
|
||||
principal,
|
||||
policyFence: fence,
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await audit.record(
|
||||
normalizeSecurityAuditRecord({
|
||||
eventId: requestValue.failureAuditEventId,
|
||||
requestId: requestValue.requestId,
|
||||
operationId: 'run.retry',
|
||||
projectId: requestValue.projectId,
|
||||
subject: principal.subject,
|
||||
authenticationId: principal.authenticationId,
|
||||
outcome: 'denied',
|
||||
reasons: [failureReason(error)],
|
||||
fence,
|
||||
occurredAtMs: observedAtMs,
|
||||
}),
|
||||
);
|
||||
} catch (auditError) {
|
||||
throw new ClusterRunManagementUnavailableError({ cause: auditError });
|
||||
}
|
||||
if (error instanceof ClusterRunManagementAuthorizationError) throw error;
|
||||
if (error instanceof InvalidRunManualRetryError) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
if (error instanceof RunManualRetryNotFoundError) {
|
||||
throw new ClusterRunManagementTargetUnavailableError();
|
||||
}
|
||||
if (error instanceof RunManualRetryFenceRejectedError) {
|
||||
throw new ClusterRunManagementConflictError();
|
||||
}
|
||||
if (error instanceof RunManualRetryRateLimitedError) {
|
||||
throw new ClusterRunManagementRateLimitedError(error.retryAfterMs);
|
||||
}
|
||||
if (error instanceof RunManualRetryUnavailableError) {
|
||||
throw new ClusterRunManagementUnavailableError({ cause: error });
|
||||
}
|
||||
throw new ClusterRunManagementUnavailableError({ cause: error });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
startClusterRunManagementProcess,
|
||||
type ClusterRunManagementProcessRuntime,
|
||||
} from './runManagementProcess';
|
||||
|
||||
const USAGE = 'Usage: ql3-run-manage';
|
||||
|
||||
function fact(error: unknown): Readonly<Record<string, unknown>> {
|
||||
const candidate = error as { readonly name?: unknown; readonly code?: unknown };
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-run-management',
|
||||
event: 'management_failed',
|
||||
name: typeof candidate?.name === 'string' ? candidate.name : 'Error',
|
||||
...(typeof candidate?.code === 'string' ? { code: candidate.code } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function emit(value: Readonly<Record<string, unknown>>): void {
|
||||
process.stdout.write(`${JSON.stringify(value)}\n`);
|
||||
}
|
||||
|
||||
async function run(argv: readonly string[]): Promise<void> {
|
||||
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
|
||||
process.stdout.write(`${USAGE}\n`);
|
||||
return;
|
||||
}
|
||||
if (argv.length !== 0) {
|
||||
process.stderr.write(`${JSON.stringify({ code: 'QL3_RUN_MANAGEMENT_CLI_USAGE_INVALID', message: USAGE })}\n`);
|
||||
process.exitCode = 64;
|
||||
return;
|
||||
}
|
||||
let runtime: Readonly<ClusterRunManagementProcessRuntime>;
|
||||
try {
|
||||
runtime = await startClusterRunManagementProcess({
|
||||
environment: process.env,
|
||||
onError: () => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_unavailable' }),
|
||||
});
|
||||
} catch (error) {
|
||||
process.stderr.write(`${JSON.stringify(fact(error))}\n`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (runtime.status === 'disabled') {
|
||||
emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_disabled' });
|
||||
return;
|
||||
}
|
||||
emit({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-run-management',
|
||||
event: 'management_started',
|
||||
address: runtime.address,
|
||||
identityGeneration: runtime.identity.generation,
|
||||
databaseContractVersion: runtime.database.contractVersion,
|
||||
databaseMigrationCount: runtime.database.migrationIds.length,
|
||||
});
|
||||
let stopping: Promise<void> | undefined;
|
||||
const stop = (): Promise<void> => {
|
||||
stopping ??= runtime.close().then(() => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_stopped' }));
|
||||
return stopping;
|
||||
};
|
||||
for (const signal of ['SIGINT', 'SIGTERM'] as const) {
|
||||
process.once(signal, () => {
|
||||
void stop().then(
|
||||
() => { process.exitCode = 0; },
|
||||
(error) => { process.stderr.write(`${JSON.stringify(fact(error))}\n`); process.exitCode = 1; },
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void run(process.argv.slice(2));
|
||||
@@ -0,0 +1,106 @@
|
||||
import {
|
||||
RUN_MANUAL_RETRY_SCHEMA,
|
||||
normalizeRunManualRetryResult,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
executeClusterAuthenticatedManagementClient,
|
||||
type ClusterAuthenticatedManagementClientResult,
|
||||
type ClusterPluginPackageManagementClientConnectionOptions,
|
||||
type ClusterPluginPackageManagementClientPaths,
|
||||
} from '../management-support/pluginPackageManagementClient';
|
||||
import {
|
||||
normalizeClusterRunManagementCommand,
|
||||
type ClusterRunManagementCommand,
|
||||
type ClusterRunManagementTransportResult,
|
||||
} from './runManagementTransport';
|
||||
|
||||
const MANAGEMENT_PATH = '/api/v3/runs/management';
|
||||
|
||||
export type ClusterRunManagementClientPaths =
|
||||
ClusterPluginPackageManagementClientPaths;
|
||||
export type ClusterRunManagementClientConnectionOptions =
|
||||
ClusterPluginPackageManagementClientConnectionOptions;
|
||||
export type ClusterRunManagementClientResult =
|
||||
ClusterAuthenticatedManagementClientResult<ClusterRunManagementTransportResult>;
|
||||
|
||||
function invalid(): never {
|
||||
throw new ClusterPluginPackageManagementClientRequestError();
|
||||
}
|
||||
|
||||
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
|
||||
const actual = Object.keys(value as object).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function validateClusterRunManagementClientResult(
|
||||
value: unknown,
|
||||
command: Readonly<ClusterRunManagementCommand>,
|
||||
): Readonly<ClusterRunManagementTransportResult> {
|
||||
const envelope = exact(value, ['schemaVersion', 'operation', 'retry']);
|
||||
if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
|
||||
const retry = exact(envelope.retry, [
|
||||
'schema',
|
||||
'status',
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'sourceRunStatus',
|
||||
'sourceRunVersion',
|
||||
'runId',
|
||||
'retryOfRunId',
|
||||
'taskId',
|
||||
'taskRevision',
|
||||
'attemptId',
|
||||
'runStatus',
|
||||
'runVersion',
|
||||
'eventSequence',
|
||||
'executorType',
|
||||
'executionRevisionDigest',
|
||||
'createdAtMs',
|
||||
]);
|
||||
if (retry.schema !== RUN_MANUAL_RETRY_SCHEMA) invalid();
|
||||
try {
|
||||
const { schema: _schema, ...result } = retry;
|
||||
const normalized = normalizeRunManualRetryResult(result as never);
|
||||
if (
|
||||
normalized.projectId !== command.request.projectId ||
|
||||
normalized.sourceRunId !== command.request.sourceRunId ||
|
||||
normalized.sourceRunVersion !== command.request.body.expectedRunVersion ||
|
||||
normalized.sourceRunStatus !== command.request.body.expectedRunStatus ||
|
||||
normalized.executorType !== 'remote_worker'
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
} catch {
|
||||
invalid();
|
||||
}
|
||||
return Object.freeze(
|
||||
envelope as unknown as ClusterRunManagementTransportResult,
|
||||
);
|
||||
}
|
||||
|
||||
const PROTOCOL = Object.freeze({
|
||||
managementPath: MANAGEMENT_PATH,
|
||||
clientCertificate: 'required' as const,
|
||||
normalizeCommand: normalizeClusterRunManagementCommand,
|
||||
validateResult: validateClusterRunManagementClientResult,
|
||||
});
|
||||
|
||||
export function executeClusterRunManagementClient(
|
||||
paths: ClusterRunManagementClientPaths,
|
||||
connectionOptions?: ClusterRunManagementClientConnectionOptions,
|
||||
): Promise<Readonly<ClusterRunManagementClientResult>> {
|
||||
return executeClusterAuthenticatedManagementClient(
|
||||
paths,
|
||||
PROTOCOL,
|
||||
connectionOptions,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { ClusterPluginPackageManagementClientRemoteError } from '../management-support/pluginPackageManagementClient';
|
||||
import { executeClusterRunManagementClient } from './runManagementClient';
|
||||
|
||||
const USAGE =
|
||||
'Usage: ql3-run-client --config=/absolute/client.json --command=/absolute/command.json --assertion=/absolute/assertion.jwt';
|
||||
|
||||
function argumentsFrom(argv: readonly string[]): Readonly<{
|
||||
configFile: string;
|
||||
commandFile: string;
|
||||
assertionFile: string;
|
||||
}> | null {
|
||||
if (argv.length !== 3) return null;
|
||||
const values = new Map<string, string>();
|
||||
for (const argument of argv) {
|
||||
const match = /^--(config|command|assertion)=(\/.+)$/.exec(argument);
|
||||
if (!match || values.has(match[1]!)) return null;
|
||||
values.set(match[1]!, match[2]!);
|
||||
}
|
||||
if (!values.has('config') || !values.has('command') || !values.has('assertion')) return null;
|
||||
return Object.freeze({
|
||||
configFile: values.get('config')!,
|
||||
commandFile: values.get('command')!,
|
||||
assertionFile: values.get('assertion')!,
|
||||
});
|
||||
}
|
||||
|
||||
function failureFact(error: unknown): Readonly<Record<string, unknown>> {
|
||||
const candidate = error as { readonly code?: unknown };
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-run-management-client',
|
||||
event: 'command_failed',
|
||||
code: typeof candidate?.code === 'string' ? candidate.code : 'QL3_RUN_MANAGEMENT_CLIENT_FAILED',
|
||||
...(error instanceof ClusterPluginPackageManagementClientRemoteError
|
||||
? {
|
||||
statusCode: error.statusCode,
|
||||
responseCode: error.responseCode,
|
||||
requestId: error.requestId,
|
||||
...(error.retryAfterSeconds === null ? {} : { retryAfterSeconds: error.retryAfterSeconds }),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function run(argv: readonly string[]): Promise<void> {
|
||||
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
|
||||
process.stdout.write(`${USAGE}\n`);
|
||||
return;
|
||||
}
|
||||
const paths = argumentsFrom(argv);
|
||||
if (!paths) {
|
||||
process.stderr.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'usage_invalid', code: 'QL3_RUN_MANAGEMENT_CLIENT_USAGE_INVALID' })}\n`);
|
||||
process.exitCode = 64;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await executeClusterRunManagementClient(paths);
|
||||
process.stdout.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'command_completed', requestId: result.requestId, result: result.result })}\n`);
|
||||
} catch (error) {
|
||||
process.stderr.write(`${JSON.stringify(failureFact(error))}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
void run(process.argv.slice(2));
|
||||
@@ -0,0 +1,24 @@
|
||||
import {
|
||||
CLUSTER_RUN_MANAGEMENT_PATH,
|
||||
startClusterPluginPackageManagementHttp,
|
||||
type ClusterPluginPackageManagementHttpApplication,
|
||||
type StartClusterPluginPackageManagementHttpOptions,
|
||||
} from '../management-support/pluginPackageManagementHttp';
|
||||
|
||||
export type ClusterRunManagementHttpApplication =
|
||||
ClusterPluginPackageManagementHttpApplication;
|
||||
|
||||
export type StartClusterRunManagementHttpOptions = Omit<
|
||||
StartClusterPluginPackageManagementHttpOptions,
|
||||
'managementPath'
|
||||
>;
|
||||
|
||||
/** Starts the shared bounded OIDC/mTLS HTTPS adapter on the Run-only path. */
|
||||
export function startClusterRunManagementHttp(
|
||||
options: StartClusterRunManagementHttpOptions,
|
||||
): Promise<Readonly<ClusterRunManagementHttpApplication>> {
|
||||
return startClusterPluginPackageManagementHttp({
|
||||
...options,
|
||||
managementPath: CLUSTER_RUN_MANAGEMENT_PATH,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,467 @@
|
||||
import type {
|
||||
OpenPostgresDatabase,
|
||||
PostgresDatabaseResource,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
PostgresRunManagementIdentityKeysetLedgerRepository,
|
||||
assertPostgresRunManagerSchemaReady,
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresTlsDnsServername,
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
loadPostgresConnectionEnvironment,
|
||||
type PostgresConnectionOptions,
|
||||
type PostgresPoolOptions,
|
||||
type PostgresSchemaReadinessReport,
|
||||
} from '@qinglong/cluster-postgres/run-manager';
|
||||
import {
|
||||
absoluteManagementEnvironmentFile,
|
||||
booleanManagementEnvironmentValue,
|
||||
boundedManagementEnvironmentValue,
|
||||
integerManagementEnvironmentValue,
|
||||
readManagementTlsFile,
|
||||
} from '../management-support/managementProcessSupport';
|
||||
import {
|
||||
createClusterRunIdentityKeysetFile,
|
||||
type ClusterPluginPackageIdentityKeysetFile,
|
||||
type ClusterPluginPackageIdentityKeysetSnapshot,
|
||||
} from '../management-support/pluginPackageIdentityKeyset';
|
||||
import { validateClusterManagementClientTrust } from '../worker-credential/management-server/workerCredentialManagementMutualTls';
|
||||
import { createClusterRunManagementService } from './runManagement';
|
||||
import {
|
||||
startClusterRunManagementHttp,
|
||||
type ClusterRunManagementHttpApplication,
|
||||
type StartClusterRunManagementHttpOptions,
|
||||
} from './runManagementHttp';
|
||||
import { createClusterRunManagementTransport } from './runManagementTransport';
|
||||
|
||||
const SAFE_HOST = /^[A-Za-z0-9][A-Za-z0-9.:-]{0,254}$/;
|
||||
const SAFE_APPLICATION_NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,62}$/;
|
||||
|
||||
export type ClusterRunManagementProcessEnvironment = Readonly<
|
||||
Record<string, string | undefined>
|
||||
>;
|
||||
|
||||
export type ClusterRunManagementProcessConfig =
|
||||
| Readonly<{ enabled: false }>
|
||||
| Readonly<{
|
||||
enabled: true;
|
||||
profile: 'cluster-admin';
|
||||
host: string;
|
||||
port: number;
|
||||
certificateFile: string;
|
||||
privateKeyFile: string;
|
||||
clientCertificateAuthorityFile: string;
|
||||
clientCertificateRevocationListFile: string;
|
||||
identityKeysetFile: string;
|
||||
http: Readonly<{
|
||||
maxBodyBytes: number;
|
||||
maxConnections: number;
|
||||
maxConcurrentRequests: number;
|
||||
requestTimeoutMs: number;
|
||||
drainTimeoutMs: number;
|
||||
rateWindowMs: number;
|
||||
peerRequestLimit: number;
|
||||
globalRequestLimit: number;
|
||||
maxRateLimitPeers: number;
|
||||
}>;
|
||||
database: Readonly<{
|
||||
connection: PostgresConnectionOptions;
|
||||
pool: PostgresPoolOptions;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export type ClusterRunManagementProcessRuntime =
|
||||
| Readonly<{ status: 'disabled'; close(): Promise<void> }>
|
||||
| Readonly<{
|
||||
status: 'active';
|
||||
address: Readonly<{ host: string; port: number }>;
|
||||
database: PostgresSchemaReadinessReport;
|
||||
identity: ClusterPluginPackageIdentityKeysetSnapshot;
|
||||
availabilityStatus(): 'ready' | 'unavailable' | 'stopped';
|
||||
close(): Promise<void>;
|
||||
}>;
|
||||
|
||||
export interface StartClusterRunManagementProcessOptions {
|
||||
readonly environment: ClusterRunManagementProcessEnvironment;
|
||||
readonly openDatabase?: OpenPostgresDatabase;
|
||||
readonly identities?: ClusterPluginPackageIdentityKeysetFile;
|
||||
readonly assertReady?: (
|
||||
pool: PostgresDatabaseResource['pool'],
|
||||
) => Promise<PostgresSchemaReadinessReport>;
|
||||
readonly startHttp?: (
|
||||
options: StartClusterRunManagementHttpOptions,
|
||||
) => Promise<Readonly<ClusterRunManagementHttpApplication>>;
|
||||
readonly now?: () => number;
|
||||
readonly randomUuid?: () => string;
|
||||
readonly onError?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
export class ClusterRunManagementProcessConfigError extends TypeError {
|
||||
readonly code = 'QL3_RUN_MANAGEMENT_PROCESS_CONFIG_INVALID';
|
||||
constructor(message: string) {
|
||||
super(`Run management process configuration is invalid: ${message}`);
|
||||
this.name = 'ClusterRunManagementProcessConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
function failure(message: string): ClusterRunManagementProcessConfigError {
|
||||
return new ClusterRunManagementProcessConfigError(message);
|
||||
}
|
||||
|
||||
function bounded(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
maximumLength: number,
|
||||
required = false,
|
||||
): string | undefined {
|
||||
return boundedManagementEnvironmentValue(
|
||||
environment,
|
||||
name,
|
||||
maximumLength,
|
||||
failure,
|
||||
required,
|
||||
);
|
||||
}
|
||||
|
||||
function bool(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
): boolean {
|
||||
return booleanManagementEnvironmentValue(environment, name, failure);
|
||||
}
|
||||
|
||||
function integer(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
fallback: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
return integerManagementEnvironmentValue(
|
||||
environment,
|
||||
name,
|
||||
fallback,
|
||||
minimum,
|
||||
maximum,
|
||||
failure,
|
||||
);
|
||||
}
|
||||
|
||||
function absolute(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
name: string,
|
||||
): string {
|
||||
return absoluteManagementEnvironmentFile(environment, name, failure);
|
||||
}
|
||||
|
||||
function loadDatabase(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
): Readonly<{
|
||||
connection: PostgresConnectionOptions;
|
||||
pool: PostgresPoolOptions;
|
||||
}> {
|
||||
let connection: PostgresConnectionOptions;
|
||||
try {
|
||||
connection = loadPostgresConnectionEnvironment(environment, {
|
||||
connectionString: 'QL3_POSTGRES_RUN_MANAGER_URL',
|
||||
host: 'QL3_POSTGRES_RUN_MANAGER_HOST',
|
||||
port: 'QL3_POSTGRES_RUN_MANAGER_PORT',
|
||||
database: 'QL3_POSTGRES_RUN_MANAGER_DATABASE',
|
||||
user: 'QL3_POSTGRES_RUN_MANAGER_USER',
|
||||
password: 'QL3_POSTGRES_RUN_MANAGER_PASSWORD',
|
||||
});
|
||||
} catch (error) {
|
||||
throw failure(
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: 'PostgreSQL run manager connection is invalid',
|
||||
);
|
||||
}
|
||||
const mode = environment.QL3_POSTGRES_RUN_MANAGER_TLS_MODE ?? 'verify-full';
|
||||
if (mode !== 'verify-full' && mode !== 'disable') {
|
||||
throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_MODE must be verify-full or disable');
|
||||
}
|
||||
if (
|
||||
mode === 'disable' &&
|
||||
!bool(environment, 'QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE')
|
||||
) {
|
||||
throw failure(
|
||||
'disabling run manager PostgreSQL TLS requires QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE=true',
|
||||
);
|
||||
}
|
||||
const servername = bounded(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME',
|
||||
253,
|
||||
);
|
||||
if (mode === 'verify-full' && !isPostgresTlsDnsServername(servername)) {
|
||||
throw failure(
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME must be an explicit DNS name',
|
||||
);
|
||||
}
|
||||
const caFile = bounded(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE',
|
||||
4_096,
|
||||
);
|
||||
if (mode === 'disable' && caFile !== undefined) {
|
||||
throw failure(
|
||||
'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE cannot be used when TLS is disabled',
|
||||
);
|
||||
}
|
||||
let ca: string | undefined;
|
||||
if (caFile !== undefined) {
|
||||
try {
|
||||
ca = loadPostgresCertificateAuthorityFile(caFile);
|
||||
} catch {
|
||||
throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE is invalid');
|
||||
}
|
||||
}
|
||||
const applicationName =
|
||||
bounded(environment, 'QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME', 63) ??
|
||||
'qinglong3-run-manager';
|
||||
if (!SAFE_APPLICATION_NAME.test(applicationName)) {
|
||||
throw failure('QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
connection: Object.freeze({
|
||||
...connection,
|
||||
tls:
|
||||
mode === 'disable'
|
||||
? Object.freeze({ mode: 'disable' as const })
|
||||
: Object.freeze({
|
||||
mode: 'verify-full' as const,
|
||||
servername: servername!,
|
||||
...(ca === undefined ? {} : { ca }),
|
||||
}),
|
||||
}),
|
||||
pool: Object.freeze({
|
||||
applicationName,
|
||||
maxConnections: integer(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_POOL_MAX',
|
||||
2,
|
||||
1,
|
||||
4,
|
||||
),
|
||||
idleTimeoutMs: integer(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_IDLE_TIMEOUT_MS',
|
||||
10_000,
|
||||
1_000,
|
||||
60_000,
|
||||
),
|
||||
connectionTimeoutMs: integer(
|
||||
environment,
|
||||
'QL3_POSTGRES_RUN_MANAGER_CONNECTION_TIMEOUT_MS',
|
||||
5_000,
|
||||
100,
|
||||
60_000,
|
||||
),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function loadClusterRunManagementProcessConfig(
|
||||
environment: ClusterRunManagementProcessEnvironment,
|
||||
): Readonly<ClusterRunManagementProcessConfig> {
|
||||
if (!environment || typeof environment !== 'object') {
|
||||
throw failure('environment is invalid');
|
||||
}
|
||||
if (!bool(environment, 'QL3_RUN_MANAGEMENT_ENABLED')) {
|
||||
return Object.freeze({ enabled: false as const });
|
||||
}
|
||||
if (environment.QL3_PROFILE !== 'cluster-admin') {
|
||||
throw failure('QL3_PROFILE must be cluster-admin when Run management is enabled');
|
||||
}
|
||||
const host = bounded(environment, 'QL3_RUN_MANAGEMENT_HOST', 255) ?? '0.0.0.0';
|
||||
if (!SAFE_HOST.test(host)) throw failure('QL3_RUN_MANAGEMENT_HOST is invalid');
|
||||
const http = Object.freeze({
|
||||
maxBodyBytes: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_BODY_BYTES', 32 * 1024, 1_024, 256 * 1024),
|
||||
maxConnections: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONNECTIONS', 32, 1, 512),
|
||||
maxConcurrentRequests: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS', 16, 1, 256),
|
||||
requestTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS', 10_000, 1_000, 60_000),
|
||||
drainTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS', 5_000, 100, 60_000),
|
||||
rateWindowMs: integer(environment, 'QL3_RUN_MANAGEMENT_RATE_WINDOW_MS', 60_000, 1_000, 5 * 60_000),
|
||||
peerRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT', 30, 1, 10_000),
|
||||
globalRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT', 300, 1, 100_000),
|
||||
maxRateLimitPeers: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS', 1_024, 1, 16_384),
|
||||
});
|
||||
if (http.globalRequestLimit < http.peerRequestLimit) {
|
||||
throw failure('global request limit cannot be below peer request limit');
|
||||
}
|
||||
return Object.freeze({
|
||||
enabled: true as const,
|
||||
profile: 'cluster-admin' as const,
|
||||
host,
|
||||
port: integer(environment, 'QL3_RUN_MANAGEMENT_PORT', 8_448, 1, 65_535),
|
||||
certificateFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_CERT_FILE'),
|
||||
privateKeyFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_KEY_FILE'),
|
||||
clientCertificateAuthorityFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CA_FILE'),
|
||||
clientCertificateRevocationListFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE'),
|
||||
identityKeysetFile: absolute(environment, 'QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE'),
|
||||
http,
|
||||
database: loadDatabase(environment),
|
||||
});
|
||||
}
|
||||
|
||||
export async function startClusterRunManagementProcess(
|
||||
options: StartClusterRunManagementProcessOptions,
|
||||
): Promise<Readonly<ClusterRunManagementProcessRuntime>> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some(
|
||||
(key) =>
|
||||
![
|
||||
'environment',
|
||||
'openDatabase',
|
||||
'identities',
|
||||
'assertReady',
|
||||
'startHttp',
|
||||
'now',
|
||||
'randomUuid',
|
||||
'onError',
|
||||
].includes(key),
|
||||
) ||
|
||||
!options.environment ||
|
||||
typeof options.environment !== 'object' ||
|
||||
(options.openDatabase !== undefined && typeof options.openDatabase !== 'function') ||
|
||||
(options.identities !== undefined &&
|
||||
(typeof options.identities.reload !== 'function' ||
|
||||
typeof options.identities.bind !== 'function')) ||
|
||||
(options.assertReady !== undefined && typeof options.assertReady !== 'function') ||
|
||||
(options.startHttp !== undefined && typeof options.startHttp !== 'function') ||
|
||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||
(options.randomUuid !== undefined && typeof options.randomUuid !== 'function') ||
|
||||
(options.onError !== undefined && typeof options.onError !== 'function')
|
||||
) {
|
||||
throw failure('options are invalid');
|
||||
}
|
||||
const config = loadClusterRunManagementProcessConfig(options.environment);
|
||||
if (!config.enabled) {
|
||||
return Object.freeze({ status: 'disabled' as const, close: () => Promise.resolve() });
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
let http: Readonly<ClusterRunManagementHttpApplication> | undefined;
|
||||
let database: PostgresDatabaseResource | undefined;
|
||||
let unavailableError: unknown;
|
||||
let closePromise: Promise<void> | undefined;
|
||||
const report = (error: unknown): void => {
|
||||
try {
|
||||
options.onError?.(error);
|
||||
} catch {
|
||||
// Diagnostics never own availability or cleanup.
|
||||
}
|
||||
};
|
||||
const openDatabase =
|
||||
options.openDatabase ??
|
||||
createPostgresDatabaseOpener({
|
||||
role: 'run-manager',
|
||||
connection: config.database.connection,
|
||||
pool: config.database.pool,
|
||||
onPoolError(error) {
|
||||
const first = unavailableError === undefined;
|
||||
unavailableError ??= error;
|
||||
http?.withdraw(error);
|
||||
if (first) report(error);
|
||||
},
|
||||
});
|
||||
try {
|
||||
database = await openDatabase();
|
||||
const evidence = await (options.assertReady ?? assertPostgresRunManagerSchemaReady)(database.pool);
|
||||
if (unavailableError !== undefined) throw unavailableError;
|
||||
const identities =
|
||||
options.identities ??
|
||||
createClusterRunIdentityKeysetFile({
|
||||
filePath: config.identityKeysetFile,
|
||||
now,
|
||||
ledger: new PostgresRunManagementIdentityKeysetLedgerRepository(
|
||||
database.pool,
|
||||
'run-management',
|
||||
),
|
||||
});
|
||||
const identity = await identities.reload();
|
||||
const service = createClusterRunManagementService({
|
||||
pool: database.pool,
|
||||
now,
|
||||
...(options.randomUuid === undefined ? {} : { randomUuid: options.randomUuid }),
|
||||
});
|
||||
const transport = createClusterRunManagementTransport({ service, now });
|
||||
const privateKey = readManagementTlsFile(config.privateKeyFile, true, failure);
|
||||
try {
|
||||
const certificate = readManagementTlsFile(config.certificateFile, false, failure);
|
||||
const clientCertificateAuthority = readManagementTlsFile(
|
||||
config.clientCertificateAuthorityFile,
|
||||
false,
|
||||
failure,
|
||||
);
|
||||
const clientCertificateRevocationList = readManagementTlsFile(
|
||||
config.clientCertificateRevocationListFile,
|
||||
false,
|
||||
failure,
|
||||
);
|
||||
validateClusterManagementClientTrust(
|
||||
clientCertificateAuthority,
|
||||
clientCertificateRevocationList,
|
||||
now(),
|
||||
failure,
|
||||
);
|
||||
http = await (options.startHttp ?? startClusterRunManagementHttp)({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
tls: {
|
||||
privateKey,
|
||||
certificate,
|
||||
clientCertificateAuthority,
|
||||
clientCertificateRevocationList,
|
||||
},
|
||||
transport,
|
||||
identities,
|
||||
limits: config.http,
|
||||
now,
|
||||
onError: report,
|
||||
});
|
||||
} finally {
|
||||
privateKey.fill(0);
|
||||
}
|
||||
if (unavailableError !== undefined) http.withdraw(unavailableError);
|
||||
return Object.freeze({
|
||||
status: 'active' as const,
|
||||
address: http.address,
|
||||
database: evidence,
|
||||
identity,
|
||||
availabilityStatus: () => http?.availabilityStatus() ?? 'stopped',
|
||||
close(): Promise<void> {
|
||||
closePromise ??= (async () => {
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
await http?.close();
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
await database?.close();
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
})();
|
||||
return closePromise;
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await http?.close();
|
||||
} catch {
|
||||
// Preserve startup failure.
|
||||
}
|
||||
try {
|
||||
await database?.close();
|
||||
} catch {
|
||||
// Preserve startup failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
import {
|
||||
createRunManualRetryResponseBody,
|
||||
parseRunManualRetryRequestBody,
|
||||
type RunManualRetryResponseBody,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import type { ClusterRunManagementService } from './runManagement';
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
|
||||
|
||||
export type ClusterRunManagementCommand = Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'run.retry';
|
||||
request: Readonly<{
|
||||
projectId: string;
|
||||
sourceRunId: string;
|
||||
requestId: string;
|
||||
auditEventId: string;
|
||||
failureAuditEventId: string;
|
||||
body: Readonly<{
|
||||
schema: 'qinglong/run-manual-retry@v1';
|
||||
mutationId: string;
|
||||
expectedRunVersion: number;
|
||||
expectedRunStatus: 'failed' | 'cancelled' | 'timed_out';
|
||||
}>;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export type ClusterRunManagementTransportResult = Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'run.retry';
|
||||
retry: Readonly<RunManualRetryResponseBody>;
|
||||
}>;
|
||||
|
||||
export interface ClusterRunManagementAuthentication {
|
||||
authenticate(): Promise<Readonly<SecurityPrincipal> | null>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementTransport {
|
||||
execute(
|
||||
command: unknown,
|
||||
authentication: ClusterRunManagementAuthentication,
|
||||
): Promise<Readonly<ClusterRunManagementTransportResult>>;
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportConfigurationError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_CONFIGURATION_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management transport configuration is invalid');
|
||||
this.name = 'ClusterRunManagementTransportConfigurationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportRequestError extends TypeError {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_REQUEST_INVALID';
|
||||
constructor() {
|
||||
super('Cluster Run management transport request is invalid');
|
||||
this.name = 'ClusterRunManagementTransportRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportAuthenticationError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_AUTHENTICATION_REQUIRED';
|
||||
constructor() {
|
||||
super('Cluster Run management transport requires a strong User principal');
|
||||
this.name = 'ClusterRunManagementTransportAuthenticationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRunManagementTransportUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_UNAVAILABLE';
|
||||
constructor() {
|
||||
super('Cluster Run management transport is unavailable');
|
||||
this.name = 'ClusterRunManagementTransportUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function invalid(): never {
|
||||
throw new ClusterRunManagementTransportRequestError();
|
||||
}
|
||||
|
||||
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
|
||||
const actual = Object.keys(value as object).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function identifier(value: unknown): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) invalid();
|
||||
return value;
|
||||
}
|
||||
|
||||
function uuid(value: unknown): string {
|
||||
if (typeof value !== 'string' || !UUID_PATTERN.test(value)) invalid();
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeClusterRunManagementCommand(
|
||||
value: unknown,
|
||||
): Readonly<ClusterRunManagementCommand> {
|
||||
const envelope = exact(value, ['schemaVersion', 'operation', 'request']);
|
||||
if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
|
||||
const request = exact(envelope.request, [
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'requestId',
|
||||
'auditEventId',
|
||||
'failureAuditEventId',
|
||||
'body',
|
||||
]);
|
||||
let body: ReturnType<typeof parseRunManualRetryRequestBody>;
|
||||
try {
|
||||
body = parseRunManualRetryRequestBody(request.body);
|
||||
} catch {
|
||||
invalid();
|
||||
}
|
||||
const auditEventId = uuid(request.auditEventId);
|
||||
const failureAuditEventId = uuid(request.failureAuditEventId);
|
||||
if (auditEventId === failureAuditEventId) invalid();
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: Object.freeze({
|
||||
projectId: identifier(request.projectId),
|
||||
sourceRunId: identifier(request.sourceRunId),
|
||||
requestId: identifier(request.requestId),
|
||||
auditEventId,
|
||||
failureAuditEventId,
|
||||
body,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterRunManagementTransport(options: Readonly<{
|
||||
service: ClusterRunManagementService;
|
||||
now?: () => number;
|
||||
}>): Readonly<ClusterRunManagementTransport> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => key !== 'service' && key !== 'now') ||
|
||||
!options.service ||
|
||||
typeof options.service.retry !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function')
|
||||
) {
|
||||
throw new ClusterRunManagementTransportConfigurationError();
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
return Object.freeze({
|
||||
async execute(
|
||||
commandValue: unknown,
|
||||
authentication: ClusterRunManagementAuthentication,
|
||||
) {
|
||||
const command = normalizeClusterRunManagementCommand(commandValue);
|
||||
if (
|
||||
!authentication ||
|
||||
typeof authentication !== 'object' ||
|
||||
Array.isArray(authentication) ||
|
||||
Object.keys(authentication).length !== 1 ||
|
||||
typeof authentication.authenticate !== 'function'
|
||||
) {
|
||||
throw new ClusterRunManagementTransportConfigurationError();
|
||||
}
|
||||
let candidate: Readonly<SecurityPrincipal> | null;
|
||||
try {
|
||||
candidate = await authentication.authenticate();
|
||||
} catch {
|
||||
throw new ClusterRunManagementTransportUnavailableError();
|
||||
}
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(candidate as SecurityPrincipal, now());
|
||||
} catch {
|
||||
throw new ClusterRunManagementTransportAuthenticationError();
|
||||
}
|
||||
if (
|
||||
principal.subject.type !== 'user' ||
|
||||
!STRONG_ASSURANCES.has(principal.assurance)
|
||||
) {
|
||||
throw new ClusterRunManagementTransportAuthenticationError();
|
||||
}
|
||||
const result = await options.service.retry({
|
||||
projectId: command.request.projectId,
|
||||
sourceRunId: command.request.sourceRunId,
|
||||
mutationId: command.request.body.mutationId,
|
||||
expectedRunVersion: command.request.body.expectedRunVersion,
|
||||
expectedRunStatus: command.request.body.expectedRunStatus,
|
||||
requestId: command.request.requestId,
|
||||
auditEventId: command.request.auditEventId,
|
||||
failureAuditEventId: command.request.failureAuditEventId,
|
||||
principal,
|
||||
});
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: createRunManualRetryResponseBody(result),
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -163,6 +163,7 @@ function database(serverVersionNum = '160014') {
|
||||
'enforce_plugin_package_stage_provenance',
|
||||
'lock_active_plugin_package_project',
|
||||
'lock_approval_policy_fence',
|
||||
'lock_run_management_policy_fence',
|
||||
'plugin_package_lifecycle_blocking_runs',
|
||||
'plugin_package_automation_start_allowed',
|
||||
'plugin_package_run_start_allowed',
|
||||
|
||||
@@ -12,6 +12,7 @@ const {
|
||||
createClusterAutomationIdentityKeysetFile,
|
||||
createClusterApprovalIdentityKeysetFile,
|
||||
createClusterModelProviderCredentialIdentityKeysetFile,
|
||||
createClusterRunIdentityKeysetFile,
|
||||
} = require('@qinglong/cluster-admin/plugin-package-identity-keyset');
|
||||
|
||||
const NOW_MS = 1_700_000_000_000;
|
||||
@@ -216,6 +217,38 @@ function providerCredentialAssertion(key, overrides = {}) {
|
||||
).toString('base64url')}`;
|
||||
}
|
||||
|
||||
function runAssertion(key, overrides = {}) {
|
||||
const header = Buffer.from(
|
||||
JSON.stringify({
|
||||
alg: 'EdDSA',
|
||||
kid: key.kid,
|
||||
typ: 'ql3-run-management+jwt',
|
||||
}),
|
||||
).toString('base64url');
|
||||
const now = Math.floor(NOW_MS / 1000);
|
||||
const payload = Buffer.from(
|
||||
JSON.stringify({
|
||||
acr: 'urn:ql3:mfa',
|
||||
amr: ['pwd', 'otp'],
|
||||
aud: 'qinglong3-run-management',
|
||||
auth_time: now - 10,
|
||||
exp: now + 120,
|
||||
iat: now,
|
||||
iss: ISSUER,
|
||||
jti: `run-assertion-${key.kid}`,
|
||||
ql3_purpose: 'run-management',
|
||||
sub: 'run-operator-1',
|
||||
...overrides,
|
||||
}),
|
||||
).toString('base64url');
|
||||
const signed = `${header}.${payload}`;
|
||||
return `${signed}.${sign(
|
||||
null,
|
||||
Buffer.from(signed, 'ascii'),
|
||||
key.privateKey,
|
||||
).toString('base64url')}`;
|
||||
}
|
||||
|
||||
async function atomicWrite(filePath, document) {
|
||||
const nextPath = `${filePath}.next`;
|
||||
await writeFile(nextPath, `${JSON.stringify(document)}\n`, { mode: 0o644 });
|
||||
@@ -374,6 +407,34 @@ test('loads a provider credential keyset isolated by type, purpose and audience'
|
||||
});
|
||||
});
|
||||
|
||||
test('loads a Run keyset isolated from every other management purpose', async () => {
|
||||
await fixture(async ({ filePath }) => {
|
||||
const key = reviewedKey('run-identity-key-1');
|
||||
await atomicWrite(filePath, {
|
||||
...keyset(1, [key]),
|
||||
audience: 'qinglong3-run-management',
|
||||
});
|
||||
const provider = createClusterRunIdentityKeysetFile({
|
||||
filePath,
|
||||
now: () => NOW_MS,
|
||||
});
|
||||
const principal = await provider.bind(runAssertion(key)).authenticate();
|
||||
assert.deepEqual(principal.subject, {
|
||||
type: 'user',
|
||||
id: 'run-operator-1',
|
||||
});
|
||||
await assert.rejects(provider.bind(approvalAssertion(key)).authenticate(), {
|
||||
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID',
|
||||
});
|
||||
await assert.rejects(
|
||||
provider
|
||||
.bind(runAssertion(key, { ql3_purpose: 'approval-management' }))
|
||||
.authenticate(),
|
||||
{ code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('supports overlap rotation then immediately revokes the previous key', async () => {
|
||||
await fixture(async ({ filePath }) => {
|
||||
const first = reviewedKey('issuer-key-1');
|
||||
|
||||
@@ -197,6 +197,7 @@ function database(serverVersionNum = '160014') {
|
||||
'plugin_package_tool_start_allowed',
|
||||
'plugin_package_workflow_admission_snapshot',
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementAuthorizationError,
|
||||
createClusterRunManagementService,
|
||||
} = require('@qinglong/cluster-admin/run-management');
|
||||
|
||||
const NOW = 1_000_000;
|
||||
const SOURCE_DIGEST = 'a'.repeat(64);
|
||||
const EXECUTION_DIGEST = 'b'.repeat(64);
|
||||
const TASK_REVISION = `qltd:v1:7:${SOURCE_DIGEST}`;
|
||||
const GENERATED = [
|
||||
'019f9500-0000-4000-8000-000000000010',
|
||||
'019f9500-0000-4000-8000-000000000011',
|
||||
'019f9500-0000-4000-8000-000000000012',
|
||||
'019f9500-0000-4000-8000-000000000013',
|
||||
];
|
||||
|
||||
function request() {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
mutationId: '019f9500-0000-4000-8000-000000000001',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9500-0000-4000-8000-000000000002',
|
||||
failureAuditEventId: '019f9500-0000-4000-8000-000000000003',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 999_000,
|
||||
expiresAtMs: 1_100_000,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function policyRow(role = 'operator') {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
projectName: 'Project 1',
|
||||
projectSlug: 'project-1',
|
||||
projectStatus: 'active',
|
||||
projectVersion: 2,
|
||||
projectCreatedAtMs: '1',
|
||||
projectUpdatedAtMs: '2',
|
||||
bindingProjectId: 'project-1',
|
||||
bindingSubjectType: 'user',
|
||||
bindingSubjectId: 'operator-1',
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: role,
|
||||
bindingMutationId: 'binding-3',
|
||||
bindingChangedByType: 'user',
|
||||
bindingChangedById: 'owner-1',
|
||||
bindingCreatedAtMs: '3',
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(role = 'operator') {
|
||||
const calls = [];
|
||||
const pool = {
|
||||
async query(sql, params = []) {
|
||||
const text = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ scope: 'pool', sql: text, params });
|
||||
if (text.includes('LEFT JOIN LATERAL')) return { rows: [policyRow(role)] };
|
||||
if (text.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`unexpected pool query: ${text}`);
|
||||
},
|
||||
async connect() {
|
||||
return {
|
||||
async query(sql, params = []) {
|
||||
const text = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ scope: 'client', sql: text, params });
|
||||
if (
|
||||
text === 'BEGIN ISOLATION LEVEL SERIALIZABLE' ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK' ||
|
||||
text.startsWith('SELECT set_config')
|
||||
) return { rows: [], rowCount: 0 };
|
||||
if (text.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: NOW }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('lock_run_management_policy_fence')) {
|
||||
return { rows: [{ matches: true }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('idempotency_key = $2')) return { rows: [] };
|
||||
if (text.includes('WHERE run.id = $1')) {
|
||||
return {
|
||||
rows: [{
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: TASK_REVISION,
|
||||
taskName: 'Task 1',
|
||||
taskSnapshotRef: TASK_REVISION,
|
||||
parentRunId: null,
|
||||
triggerType: 'task_start',
|
||||
executionOwner: 'runtime',
|
||||
inputRef: null,
|
||||
priority: 1,
|
||||
runStatus: 'failed',
|
||||
runVersion: 7,
|
||||
attemptExecutorType: 'remote_worker',
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."task_definitions"')) {
|
||||
return { rows: [{ enabled: true }] };
|
||||
}
|
||||
if (text.includes('task_execution_revisions')) {
|
||||
return { rows: [{ sourceContentDigest: SOURCE_DIGEST, contentDigest: EXECUTION_DIGEST }] };
|
||||
}
|
||||
if (text.startsWith('SELECT') && text.includes("trigger_type = 'run_manual_retry'")) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
|
||||
throw new Error(`unexpected client query: ${text}`);
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
},
|
||||
};
|
||||
let index = 0;
|
||||
return {
|
||||
calls,
|
||||
service: createClusterRunManagementService({
|
||||
pool,
|
||||
now: () => NOW,
|
||||
randomUuid: () => GENERATED[index++],
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
test('authorizes run.retry and keeps all generated aggregate identities server-side', async () => {
|
||||
const { calls, service } = fixture();
|
||||
const result = await service.retry(request());
|
||||
assert.equal(result.status, 'accepted');
|
||||
assert.equal(result.runId, GENERATED[0]);
|
||||
assert.equal(result.attemptId, GENERATED[1]);
|
||||
const runInsert = calls.find(({ sql }) => sql.startsWith('INSERT INTO "ql3"."runs"'));
|
||||
assert.equal(runInsert.params[0], GENERATED[0]);
|
||||
assert.equal(runInsert.params.includes(GENERATED[2]), false);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('denied policy writes only the caller-supplied failure audit', async () => {
|
||||
const { calls, service } = fixture('viewer');
|
||||
await assert.rejects(service.retry(request()), ClusterRunManagementAuthorizationError);
|
||||
const audits = calls.filter(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
|
||||
);
|
||||
assert.equal(audits.length, 1);
|
||||
assert.equal(audits[0].params[0], request().failureAuditEventId);
|
||||
assert.equal(calls.some(({ scope }) => scope === 'client'), false);
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
validateClusterRunManagementClientResult,
|
||||
} = require('@qinglong/cluster-admin/run-management-client');
|
||||
const {
|
||||
normalizeClusterRunManagementCommand,
|
||||
} = require('@qinglong/cluster-admin/run-management-transport');
|
||||
const {
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
} = require('@qinglong/cluster-admin/plugin-package-management-client');
|
||||
|
||||
const command = normalizeClusterRunManagementCommand({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9400-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9400-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9400-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
function response(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: '019f9400-0000-4000-8000-000000000010',
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
attemptId: '019f9400-0000-4000-8000-000000000011',
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: 1_000_000,
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('validates one low-sensitive retry response against the request fence', () => {
|
||||
assert.deepEqual(validateClusterRunManagementClientResult(response(), command), response());
|
||||
});
|
||||
|
||||
test('rejects response target, execution placement and shape drift', () => {
|
||||
for (const candidate of [
|
||||
response({ projectId: 'project-2' }),
|
||||
response({ executorType: 'local_process' }),
|
||||
response({ sourceRunVersion: 8 }),
|
||||
{ ...response(), principal: { type: 'user', id: 'operator-1' } },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => validateClusterRunManagementClientResult(candidate, command),
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { readFileSync } = require('node:fs');
|
||||
const { request: httpsRequest } = require('node:https');
|
||||
const { resolve } = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementRateLimitedError,
|
||||
} = require('@qinglong/cluster-admin/run-management');
|
||||
const {
|
||||
startClusterRunManagementHttp,
|
||||
} = require('@qinglong/cluster-admin/run-management-http');
|
||||
|
||||
const SERVER_KEY = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-key.pem');
|
||||
const SERVER_CERT = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem');
|
||||
const PATH = '/api/v3/runs/management';
|
||||
|
||||
function post(port, path = PATH) {
|
||||
const body = Buffer.from(JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9600-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9600-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9600-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
},
|
||||
}));
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const outgoing = httpsRequest({
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
path,
|
||||
method: 'POST',
|
||||
rejectUnauthorized: false,
|
||||
agent: false,
|
||||
headers: {
|
||||
authorization: 'Bearer assertion',
|
||||
'content-type': 'application/json',
|
||||
'content-length': String(body.length),
|
||||
},
|
||||
}, (incoming) => {
|
||||
const chunks = [];
|
||||
incoming.on('data', (chunk) => chunks.push(chunk));
|
||||
incoming.once('end', () => {
|
||||
const bytes = Buffer.concat(chunks);
|
||||
resolvePromise({
|
||||
statusCode: incoming.statusCode,
|
||||
headers: incoming.headers,
|
||||
body: bytes.length ? JSON.parse(bytes.toString('utf8')) : null,
|
||||
});
|
||||
});
|
||||
});
|
||||
outgoing.once('error', reject);
|
||||
outgoing.end(body);
|
||||
});
|
||||
}
|
||||
|
||||
test('serves only the Run path and maps durable quota to bounded HTTP facts', async () => {
|
||||
const application = await startClusterRunManagementHttp({
|
||||
host: '127.0.0.1',
|
||||
port: 0,
|
||||
tls: {
|
||||
privateKey: Buffer.from(readFileSync(SERVER_KEY)),
|
||||
certificate: Buffer.from(readFileSync(SERVER_CERT)),
|
||||
},
|
||||
identities: {
|
||||
async reload() { throw new Error('not used'); },
|
||||
bind() {
|
||||
return { authenticate: async () => ({
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 900,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'hardware',
|
||||
}) };
|
||||
},
|
||||
},
|
||||
transport: {
|
||||
async execute(_command, authentication) {
|
||||
await authentication.authenticate();
|
||||
throw new ClusterRunManagementRateLimitedError(1_500);
|
||||
},
|
||||
},
|
||||
now: () => 1_000,
|
||||
});
|
||||
try {
|
||||
const limited = await post(application.address.port);
|
||||
assert.equal(limited.statusCode, 429);
|
||||
assert.equal(limited.headers['retry-after'], '2');
|
||||
assert.equal(limited.body.error.code, 'rate_limited');
|
||||
assert.match(limited.body.requestId, /^[0-9a-f-]{36}$/);
|
||||
const absent = await post(application.address.port, '/api/v3/approvals/management');
|
||||
assert.equal(absent.statusCode, 404);
|
||||
assert.equal(absent.body.error.code, 'not_found');
|
||||
} finally {
|
||||
await application.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementProcessConfigError,
|
||||
loadClusterRunManagementProcessConfig,
|
||||
startClusterRunManagementProcess,
|
||||
} = require('@qinglong/cluster-admin/run-management-process');
|
||||
|
||||
function enabled(overrides = {}) {
|
||||
return {
|
||||
QL3_RUN_MANAGEMENT_ENABLED: 'true',
|
||||
QL3_PROFILE: 'cluster-admin',
|
||||
QL3_RUN_MANAGEMENT_TLS_CERT_FILE: '/run/ql3/run/tls.crt',
|
||||
QL3_RUN_MANAGEMENT_TLS_KEY_FILE: '/run/ql3/run/tls.key',
|
||||
QL3_RUN_MANAGEMENT_CLIENT_CA_FILE: '/run/ql3/run/client-ca.crt',
|
||||
QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE: '/run/ql3/run/client.crl',
|
||||
QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE: '/run/ql3/run/identity.json',
|
||||
QL3_POSTGRES_RUN_MANAGER_HOST: 'postgres.qinglong3-system.svc',
|
||||
QL3_POSTGRES_RUN_MANAGER_DATABASE: 'qinglong3',
|
||||
QL3_POSTGRES_RUN_MANAGER_USER: 'ql3_run_manager',
|
||||
QL3_POSTGRES_RUN_MANAGER_PASSWORD: 'secret',
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: 'postgres.qinglong3-system.svc',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('disabled Run manager acquires no PostgreSQL or file authority', async () => {
|
||||
let opened = false;
|
||||
const runtime = await startClusterRunManagementProcess({
|
||||
environment: {},
|
||||
openDatabase: async () => {
|
||||
opened = true;
|
||||
throw new Error('must not open');
|
||||
},
|
||||
});
|
||||
assert.equal(runtime.status, 'disabled');
|
||||
assert.equal(opened, false);
|
||||
await runtime.close();
|
||||
});
|
||||
|
||||
test('loads a bounded opt-in Run-only process configuration', () => {
|
||||
const config = loadClusterRunManagementProcessConfig(enabled());
|
||||
assert.equal(config.enabled, true);
|
||||
assert.equal(config.port, 8448);
|
||||
assert.equal(config.http.maxConcurrentRequests, 16);
|
||||
assert.equal(config.database.pool.maxConnections, 2);
|
||||
assert.equal(config.database.connection.user, 'ql3_run_manager');
|
||||
assert.deepEqual(config.database.connection.tls, {
|
||||
mode: 'verify-full',
|
||||
servername: 'postgres.qinglong3-system.svc',
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects profile drift and implicit insecure PostgreSQL', () => {
|
||||
assert.throws(
|
||||
() => loadClusterRunManagementProcessConfig(enabled({ QL3_PROFILE: 'cluster-control' })),
|
||||
ClusterRunManagementProcessConfigError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
loadClusterRunManagementProcessConfig(
|
||||
enabled({
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_MODE: 'disable',
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: undefined,
|
||||
}),
|
||||
),
|
||||
ClusterRunManagementProcessConfigError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
ClusterRunManagementTransportRequestError,
|
||||
createClusterRunManagementTransport,
|
||||
normalizeClusterRunManagementCommand,
|
||||
} = require('@qinglong/cluster-admin/run-management-transport');
|
||||
|
||||
const NOW = 1_000_000;
|
||||
|
||||
function principal(overrides = {}) {
|
||||
return {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 999_000,
|
||||
expiresAtMs: 1_100_000,
|
||||
assurance: 'multi_factor',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9300-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9300-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9300-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function retryResult() {
|
||||
return {
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: '019f9300-0000-4000-8000-000000000010',
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
attemptId: '019f9300-0000-4000-8000-000000000011',
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: NOW,
|
||||
};
|
||||
}
|
||||
|
||||
test('routes one exact strong User retry and emits the shared response', async () => {
|
||||
const calls = [];
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry(request) {
|
||||
calls.push(request);
|
||||
return retryResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await transport.execute(command(), {
|
||||
authenticate: async () => principal(),
|
||||
});
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].mutationId, command().request.body.mutationId);
|
||||
assert.equal(calls[0].principal.assurance, 'multi_factor');
|
||||
assert.deepEqual(result, {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
...retryResult(),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects weak or non-User identity before service authority', async () => {
|
||||
let called = false;
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry() {
|
||||
called = true;
|
||||
return retryResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
transport.execute(command(), {
|
||||
authenticate: async () => principal({ assurance: 'single_factor' }),
|
||||
}),
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
);
|
||||
await assert.rejects(
|
||||
transport.execute(command(), {
|
||||
authenticate: async () =>
|
||||
principal({ subject: { type: 'agent', id: 'agent-1' } }),
|
||||
}),
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
);
|
||||
assert.equal(called, false);
|
||||
});
|
||||
|
||||
test('rejects widened commands and ambiguous audit identity', () => {
|
||||
assert.throws(
|
||||
() => normalizeClusterRunManagementCommand({ ...command(), principal: principal() }),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeClusterRunManagementCommand(
|
||||
command({ failureAuditEventId: command().request.auditEventId }),
|
||||
),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeClusterRunManagementCommand(
|
||||
command({ body: { ...command().request.body, expectedRunStatus: 'lost' } }),
|
||||
),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
});
|
||||
@@ -334,6 +334,7 @@ function databaseResource(events, options = {}) {
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'plugin_package_run_start_allowed',
|
||||
'plugin_package_tool_start_allowed',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
|
||||
@@ -243,6 +243,7 @@ function databaseResource(events, overrides = {}) {
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'plugin_package_run_start_allowed',
|
||||
'plugin_package_tool_start_allowed',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
|
||||
@@ -75,6 +75,11 @@
|
||||
"require": "./dist/run-management/runManualRetryRepository.js",
|
||||
"default": "./dist/run-management/runManualRetryRepository.js"
|
||||
},
|
||||
"./run-manager": {
|
||||
"types": "./dist/entrypoints/runManager.d.ts",
|
||||
"require": "./dist/entrypoints/runManager.js",
|
||||
"default": "./dist/entrypoints/runManager.js"
|
||||
},
|
||||
"./approval-manager": {
|
||||
"types": "./dist/approval-management/index.d.ts",
|
||||
"require": "./dist/approval-management/index.js",
|
||||
|
||||
@@ -19,6 +19,7 @@ const DEFAULT_AI_CREDENTIAL_TESTER_APPLICATION_NAME =
|
||||
const DEFAULT_AUTOMATION_MANAGER_APPLICATION_NAME =
|
||||
'qinglong-automation-manager';
|
||||
const DEFAULT_APPROVAL_MANAGER_APPLICATION_NAME = 'qinglong-approval-manager';
|
||||
const DEFAULT_RUN_MANAGER_APPLICATION_NAME = 'qinglong-run-manager';
|
||||
const DEFAULT_PACKAGE_MANAGER_APPLICATION_NAME = 'qinglong-package-manager';
|
||||
const DEFAULT_PACKAGE_EXECUTOR_APPLICATION_NAME = 'qinglong-package-executor';
|
||||
const DEFAULT_WORKER_CREDENTIAL_MANAGER_APPLICATION_NAME =
|
||||
@@ -89,6 +90,7 @@ export type PostgresDatabaseRole =
|
||||
| 'admin'
|
||||
| 'automation-manager'
|
||||
| 'approval-manager'
|
||||
| 'run-manager'
|
||||
| 'package-manager'
|
||||
| 'package-executor'
|
||||
| 'worker-credential-manager'
|
||||
@@ -255,6 +257,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
'admin',
|
||||
'automation-manager',
|
||||
'approval-manager',
|
||||
'run-manager',
|
||||
'package-manager',
|
||||
'package-executor',
|
||||
'worker-credential-manager',
|
||||
@@ -296,6 +299,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
const isAdmin = options.role === 'admin';
|
||||
const isAutomationManager = options.role === 'automation-manager';
|
||||
const isApprovalManager = options.role === 'approval-manager';
|
||||
const isRunManager = options.role === 'run-manager';
|
||||
const isPackageManager = options.role === 'package-manager';
|
||||
const isPackageExecutor = options.role === 'package-executor';
|
||||
const isWorkerCredentialManager =
|
||||
@@ -309,6 +313,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
isAdmin ||
|
||||
isAutomationManager ||
|
||||
isApprovalManager ||
|
||||
isRunManager ||
|
||||
isPackageManager ||
|
||||
isPackageExecutor ||
|
||||
isWorkerCredentialManager ||
|
||||
@@ -370,6 +375,8 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
|
||||
? DEFAULT_AUTOMATION_MANAGER_APPLICATION_NAME
|
||||
: isApprovalManager
|
||||
? DEFAULT_APPROVAL_MANAGER_APPLICATION_NAME
|
||||
: isRunManager
|
||||
? DEFAULT_RUN_MANAGER_APPLICATION_NAME
|
||||
: isPackageManager
|
||||
? DEFAULT_PACKAGE_MANAGER_APPLICATION_NAME
|
||||
: isPackageExecutor
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
export { PostgresRunManualRetryRepository } from '../run-management/runManualRetryRepository';
|
||||
export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository';
|
||||
export { PostgresSecurityAuditRepository } from '../security/securityAuditRepository';
|
||||
export {
|
||||
PostgresPluginPackageIdentityKeysetLedgerRepository as PostgresRunManagementIdentityKeysetLedgerRepository,
|
||||
type ClusterManagementIdentityAuthority,
|
||||
} from '../management/pluginPackageIdentityKeysetLedgerRepository';
|
||||
export {
|
||||
PgPoolBinding,
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresTlsDnsServername,
|
||||
type OpenPostgresDatabaseOptions,
|
||||
type PostgresConnectionOptions,
|
||||
type PostgresDatabaseRole,
|
||||
type PostgresPoolOptions,
|
||||
} from '../connection/pool';
|
||||
export {
|
||||
PostgresConnectionEnvironmentError,
|
||||
loadPostgresConnectionEnvironment,
|
||||
type PostgresConnectionEnvironment,
|
||||
type PostgresConnectionEnvironmentKeys,
|
||||
} from '../connection/connectionEnvironment';
|
||||
export {
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
type PostgresCertificateAuthorityFileInspection,
|
||||
} from '../connection/certificateAuthority';
|
||||
export {
|
||||
PostgresSchemaReadinessError,
|
||||
assertPostgresRunManagerSchemaReady,
|
||||
type PostgresSchemaReadinessReport,
|
||||
} from '../schema/schemaReadiness';
|
||||
+4
-2
@@ -5,7 +5,8 @@ export type ClusterManagementIdentityAuthority =
|
||||
| 'plugin-package-management'
|
||||
| 'worker-credential-management'
|
||||
| 'automation-management'
|
||||
| 'approval-management';
|
||||
| 'approval-management'
|
||||
| 'run-management';
|
||||
const DIGEST_PATTERN = /^[A-Za-z0-9_-]{43}$/;
|
||||
const KEY_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
|
||||
@@ -189,7 +190,8 @@ export class PostgresPluginPackageIdentityKeysetLedgerRepository
|
||||
authority !== 'plugin-package-management' &&
|
||||
authority !== 'worker-credential-management' &&
|
||||
authority !== 'automation-management' &&
|
||||
authority !== 'approval-management'
|
||||
authority !== 'approval-management' &&
|
||||
authority !== 'run-management'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL management identity keyset authority is invalid',
|
||||
|
||||
@@ -283,5 +283,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest =
|
||||
checksum:
|
||||
'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30',
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'pg-0056-run-management-boundary',
|
||||
checksum:
|
||||
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
|
||||
}),
|
||||
]),
|
||||
});
|
||||
|
||||
@@ -58,6 +58,7 @@ import { pg0052AutomationManagementIdentityKeysetLedgerMigration } from './pg-00
|
||||
import { pg0053PluginPackageWorkflowRunListIndexMigration } from './pg-0053-plugin-package-workflow-run-list-index';
|
||||
import { pg0054ApprovalManagementBoundaryMigration } from './pg-0054-approval-management-boundary';
|
||||
import { pg0055RunAttemptLogRetentionMigration } from './pg-0055-run-attempt-log-retention';
|
||||
import { pg0056RunManagementBoundaryMigration } from '../run-management/pg-0056-run-management-boundary';
|
||||
|
||||
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
|
||||
Object.freeze({
|
||||
@@ -121,5 +122,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMi
|
||||
pg0053PluginPackageWorkflowRunListIndexMigration,
|
||||
pg0054ApprovalManagementBoundaryMigration,
|
||||
pg0055RunAttemptLogRetentionMigration,
|
||||
pg0056RunManagementBoundaryMigration,
|
||||
]),
|
||||
});
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { CAPABILITIES_V54 } from '../migrations/pg-0055-run-attempt-log-retention';
|
||||
import { definePostgresSqlMigration } from '../migrations/sqlMigration';
|
||||
|
||||
export const CAPABILITIES_V55 = CAPABILITIES_V54.replace(
|
||||
'"run_core":1,',
|
||||
'"run_management_boundary":1,"run_core":1,',
|
||||
);
|
||||
|
||||
export const pg0056RunManagementBoundaryMigration =
|
||||
definePostgresSqlMigration({
|
||||
id: 'pg-0056-run-management-boundary',
|
||||
statements: [
|
||||
`
|
||||
DO $ql3$
|
||||
DECLARE
|
||||
role_invalid boolean;
|
||||
BEGIN
|
||||
SELECT roles.rolname IS NULL
|
||||
OR roles.rolcanlogin IS NOT TRUE
|
||||
OR roles.rolsuper IS NOT FALSE
|
||||
OR roles.rolcreatedb IS NOT FALSE
|
||||
OR roles.rolcreaterole IS NOT FALSE
|
||||
OR roles.rolreplication IS NOT FALSE
|
||||
OR roles.rolbypassrls IS NOT FALSE
|
||||
INTO role_invalid
|
||||
FROM (SELECT 1) AS expected
|
||||
LEFT JOIN pg_catalog.pg_roles AS roles
|
||||
ON roles.rolname = 'ql3_run_manager';
|
||||
|
||||
IF role_invalid IS NOT FALSE THEN
|
||||
RAISE EXCEPTION
|
||||
'required QingLong run manager role is missing or privileged'
|
||||
USING ERRCODE = 'insufficient_privilege';
|
||||
END IF;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
`
|
||||
DO $ql3$
|
||||
BEGIN
|
||||
EXECUTE format(
|
||||
'GRANT CONNECT ON DATABASE %I TO ql3_run_manager',
|
||||
current_database()
|
||||
);
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
`GRANT USAGE ON SCHEMA "ql3" TO ql3_run_manager`,
|
||||
`REVOKE ALL ON ALL TABLES IN SCHEMA "ql3" FROM ql3_run_manager`,
|
||||
`REVOKE ALL ON ALL FUNCTIONS IN SCHEMA "ql3" FROM ql3_run_manager`,
|
||||
`
|
||||
CREATE FUNCTION "ql3"."lock_run_management_policy_fence"(
|
||||
varchar,
|
||||
varchar,
|
||||
varchar,
|
||||
integer,
|
||||
integer
|
||||
)
|
||||
RETURNS boolean
|
||||
LANGUAGE plpgsql
|
||||
VOLATILE
|
||||
SECURITY DEFINER
|
||||
SET search_path = pg_catalog, ql3
|
||||
AS $ql3$
|
||||
DECLARE
|
||||
project_status varchar;
|
||||
project_version integer;
|
||||
binding_state varchar;
|
||||
binding_role varchar;
|
||||
binding_version integer;
|
||||
BEGIN
|
||||
SELECT project.status, project.version
|
||||
INTO project_status, project_version
|
||||
FROM "ql3"."projects" AS project
|
||||
WHERE project.id = $1
|
||||
FOR UPDATE;
|
||||
|
||||
SELECT binding.state, binding.role, binding.version
|
||||
INTO binding_state, binding_role, binding_version
|
||||
FROM "ql3"."project_role_bindings" AS binding
|
||||
WHERE binding.project_id = $1
|
||||
AND binding.subject_type = $2
|
||||
AND binding.subject_id = $3
|
||||
ORDER BY binding.version DESC
|
||||
LIMIT 1;
|
||||
|
||||
RETURN project_status = 'active'
|
||||
AND project_version = $4
|
||||
AND binding_state = 'active'
|
||||
AND binding_role IN ('owner', 'admin', 'operator')
|
||||
AND binding_version = $5;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
`REVOKE ALL ON FUNCTION "ql3"."lock_run_management_policy_fence"(varchar, varchar, varchar, integer, integer) FROM PUBLIC`,
|
||||
`GRANT EXECUTE ON FUNCTION "ql3"."lock_run_management_policy_fence"(varchar, varchar, varchar, integer, integer) TO ql3_runtime, ql3_run_manager`,
|
||||
`GRANT SELECT ON "ql3"."schema_migrations", "ql3"."schema_capabilities", "ql3"."projects", "ql3"."project_role_bindings", "ql3"."task_definitions", "ql3"."task_definition_revisions", "ql3"."task_execution_revisions" TO ql3_run_manager`,
|
||||
`GRANT SELECT, INSERT ON "ql3"."runs", "ql3"."run_attempts", "ql3"."run_events", "ql3"."security_audit_events" TO ql3_run_manager`,
|
||||
`GRANT SELECT, INSERT, UPDATE ON "ql3"."plugin_package_identity_keyset_ledger" TO ql3_run_manager`,
|
||||
`ALTER TABLE "ql3"."plugin_package_identity_keyset_ledger" DROP CONSTRAINT ql3_plugin_package_identity_keyset_authority_check`,
|
||||
`ALTER TABLE "ql3"."plugin_package_identity_keyset_ledger" ADD CONSTRAINT ql3_plugin_package_identity_keyset_authority_check CHECK (authority IN ('plugin-package-management', 'worker-credential-management', 'automation-management', 'approval-management', 'run-management'))`,
|
||||
`
|
||||
DO $ql3$
|
||||
BEGIN
|
||||
UPDATE "ql3"."schema_capabilities"
|
||||
SET contract_version = 55,
|
||||
migration_id = 'pg-0056-run-management-boundary',
|
||||
capabilities = '${CAPABILITIES_V55}'::jsonb,
|
||||
updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint
|
||||
WHERE contract_name = 'control-core'
|
||||
AND contract_version = 54
|
||||
AND migration_id = 'pg-0055-run-attempt-log-retention'
|
||||
AND capabilities = '${CAPABILITIES_V54}'::jsonb;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'control-core capability is not at version 54'
|
||||
USING ERRCODE = 'check_violation';
|
||||
END IF;
|
||||
END
|
||||
$ql3$
|
||||
`.trim(),
|
||||
],
|
||||
});
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
RunManualRetryUnavailableError,
|
||||
normalizeRunManualRetryCommand,
|
||||
normalizeRunManualRetryResult,
|
||||
type RunManualRetryAllowedRole,
|
||||
type RunManualRetryCommand,
|
||||
type RunManualRetryRepository,
|
||||
type RunManualRetryResult,
|
||||
@@ -33,11 +32,6 @@ type Row = Record<string, unknown>;
|
||||
export const CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS = 60_000;
|
||||
export const CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT = 64;
|
||||
|
||||
const ALLOWED_ROLES = new Set<RunManualRetryAllowedRole>([
|
||||
'owner',
|
||||
'admin',
|
||||
'operator',
|
||||
]);
|
||||
const CLUSTER_STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
|
||||
const TASK_REVISION_PATTERN = /^qltd:v1:([1-9]\d*):([0-9a-f]{64})$/;
|
||||
|
||||
@@ -128,45 +122,23 @@ async function confirmAuthorization(
|
||||
client: PostgresClient,
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): Promise<void> {
|
||||
const project = await client.query<Row>(
|
||||
const result = await client.query<Row>(
|
||||
`
|
||||
SELECT status AS "projectStatus", version AS "projectVersion"
|
||||
FROM "ql3"."projects" WHERE id = $1 FOR UPDATE
|
||||
`,
|
||||
[command.projectId],
|
||||
);
|
||||
if (project.rows.length === 0) throw new RunManualRetryNotFoundError();
|
||||
if (project.rows.length !== 1) throw unavailable();
|
||||
// Authorized management mutations take the same Project lock. Keeping this
|
||||
// append-only RoleBinding read lock-free avoids granting UPDATE authority to
|
||||
// the runtime role merely to use PostgreSQL row-lock syntax.
|
||||
const binding = await client.query<Row>(
|
||||
`
|
||||
SELECT version AS "bindingVersion", state AS "bindingState",
|
||||
role AS "bindingRole"
|
||||
FROM "ql3"."project_role_bindings"
|
||||
WHERE project_id = $1 AND subject_type = $2 AND subject_id = $3
|
||||
ORDER BY version DESC LIMIT 1
|
||||
SELECT "ql3"."lock_run_management_policy_fence"(
|
||||
$1::varchar, $2::varchar, $3::varchar, $4::integer, $5::integer
|
||||
) AS "matches"
|
||||
`,
|
||||
[
|
||||
command.projectId,
|
||||
command.principal.subject.type,
|
||||
command.principal.subject.id,
|
||||
command.policyFence.projectVersion,
|
||||
command.policyFence.bindingVersion,
|
||||
],
|
||||
);
|
||||
const currentProject = project.rows[0]!;
|
||||
const currentBinding = binding.rows[0];
|
||||
if (
|
||||
text(currentProject, 'projectStatus') !== 'active' ||
|
||||
integer(currentProject, 'projectVersion') !==
|
||||
command.policyFence.projectVersion ||
|
||||
!currentBinding ||
|
||||
integer(currentBinding, 'bindingVersion') !==
|
||||
command.policyFence.bindingVersion ||
|
||||
text(currentBinding, 'bindingState') !== 'active' ||
|
||||
!ALLOWED_ROLES.has(
|
||||
text(currentBinding, 'bindingRole') as RunManualRetryAllowedRole,
|
||||
)
|
||||
result.rows.length !== 1 ||
|
||||
!postgresRequiredBoolean(result.rows[0]!.matches, unavailable)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('authorization_changed');
|
||||
}
|
||||
@@ -185,8 +157,6 @@ async function findReplay(
|
||||
run.execution_origin AS "executionOrigin",
|
||||
run.execution_owner AS "executionOwner",
|
||||
run.triggered_by AS "triggeredBy", run.request_id AS "requestId",
|
||||
run.status AS "runStatus", run.version AS "runVersion",
|
||||
run.event_sequence AS "eventSequence",
|
||||
run.created_at_ms AS "createdAtMs",
|
||||
attempt.id AS "attemptId", attempt.executor_type AS "executorType",
|
||||
created.actor_type AS "createdActorType",
|
||||
@@ -205,7 +175,6 @@ async function findReplay(
|
||||
ON queued.run_id = run.id AND queued.sequence = 2
|
||||
AND queued.type = 'run.queued'
|
||||
WHERE run.project_id = $1 AND run.idempotency_key = $2
|
||||
FOR UPDATE OF run
|
||||
`,
|
||||
[command.projectId, `ql3:run-manual-retry:v1:${command.mutationId}`],
|
||||
);
|
||||
@@ -227,9 +196,6 @@ function replayResult(
|
||||
text(row, 'executionOwner') !== 'runtime' ||
|
||||
text(row, 'triggeredBy') !== command.principal.subject.id ||
|
||||
text(row, 'requestId') !== command.mutationId ||
|
||||
text(row, 'runStatus') !== 'queued' ||
|
||||
integer(row, 'runVersion') !== 2 ||
|
||||
integer(row, 'eventSequence') !== 2 ||
|
||||
text(row, 'executorType') !== 'remote_worker' ||
|
||||
text(row, 'createdActorType') !== command.principal.subject.type ||
|
||||
text(row, 'createdActorId') !== command.principal.subject.id ||
|
||||
@@ -295,7 +261,6 @@ async function findSource(
|
||||
LIMIT 1
|
||||
) AS attempt ON true
|
||||
WHERE run.id = $1
|
||||
FOR UPDATE OF run
|
||||
`,
|
||||
[command.sourceRunId],
|
||||
);
|
||||
|
||||
@@ -15,13 +15,14 @@ export interface PostgresSchemaContractFunction {
|
||||
export interface PostgresSchemaContract {
|
||||
readonly schema: 'ql3';
|
||||
readonly contractName: 'control-core';
|
||||
readonly contractVersion: 54;
|
||||
readonly migrationId: 'pg-0055-run-attempt-log-retention';
|
||||
readonly contractVersion: 55;
|
||||
readonly migrationId: 'pg-0056-run-management-boundary';
|
||||
readonly minimumServerMajor: 16;
|
||||
readonly maximumServerMajor: 18;
|
||||
readonly capabilities: Readonly<{
|
||||
run_core: 1;
|
||||
run_attempt_log_retention: 1;
|
||||
run_management_boundary: 1;
|
||||
run_dispatch_lease: 1;
|
||||
run_retry_policy: 1;
|
||||
project_policy: 1;
|
||||
@@ -101,8 +102,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
Object.freeze({
|
||||
schema: 'ql3',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 54,
|
||||
migrationId: 'pg-0055-run-attempt-log-retention',
|
||||
contractVersion: 55,
|
||||
migrationId: 'pg-0056-run-management-boundary',
|
||||
minimumServerMajor: 16,
|
||||
maximumServerMajor: 18,
|
||||
capabilities: Object.freeze({
|
||||
@@ -143,6 +144,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
project_tool_definition_snapshot: 1,
|
||||
run_core: 1,
|
||||
run_attempt_log_retention: 1,
|
||||
run_management_boundary: 1,
|
||||
run_dispatch_lease: 1,
|
||||
run_retry_policy: 1,
|
||||
security_audit: 1,
|
||||
@@ -2323,6 +2325,15 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
volatility: 'volatile',
|
||||
configuration: Object.freeze(['search_path=pg_catalog, ql3']),
|
||||
}),
|
||||
Object.freeze({
|
||||
name: 'lock_run_management_policy_fence',
|
||||
identityArguments:
|
||||
'character varying, character varying, character varying, integer, integer',
|
||||
owner: 'ql3_migration',
|
||||
securityDefiner: true,
|
||||
volatility: 'volatile',
|
||||
configuration: Object.freeze(['search_path=pg_catalog, ql3']),
|
||||
}),
|
||||
Object.freeze({
|
||||
name: 'commit_plugin_package_task_reconciliation',
|
||||
identityArguments:
|
||||
|
||||
@@ -19,6 +19,7 @@ export const POSTGRES_SCHEMA_READINESS_ERROR_CODES = [
|
||||
'admin_role_invalid',
|
||||
'automation_manager_role_invalid',
|
||||
'approval_manager_role_invalid',
|
||||
'run_manager_role_invalid',
|
||||
'package_manager_role_invalid',
|
||||
'package_executor_role_invalid',
|
||||
'worker_credential_manager_role_invalid',
|
||||
@@ -1361,6 +1362,37 @@ const REQUIRED_APPROVAL_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze(
|
||||
),
|
||||
);
|
||||
|
||||
const REQUIRED_RUN_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze(
|
||||
Object.fromEntries(
|
||||
postgresqlControlSchemaContract.tables.map(({ name }) => [
|
||||
name,
|
||||
Object.freeze(
|
||||
name === 'schema_migrations' ||
|
||||
name === 'schema_capabilities' ||
|
||||
name === 'projects' ||
|
||||
name === 'project_role_bindings' ||
|
||||
name === 'task_definitions' ||
|
||||
name === 'task_definition_revisions' ||
|
||||
name === 'task_execution_revisions'
|
||||
? { ...NO_TABLE_PRIVILEGES, select: true }
|
||||
: name === 'runs' ||
|
||||
name === 'run_attempts' ||
|
||||
name === 'run_events' ||
|
||||
name === 'security_audit_events'
|
||||
? { ...NO_TABLE_PRIVILEGES, select: true, insert: true }
|
||||
: name === 'plugin_package_identity_keyset_ledger'
|
||||
? {
|
||||
...NO_TABLE_PRIVILEGES,
|
||||
select: true,
|
||||
insert: true,
|
||||
update: true,
|
||||
}
|
||||
: NO_TABLE_PRIVILEGES,
|
||||
),
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
const REQUIRED_WORKER_CREDENTIAL_MANAGER_PRIVILEGES: RequiredPrivileges =
|
||||
Object.freeze(
|
||||
Object.fromEntries(
|
||||
@@ -1448,6 +1480,7 @@ const REQUIRED_RUNTIME_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
|
||||
enforce_plugin_package_stage_provenance: false,
|
||||
lock_active_plugin_package_project: false,
|
||||
lock_approval_policy_fence: false,
|
||||
lock_run_management_policy_fence: true,
|
||||
plugin_package_automation_start_allowed: true,
|
||||
plugin_package_workflow_admission_snapshot: true,
|
||||
plugin_package_workflow_task_attempt_snapshot: true,
|
||||
@@ -1464,6 +1497,7 @@ const REQUIRED_PACKAGE_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
|
||||
enforce_plugin_package_stage_provenance: false,
|
||||
lock_active_plugin_package_project: false,
|
||||
lock_approval_policy_fence: true,
|
||||
lock_run_management_policy_fence: false,
|
||||
plugin_package_automation_start_allowed: false,
|
||||
plugin_package_workflow_admission_snapshot: false,
|
||||
plugin_package_workflow_task_attempt_snapshot: false,
|
||||
@@ -1480,6 +1514,7 @@ const REQUIRED_PACKAGE_EXECUTOR_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges
|
||||
enforce_plugin_package_stage_provenance: false,
|
||||
lock_active_plugin_package_project: true,
|
||||
lock_approval_policy_fence: true,
|
||||
lock_run_management_policy_fence: false,
|
||||
plugin_package_automation_start_allowed: false,
|
||||
plugin_package_workflow_admission_snapshot: false,
|
||||
plugin_package_workflow_task_attempt_snapshot: false,
|
||||
@@ -1500,6 +1535,12 @@ const REQUIRED_APPROVAL_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges
|
||||
lock_approval_policy_fence: true,
|
||||
});
|
||||
|
||||
const REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
|
||||
Object.freeze({
|
||||
...NO_FUNCTION_PRIVILEGES,
|
||||
lock_run_management_policy_fence: true,
|
||||
});
|
||||
|
||||
function safeInteger(value: unknown): number | null {
|
||||
if (typeof value === 'number' && Number.isSafeInteger(value)) return value;
|
||||
if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) {
|
||||
@@ -1864,6 +1905,7 @@ async function assertRole(
|
||||
| 'admin_role_invalid'
|
||||
| 'automation_manager_role_invalid'
|
||||
| 'approval_manager_role_invalid'
|
||||
| 'run_manager_role_invalid'
|
||||
| 'package_manager_role_invalid'
|
||||
| 'package_executor_role_invalid'
|
||||
| 'worker_credential_manager_role_invalid'
|
||||
@@ -2120,6 +2162,30 @@ export async function assertPostgresApprovalManagerSchemaReady(
|
||||
});
|
||||
}
|
||||
|
||||
export async function assertPostgresRunManagerSchemaReady(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
|
||||
): Promise<PostgresSchemaReadinessReport> {
|
||||
const server = await readServer(queryable, contract);
|
||||
const migrationIds = await assertHistory(queryable);
|
||||
await assertCapability(queryable, contract);
|
||||
await assertSchemaContract(queryable, contract);
|
||||
await assertRole(
|
||||
queryable,
|
||||
contract,
|
||||
REQUIRED_RUN_MANAGER_PRIVILEGES,
|
||||
REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES,
|
||||
'run_manager_role_invalid',
|
||||
);
|
||||
return Object.freeze({
|
||||
ready: true,
|
||||
...server,
|
||||
contractName: contract.contractName,
|
||||
contractVersion: contract.contractVersion,
|
||||
migrationIds,
|
||||
});
|
||||
}
|
||||
|
||||
export async function assertPostgresPackageManagerSchemaReady(
|
||||
queryable: PostgresMigrationQueryable,
|
||||
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
|
||||
|
||||
+9
-1
@@ -108,7 +108,7 @@ test('serializes first observation, exact replay and append-only rotation', asyn
|
||||
assert.equal(value.releases(), 3);
|
||||
});
|
||||
|
||||
test('isolates Plugin, Worker, automation and Approval generations by authority key', async () => {
|
||||
test('isolates Plugin, Worker, automation, Approval and Run generations by authority key', async () => {
|
||||
const value = fixture('worker-credential-management');
|
||||
await value.repository.observe(
|
||||
snapshot(1, { audience: 'qinglong3-worker-credential-management' }),
|
||||
@@ -131,6 +131,14 @@ test('isolates Plugin, Worker, automation and Approval generations by authority
|
||||
approval.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
|
||||
'approval-management',
|
||||
);
|
||||
const run = fixture('run-management');
|
||||
await run.repository.observe(
|
||||
snapshot(1, { audience: 'qinglong3-run-management' }),
|
||||
);
|
||||
assert.equal(
|
||||
run.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
|
||||
'run-management',
|
||||
);
|
||||
assert.throws(
|
||||
() => fixture('worker-credential-executor'),
|
||||
TypeError,
|
||||
|
||||
@@ -104,6 +104,7 @@ test('enforces role-specific bounded pool sizes', () => {
|
||||
'ai-credential-tester',
|
||||
'automation-manager',
|
||||
'approval-manager',
|
||||
'run-manager',
|
||||
'worker-credential-manager',
|
||||
'worker-credential-executor',
|
||||
]) {
|
||||
|
||||
@@ -104,6 +104,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
|
||||
'pg-0053-plugin-package-workflow-run-list-index',
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
],
|
||||
);
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
@@ -514,6 +515,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
|
||||
checksum:
|
||||
'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30',
|
||||
},
|
||||
{
|
||||
id: 'pg-0056-run-management-boundary',
|
||||
checksum:
|
||||
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
|
||||
},
|
||||
];
|
||||
assert.deepEqual(
|
||||
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
|
||||
@@ -1904,3 +1910,30 @@ test('advances capability v54 with durable Cluster log retention authority', asy
|
||||
/migration_id = 'pg-0054-approval-management-boundary'/,
|
||||
);
|
||||
});
|
||||
|
||||
test('advances capability v55 with isolated strong Run management authority', async () => {
|
||||
const migration = migrationById('pg-0056-run-management-boundary');
|
||||
const statements = [];
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(sql, /ql3_run_manager/);
|
||||
assert.match(sql, /lock_run_management_policy_fence/);
|
||||
assert.match(
|
||||
sql,
|
||||
/GRANT SELECT, INSERT ON "ql3"\."runs", "ql3"\."run_attempts", "ql3"\."run_events", "ql3"\."security_audit_events" TO ql3_run_manager/,
|
||||
);
|
||||
assert.doesNotMatch(sql, /GRANT UPDATE ON "ql3"\."runs"/);
|
||||
assert.match(sql, /'run-management'/);
|
||||
assert.match(sql, /contract_version = 55/);
|
||||
assert.match(sql, /"run_management_boundary":1/);
|
||||
assert.match(sql, /contract_version = 54/);
|
||||
assert.match(
|
||||
sql,
|
||||
/migration_id = 'pg-0055-run-attempt-log-retention'/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ const {
|
||||
PostgresSchemaReadinessError,
|
||||
assertPostgresAdminSchemaReady,
|
||||
assertPostgresApprovalManagerSchemaReady,
|
||||
assertPostgresRunManagerSchemaReady,
|
||||
assertPostgresAutomationManagerSchemaReady,
|
||||
assertPostgresPackageExecutorSchemaReady,
|
||||
assertPostgresPackageManagerSchemaReady,
|
||||
@@ -474,6 +475,37 @@ function approvalManagerPrivileges() {
|
||||
}));
|
||||
}
|
||||
|
||||
function runManagerPrivileges() {
|
||||
const readable = new Set([
|
||||
'schema_migrations',
|
||||
'schema_capabilities',
|
||||
'projects',
|
||||
'project_role_bindings',
|
||||
'task_definitions',
|
||||
'task_definition_revisions',
|
||||
'task_execution_revisions',
|
||||
'runs',
|
||||
'run_attempts',
|
||||
'run_events',
|
||||
'security_audit_events',
|
||||
'plugin_package_identity_keyset_ledger',
|
||||
]);
|
||||
return postgresqlControlSchemaContract.tables.map(({ name: tableName }) => ({
|
||||
tableName,
|
||||
selectAllowed: readable.has(tableName),
|
||||
insertAllowed: [
|
||||
'runs',
|
||||
'run_attempts',
|
||||
'run_events',
|
||||
'security_audit_events',
|
||||
'plugin_package_identity_keyset_ledger',
|
||||
].includes(tableName),
|
||||
updateAllowed: tableName === 'plugin_package_identity_keyset_ledger',
|
||||
deleteAllowed: false,
|
||||
isOwner: false,
|
||||
}));
|
||||
}
|
||||
|
||||
function workerCredentialPrivileges(kind) {
|
||||
const manager = kind === 'manager';
|
||||
const readable = new Set([
|
||||
@@ -634,6 +666,8 @@ function queryable(overrides = {}) {
|
||||
executeAllowed:
|
||||
overrides.functionMode === 'manager'
|
||||
? functionName === 'lock_approval_policy_fence'
|
||||
: overrides.functionMode === 'run-manager'
|
||||
? functionName === 'lock_run_management_policy_fence'
|
||||
: overrides.functionMode === 'executor'
|
||||
? [
|
||||
'commit_plugin_package_lifecycle',
|
||||
@@ -651,6 +685,7 @@ function queryable(overrides = {}) {
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'plugin_package_run_start_allowed',
|
||||
'plugin_package_tool_start_allowed',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
@@ -696,7 +731,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
serverMajor: 16,
|
||||
currentUser: 'ql3_runtime',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 54,
|
||||
contractVersion: 55,
|
||||
migrationIds: [
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
@@ -753,6 +788,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
'pg-0053-plugin-package-workflow-run-list-index',
|
||||
'pg-0054-approval-management-boundary',
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
],
|
||||
});
|
||||
});
|
||||
@@ -783,10 +819,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_admin');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -799,10 +835,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_automation_manager');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
|
||||
const widened = automationManagerPrivileges();
|
||||
@@ -831,10 +867,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_approval_manager');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
|
||||
const widened = approvalManagerPrivileges();
|
||||
@@ -856,6 +892,35 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts the isolated least-privilege Run manager role', async () => {
|
||||
const report = await assertPostgresRunManagerSchemaReady(
|
||||
queryable({
|
||||
currentUser: 'ql3_run_manager',
|
||||
privileges: runManagerPrivileges(),
|
||||
functionMode: 'run-manager',
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_run_manager');
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(report.migrationIds.at(-1), 'pg-0056-run-management-boundary');
|
||||
|
||||
const widened = runManagerPrivileges();
|
||||
widened.find(({ tableName }) => tableName === 'runs').updateAllowed = true;
|
||||
await assert.rejects(
|
||||
assertPostgresRunManagerSchemaReady(
|
||||
queryable({
|
||||
currentUser: 'ql3_run_manager',
|
||||
privileges: widened,
|
||||
functionMode: 'run-manager',
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'run_manager_role_invalid' &&
|
||||
error.facts.includes('table-privileges:runs'),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts isolated Package manager and executor roles', async () => {
|
||||
const manager = await assertPostgresPackageManagerSchemaReady(
|
||||
queryable({
|
||||
@@ -941,10 +1006,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_worker_ingress');
|
||||
assert.equal(report.contractVersion, 54);
|
||||
assert.equal(report.contractVersion, 55);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0055-run-attempt-log-retention',
|
||||
'pg-0056-run-management-boundary',
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -138,21 +138,11 @@ function fixture(options = {}) {
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: 1_000_000 }], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."projects"')) {
|
||||
const rows = options.projectRows ?? [
|
||||
{ projectStatus: 'active', projectVersion: 2 },
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('project_role_bindings')) {
|
||||
const rows = options.bindingRows ?? [
|
||||
{
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: 'operator',
|
||||
},
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
if (normalized.includes('lock_run_management_policy_fence')) {
|
||||
return {
|
||||
rows: [{ matches: options.authorizationMatches ?? true }],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (normalized.includes('idempotency_key = $2')) {
|
||||
const rows = options.replayRows ?? [];
|
||||
@@ -245,7 +235,11 @@ test('atomically appends a linked queued Run, remote Attempt, events and allowed
|
||||
});
|
||||
|
||||
test('returns durable identities for an exact replay without appending again', async () => {
|
||||
const { calls, repository } = fixture({ replayRows: [replayRow()] });
|
||||
const { calls, repository } = fixture({
|
||||
replayRows: [
|
||||
replayRow({ runStatus: 'running', runVersion: 4, eventSequence: 4 }),
|
||||
],
|
||||
});
|
||||
const result = await repository.retryRun(
|
||||
command({
|
||||
runId: '019f9200-0000-4000-8000-000000000102',
|
||||
@@ -257,6 +251,7 @@ test('returns durable identities for an exact replay without appending again', a
|
||||
assert.equal(result.status, 'existing');
|
||||
assert.equal(result.runId, IDS.runId);
|
||||
assert.equal(result.attemptId, IDS.attemptId);
|
||||
assert.equal(result.runStatus, 'queued');
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.startsWith('INSERT INTO')),
|
||||
false,
|
||||
@@ -283,15 +278,11 @@ test('rejects stale authentication and changed authorization inside the transact
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
stale.calls.some(({ sql }) => sql.includes('FROM "ql3"."projects"')),
|
||||
stale.calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')),
|
||||
false,
|
||||
);
|
||||
|
||||
const changed = fixture({
|
||||
bindingRows: [
|
||||
{ bindingVersion: 4, bindingState: 'active', bindingRole: 'operator' },
|
||||
],
|
||||
});
|
||||
const changed = fixture({ authorizationMatches: false });
|
||||
await assert.rejects(
|
||||
changed.repository.retryRun(command()),
|
||||
(error) =>
|
||||
|
||||
@@ -65,6 +65,10 @@ const EXPECTED_ROLES = Object.freeze({
|
||||
connectionLimit: 32,
|
||||
secret: 'ql3-postgres-runtime-auth',
|
||||
}),
|
||||
ql3_run_manager: Object.freeze({
|
||||
connectionLimit: 4,
|
||||
secret: 'ql3-postgres-run-manager-auth',
|
||||
}),
|
||||
ql3_worker_credential_executor: Object.freeze({
|
||||
connectionLimit: 4,
|
||||
secret: 'ql3-postgres-worker-credential-executor-auth',
|
||||
@@ -258,7 +262,7 @@ function assertRolesAndDatabase(readFile, root, findings) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_CNPG_DATABASE_ROLE',
|
||||
'DatabaseRole resources must match the thirteen fixed least-privilege identities',
|
||||
'DatabaseRole resources must match the fourteen fixed least-privilege identities',
|
||||
),
|
||||
);
|
||||
break;
|
||||
@@ -272,7 +276,7 @@ function assertRolesAndDatabase(readFile, root, findings) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_CNPG_DATABASE_ROLE_SET',
|
||||
'exactly the thirteen reviewed migration, runtime, AI maintenance, AI credential management/testing, admin, automation, Approval, package and Worker roles are required',
|
||||
'exactly the fourteen reviewed migration, runtime, AI maintenance, AI credential management/testing, admin, automation, Approval, Run, package and Worker roles are required',
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -332,6 +336,7 @@ function assertSecretBoundary(readFile, root, findings) {
|
||||
['ql3-postgres-admin-auth', 'ql3_admin'],
|
||||
['ql3-postgres-automation-manager-auth', 'ql3_automation_manager'],
|
||||
['ql3-postgres-approval-manager-auth', 'ql3_approval_manager'],
|
||||
['ql3-postgres-run-manager-auth', 'ql3_run_manager'],
|
||||
['ql3-postgres-worker-ingress-auth', 'ql3_worker_ingress'],
|
||||
['ql3-postgres-package-manager-auth', 'ql3_package_manager'],
|
||||
['ql3-postgres-package-executor-auth', 'ql3_package_executor'],
|
||||
@@ -417,7 +422,7 @@ function assertSecretBoundary(readFile, root, findings) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_CNPG_SECRET_EXAMPLE_SET',
|
||||
'the example must describe all thirteen database credentials and the reviewed workload Secrets',
|
||||
'the example must describe all fourteen database credentials and the reviewed workload Secrets',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2366,6 +2366,11 @@ function auditSourceImports(root, packagePath, findings) {
|
||||
'src/approval-management/approvalManagementProcess.ts',
|
||||
].includes(path.relative(packageDirectory, filePath)) &&
|
||||
specifier === '@qinglong/cluster-postgres/approval-manager') ||
|
||||
([
|
||||
'src/run-management/runManagement.ts',
|
||||
'src/run-management/runManagementProcess.ts',
|
||||
].includes(path.relative(packageDirectory, filePath)) &&
|
||||
specifier === '@qinglong/cluster-postgres/run-manager') ||
|
||||
([
|
||||
'src/prompt-output/retention/promptOutputGcCli.ts',
|
||||
'src/prompt-output/retention/promptOutputGcProcess.ts',
|
||||
|
||||
@@ -201,6 +201,9 @@ const {
|
||||
const {
|
||||
assertPostgresApprovalManagerSchemaReady,
|
||||
} = require('../packages/ql3-cluster-postgres/dist/approval-management/index.js');
|
||||
const {
|
||||
assertPostgresRunManagerSchemaReady,
|
||||
} = require('../packages/ql3-cluster-postgres/dist/entrypoints/runManager.js');
|
||||
const {
|
||||
assertPostgresPackageExecutorSchemaReady,
|
||||
PostgresPluginPackageMaterializedRevisionRepository,
|
||||
@@ -341,6 +344,8 @@ const AUTOMATION_MANAGER_USER = 'ql3_automation_manager';
|
||||
const AUTOMATION_MANAGER_PASSWORD = 'ql3_automation_manager_test';
|
||||
const APPROVAL_MANAGER_USER = 'ql3_approval_manager';
|
||||
const APPROVAL_MANAGER_PASSWORD = 'ql3_approval_manager_test';
|
||||
const RUN_MANAGER_USER = 'ql3_run_manager';
|
||||
const RUN_MANAGER_PASSWORD = 'ql3_run_manager_test';
|
||||
const PACKAGE_MANAGER_USER = 'ql3_package_manager';
|
||||
const PACKAGE_MANAGER_PASSWORD = 'ql3_package_manager_test';
|
||||
const PACKAGE_EXECUTOR_USER = 'ql3_package_executor';
|
||||
@@ -3721,6 +3726,9 @@ async function provisionCredentialRoles(database) {
|
||||
await database.pool.query(
|
||||
`CREATE ROLE ${APPROVAL_MANAGER_USER} LOGIN PASSWORD '${APPROVAL_MANAGER_PASSWORD}'`,
|
||||
);
|
||||
await database.pool.query(
|
||||
`CREATE ROLE ${RUN_MANAGER_USER} LOGIN PASSWORD '${RUN_MANAGER_PASSWORD}'`,
|
||||
);
|
||||
await database.pool.query(
|
||||
`CREATE ROLE ${PACKAGE_MANAGER_USER} LOGIN PASSWORD '${PACKAGE_MANAGER_PASSWORD}'`,
|
||||
);
|
||||
@@ -5010,54 +5018,70 @@ async function runIdentityKeysetLedgerMatrix(options) {
|
||||
authority === 'plugin-package-management' ||
|
||||
authority === 'worker-credential-management' ||
|
||||
authority === 'automation-management' ||
|
||||
authority === 'approval-management',
|
||||
authority === 'approval-management' ||
|
||||
authority === 'run-management',
|
||||
);
|
||||
const workerAuthority = authority === 'worker-credential-management';
|
||||
const automationAuthority = authority === 'automation-management';
|
||||
const approvalAuthority = authority === 'approval-management';
|
||||
const user = approvalAuthority
|
||||
const runAuthority = authority === 'run-management';
|
||||
const user = runAuthority
|
||||
? RUN_MANAGER_USER
|
||||
: approvalAuthority
|
||||
? APPROVAL_MANAGER_USER
|
||||
: automationAuthority
|
||||
? AUTOMATION_MANAGER_USER
|
||||
: workerAuthority
|
||||
? WORKER_CREDENTIAL_MANAGER_USER
|
||||
: PACKAGE_MANAGER_USER;
|
||||
const password = approvalAuthority
|
||||
const password = runAuthority
|
||||
? RUN_MANAGER_PASSWORD
|
||||
: approvalAuthority
|
||||
? APPROVAL_MANAGER_PASSWORD
|
||||
: automationAuthority
|
||||
? AUTOMATION_MANAGER_PASSWORD
|
||||
: workerAuthority
|
||||
? WORKER_CREDENTIAL_MANAGER_PASSWORD
|
||||
: PACKAGE_MANAGER_PASSWORD;
|
||||
const role = approvalAuthority
|
||||
const role = runAuthority
|
||||
? 'run-manager'
|
||||
: approvalAuthority
|
||||
? 'approval-manager'
|
||||
: automationAuthority
|
||||
? 'automation-manager'
|
||||
: workerAuthority
|
||||
? 'worker-credential-manager'
|
||||
: 'package-manager';
|
||||
const applicationPrefix = approvalAuthority
|
||||
const applicationPrefix = runAuthority
|
||||
? 'ql3-ha-run-keyset-ledger'
|
||||
: approvalAuthority
|
||||
? 'ql3-ha-approval-keyset-ledger'
|
||||
: automationAuthority
|
||||
? 'ql3-ha-automation-keyset-ledger'
|
||||
: workerAuthority
|
||||
? 'ql3-ha-worker-keyset-ledger'
|
||||
: 'ql3-ha-keyset-ledger';
|
||||
const keyPrefix = approvalAuthority
|
||||
const keyPrefix = runAuthority
|
||||
? 'ha-run-identity-key'
|
||||
: approvalAuthority
|
||||
? 'ha-approval-identity-key'
|
||||
: automationAuthority
|
||||
? 'ha-automation-identity-key'
|
||||
: workerAuthority
|
||||
? 'ha-worker-identity-key'
|
||||
: 'ha-identity-key';
|
||||
const issuer = approvalAuthority
|
||||
const issuer = runAuthority
|
||||
? 'https://run-identity.ha.example.test/'
|
||||
: approvalAuthority
|
||||
? 'https://approval-identity.ha.example.test/'
|
||||
: automationAuthority
|
||||
? 'https://automation-identity.ha.example.test/'
|
||||
: workerAuthority
|
||||
? 'https://worker-identity.ha.example.test/'
|
||||
: 'https://identity.ha.example.test/';
|
||||
const audience = approvalAuthority
|
||||
const audience = runAuthority
|
||||
? 'qinglong3-run-management'
|
||||
: approvalAuthority
|
||||
? 'qinglong3-approval-management'
|
||||
: automationAuthority
|
||||
? 'qinglong3-automation-management'
|
||||
@@ -9446,13 +9470,13 @@ async function runManualRunRetryHaEvidence(options) {
|
||||
})
|
||||
).definition;
|
||||
const firstRuntime = await databaseOpener(
|
||||
'runtime',
|
||||
databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, primaryPort),
|
||||
'run-manager',
|
||||
databaseUrl(RUN_MANAGER_USER, RUN_MANAGER_PASSWORD, primaryPort),
|
||||
'ql3-ha-manual-run-retry-a',
|
||||
)();
|
||||
const secondRuntime = await databaseOpener(
|
||||
'runtime',
|
||||
databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, primaryPort),
|
||||
'run-manager',
|
||||
databaseUrl(RUN_MANAGER_USER, RUN_MANAGER_PASSWORD, primaryPort),
|
||||
'ql3-ha-manual-run-retry-b',
|
||||
)();
|
||||
try {
|
||||
@@ -9707,6 +9731,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
let workerCredentialIdentityKeysetLedger;
|
||||
let automationIdentityKeysetLedger;
|
||||
let approvalIdentityKeysetLedger;
|
||||
let runManagementIdentityKeysetLedger;
|
||||
let automationManagementInspection;
|
||||
let pluginPackageLifecycle;
|
||||
let pluginPackageQuarantine;
|
||||
@@ -9933,6 +9958,11 @@ async function main(argv = process.argv.slice(2)) {
|
||||
),
|
||||
'ql3-ha-approval-manager-readiness',
|
||||
)();
|
||||
const runManagerReadinessDatabase = await databaseOpener(
|
||||
'run-manager',
|
||||
databaseUrl(RUN_MANAGER_USER, RUN_MANAGER_PASSWORD, primaryPort),
|
||||
'ql3-ha-run-manager-readiness',
|
||||
)();
|
||||
const packageManagerReadinessDatabase = await databaseOpener(
|
||||
'package-manager',
|
||||
databaseUrl(PACKAGE_MANAGER_USER, PACKAGE_MANAGER_PASSWORD, primaryPort),
|
||||
@@ -9974,6 +10004,9 @@ async function main(argv = process.argv.slice(2)) {
|
||||
await assertPostgresApprovalManagerSchemaReady(
|
||||
approvalManagerReadinessDatabase.pool,
|
||||
);
|
||||
await assertPostgresRunManagerSchemaReady(
|
||||
runManagerReadinessDatabase.pool,
|
||||
);
|
||||
await assertPostgresPackageManagerSchemaReady(
|
||||
packageManagerReadinessDatabase.pool,
|
||||
);
|
||||
@@ -9992,6 +10025,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
adminReadinessDatabase.close(),
|
||||
automationManagerReadinessDatabase.close(),
|
||||
approvalManagerReadinessDatabase.close(),
|
||||
runManagerReadinessDatabase.close(),
|
||||
packageManagerReadinessDatabase.close(),
|
||||
packageExecutorReadinessDatabase.close(),
|
||||
workerManagerReadinessDatabase.close(),
|
||||
@@ -10031,6 +10065,10 @@ async function main(argv = process.argv.slice(2)) {
|
||||
primaryPort,
|
||||
authority: 'approval-management',
|
||||
});
|
||||
runManagementIdentityKeysetLedger = await runIdentityKeysetLedgerMatrix({
|
||||
primaryPort,
|
||||
authority: 'run-management',
|
||||
});
|
||||
timeline.push({
|
||||
state: 'durable_identity_keyset_ledger_verified',
|
||||
atMs: Number((performance.now() - startedAt).toFixed(3)),
|
||||
@@ -11968,6 +12006,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
workerCredentialIdentityKeysetLedger,
|
||||
automationIdentityKeysetLedger,
|
||||
approvalIdentityKeysetLedger,
|
||||
runManagementIdentityKeysetLedger,
|
||||
automationManagementInspection,
|
||||
pluginPackageLifecycle: pluginPackageLifecycle.report,
|
||||
pluginPackageQuarantine: pluginPackageQuarantine.report,
|
||||
|
||||
@@ -32,6 +32,7 @@ test('accepts the locked CloudNativePG HA and authority profile', () => {
|
||||
'ql3_migration',
|
||||
'ql3_package_executor',
|
||||
'ql3_package_manager',
|
||||
'ql3_run_manager',
|
||||
'ql3_runtime',
|
||||
'ql3_worker_credential_executor',
|
||||
'ql3_worker_credential_manager',
|
||||
|
||||
@@ -340,10 +340,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', (
|
||||
rootSourceFileRoles: clusterAdmin.rootSourceFileRoles,
|
||||
},
|
||||
{
|
||||
sourceFiles: 87,
|
||||
sourceFiles: 94,
|
||||
rootSourceFiles: 1,
|
||||
rootSourceLines: 61,
|
||||
nestedSourceFiles: 86,
|
||||
nestedSourceFiles: 93,
|
||||
rootSourceFileRoles: {
|
||||
'modelInvocationMigrationCli.ts': 'binary_entry',
|
||||
},
|
||||
@@ -421,10 +421,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', (
|
||||
rootSourceFileRoles: clusterPostgres.rootSourceFileRoles,
|
||||
},
|
||||
{
|
||||
sourceFiles: 150,
|
||||
sourceFiles: 152,
|
||||
rootSourceFiles: 1,
|
||||
rootSourceLines: 125,
|
||||
nestedSourceFiles: 149,
|
||||
nestedSourceFiles: 151,
|
||||
rootSourceFileRoles: { 'index.ts': 'public_export' },
|
||||
},
|
||||
);
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const yaml = require('js-yaml');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
|
||||
function document(relativePath) {
|
||||
return yaml.load(fs.readFileSync(path.join(ROOT, relativePath), 'utf8'));
|
||||
}
|
||||
|
||||
test('keeps strong Run management opt-in, private and least-resource', () => {
|
||||
const directory = 'deploy/kubernetes/ql3-cluster/operations/run-management/base';
|
||||
const deployment = document(`${directory}/deployment.yaml`);
|
||||
const service = document(`${directory}/service.yaml`);
|
||||
const policy = document(`${directory}/network-policy.yaml`);
|
||||
const kustomization = document(`${directory}/kustomization.yaml`);
|
||||
const container = deployment.spec.template.spec.containers[0];
|
||||
const environment = new Map(container.env.map((entry) => [entry.name, entry]));
|
||||
|
||||
assert.equal(deployment.metadata.name, 'ql3-run-management');
|
||||
assert.equal(deployment.spec.replicas, 2);
|
||||
assert.equal(deployment.spec.template.spec.automountServiceAccountToken, false);
|
||||
assert.equal(container.securityContext.readOnlyRootFilesystem, true);
|
||||
assert.equal(container.securityContext.allowPrivilegeEscalation, false);
|
||||
assert.deepEqual(container.securityContext.capabilities.drop, ['ALL']);
|
||||
assert.deepEqual(container.command, [
|
||||
'node',
|
||||
'/opt/qinglong/node_modules/@qinglong/cluster-admin/dist/run-management/runManagementCli.js',
|
||||
]);
|
||||
assert.equal(environment.get('QL3_RUN_MANAGEMENT_ENABLED').value, 'true');
|
||||
assert.equal(environment.get('QL3_RUN_MANAGEMENT_PORT').value, '8448');
|
||||
assert.equal(environment.get('QL3_POSTGRES_RUN_MANAGER_POOL_MAX').value, '2');
|
||||
assert.equal(container.resources.requests.memory, '96Mi');
|
||||
assert.equal(service.spec.type, 'ClusterIP');
|
||||
assert.equal(service.spec.ports[0].port, 8448);
|
||||
assert.equal(policy.spec.ingress[0].from[0].podSelector.matchLabels['qinglong.io/run-management-client'], 'true');
|
||||
assert.equal(policy.spec.egress.length, 1);
|
||||
assert.deepEqual(kustomization.resources, [
|
||||
'service-account.yaml',
|
||||
'service.yaml',
|
||||
'deployment.yaml',
|
||||
'pod-disruption-budget.yaml',
|
||||
'network-policy.yaml',
|
||||
]);
|
||||
|
||||
for (const defaultPath of [
|
||||
'deploy/kubernetes/ql3-cluster/base/kustomization.yaml',
|
||||
'deploy/kubernetes/ql3-cluster/overlays/cloudnative-pg/kustomization.yaml',
|
||||
]) {
|
||||
assert.doesNotMatch(fs.readFileSync(path.join(ROOT, defaultPath), 'utf8'), /run-management/);
|
||||
}
|
||||
});
|
||||
|
||||
test('binds the CloudNativePG overlay only to the dedicated Run manager role', () => {
|
||||
const directory = 'deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg';
|
||||
const patch = yaml.load(fs.readFileSync(path.join(ROOT, directory, 'deployment-patch.yaml'), 'utf8'));
|
||||
const environment = new Map(patch[0].value.map((entry) => [entry.name, entry]));
|
||||
const policy = document(`${directory}/network-policy-patch.yaml`);
|
||||
const manifest = JSON.parse(
|
||||
fs.readFileSync(path.join(ROOT, 'packages/ql3-cluster-admin/package.json'), 'utf8'),
|
||||
);
|
||||
|
||||
assert.equal(environment.has('QL3_POSTGRES_RUN_MANAGER_URL'), false);
|
||||
assert.equal(
|
||||
environment.get('QL3_POSTGRES_RUN_MANAGER_USER').valueFrom.secretKeyRef.name,
|
||||
'ql3-postgres-run-manager-auth',
|
||||
);
|
||||
assert.equal(
|
||||
environment.get('QL3_POSTGRES_RUN_MANAGER_PASSWORD').valueFrom.secretKeyRef.name,
|
||||
'ql3-postgres-run-manager-auth',
|
||||
);
|
||||
assert.equal(environment.get('QL3_POSTGRES_RUN_MANAGER_HOST').value, 'ql3-postgres-rw.qinglong3-system.svc');
|
||||
assert.equal(policy.spec.egress.length, 2);
|
||||
assert.deepEqual(policy.spec.egress[1].to[0].podSelector.matchLabels, {
|
||||
'cnpg.io/cluster': 'ql3-postgres',
|
||||
});
|
||||
assert.equal(manifest.bin['ql3-run-manage'], 'dist/run-management/runManagementCli.js');
|
||||
assert.equal(manifest.bin['ql3-run-client'], 'dist/run-management/runManagementClientCli.js');
|
||||
assert.equal(
|
||||
manifest.exports['./run-management-process'].require,
|
||||
'./dist/run-management/runManagementProcess.js',
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user