feat(ql3): add strong cluster run management

This commit is contained in:
whyour
2026-08-12 07:28:43 +08:00
parent e38b143dbb
commit c0ab62e64a
58 changed files with 3087 additions and 105 deletions
@@ -0,0 +1,198 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ql3-run-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
app.kubernetes.io/part-of: qinglong3
spec:
replicas: 2
minReadySeconds: 10
revisionHistoryLimit: 3
progressDeadlineSeconds: 600
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
template:
metadata:
annotations:
qinglong.io/run-management-client-ca-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
qinglong.io/run-management-client-crl-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
labels:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
app.kubernetes.io/part-of: qinglong3
spec:
serviceAccountName: ql3-run-management
automountServiceAccountToken: false
terminationGracePeriodSeconds: 30
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- topologyKey: kubernetes.io/hostname
labelSelector:
matchLabels:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
containers:
- name: management
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/run-management/runManagementCli.js
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: QL3_PROFILE
value: cluster-admin
- name: QL3_RUN_MANAGEMENT_ENABLED
value: 'true'
- name: QL3_RUN_MANAGEMENT_HOST
value: 0.0.0.0
- name: QL3_RUN_MANAGEMENT_PORT
value: '8448'
- name: QL3_RUN_MANAGEMENT_TLS_CERT_FILE
value: /var/run/secrets/qinglong3/run-management-tls/tls.crt
- name: QL3_RUN_MANAGEMENT_TLS_KEY_FILE
value: /var/run/secrets/qinglong3/run-management-tls/tls.key
- name: QL3_RUN_MANAGEMENT_CLIENT_CA_FILE
value: /var/run/secrets/qinglong3/run-management-tls/ca.crt
- name: QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE
value: /var/run/secrets/qinglong3/run-management-tls/client.crl
- name: QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE
value: /var/run/qinglong3/run-management-identity/keyset.json
- name: QL3_RUN_MANAGEMENT_MAX_BODY_BYTES
value: '32768'
- name: QL3_RUN_MANAGEMENT_MAX_CONNECTIONS
value: '32'
- name: QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS
value: '16'
- name: QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS
value: '10000'
- name: QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS
value: '5000'
- name: QL3_RUN_MANAGEMENT_RATE_WINDOW_MS
value: '60000'
- name: QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT
value: '30'
- name: QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT
value: '300'
- name: QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS
value: '1024'
- name: QL3_POSTGRES_RUN_MANAGER_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-run-manager/ca.crt
- name: QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME
value: qinglong3-run-manager
- name: QL3_POSTGRES_RUN_MANAGER_POOL_MAX
value: '2'
- name: QL3_POSTGRES_RUN_MANAGER_URL
valueFrom:
secretKeyRef:
name: ql3-cluster-run-management-database
key: postgres-run-manager-url
- name: QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME
valueFrom:
secretKeyRef:
name: ql3-cluster-run-management-database
key: postgres-tls-servername
ports:
- name: https
containerPort: 8448
protocol: TCP
startupProbe:
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 2
timeoutSeconds: 1
failureThreshold: 30
readinessProbe:
httpGet:
path: /readyz
port: https
scheme: HTTPS
periodSeconds: 5
timeoutSeconds: 2
failureThreshold: 2
livenessProbe:
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
resources:
requests:
cpu: 50m
memory: 96Mi
limits:
cpu: '1'
memory: 384Mi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: management-tls
mountPath: /var/run/secrets/qinglong3/run-management-tls
readOnly: true
- name: management-identity
mountPath: /var/run/qinglong3/run-management-identity
readOnly: true
- name: postgres-run-manager-ca
mountPath: /var/run/secrets/qinglong3/postgres-run-manager
readOnly: true
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 16Mi
- name: management-tls
secret:
secretName: ql3-run-management-tls
defaultMode: 288
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
- key: ca.crt
path: ca.crt
- key: client.crl
path: client.crl
- name: management-identity
secret:
secretName: ql3-run-management-identity
defaultMode: 292
items:
- key: keyset.json
path: keyset.json
- name: postgres-run-manager-ca
secret:
secretName: ql3-cluster-run-management-database
defaultMode: 292
items:
- key: postgres-ca.crt
path: ca.crt
@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- service-account.yaml
- service.yaml
- deployment.yaml
- pod-disruption-budget.yaml
- network-policy.yaml
@@ -0,0 +1,38 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-run-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
app.kubernetes.io/part-of: qinglong3
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
qinglong.io/run-management-client: 'true'
ports:
- protocol: TCP
port: 8448
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
@@ -0,0 +1,15 @@
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: ql3-run-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
app.kubernetes.io/part-of: qinglong3
spec:
minAvailable: 1
selector:
matchLabels:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: ql3-run-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
app.kubernetes.io/part-of: qinglong3
automountServiceAccountToken: false
@@ -0,0 +1,19 @@
apiVersion: v1
kind: Service
metadata:
name: ql3-run-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
app.kubernetes.io/part-of: qinglong3
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: ql3-run-management
app.kubernetes.io/component: run-management
ports:
- name: https
port: 8448
targetPort: https
protocol: TCP
@@ -0,0 +1,71 @@
- op: replace
path: /spec/template/spec/containers/0/env
value:
- name: QL3_PROFILE
value: cluster-admin
- name: QL3_RUN_MANAGEMENT_ENABLED
value: 'true'
- name: QL3_RUN_MANAGEMENT_HOST
value: 0.0.0.0
- name: QL3_RUN_MANAGEMENT_PORT
value: '8448'
- name: QL3_RUN_MANAGEMENT_TLS_CERT_FILE
value: /var/run/secrets/qinglong3/run-management-tls/tls.crt
- name: QL3_RUN_MANAGEMENT_TLS_KEY_FILE
value: /var/run/secrets/qinglong3/run-management-tls/tls.key
- name: QL3_RUN_MANAGEMENT_CLIENT_CA_FILE
value: /var/run/secrets/qinglong3/run-management-tls/ca.crt
- name: QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE
value: /var/run/secrets/qinglong3/run-management-tls/client.crl
- name: QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE
value: /var/run/qinglong3/run-management-identity/keyset.json
- name: QL3_RUN_MANAGEMENT_MAX_BODY_BYTES
value: '32768'
- name: QL3_RUN_MANAGEMENT_MAX_CONNECTIONS
value: '32'
- name: QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS
value: '16'
- name: QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS
value: '10000'
- name: QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS
value: '5000'
- name: QL3_RUN_MANAGEMENT_RATE_WINDOW_MS
value: '60000'
- name: QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT
value: '30'
- name: QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT
value: '300'
- name: QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS
value: '1024'
- name: QL3_POSTGRES_RUN_MANAGER_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-run-manager/ca.crt
- name: QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME
value: qinglong3-run-manager
- name: QL3_POSTGRES_RUN_MANAGER_POOL_MAX
value: '2'
- name: QL3_POSTGRES_RUN_MANAGER_HOST
value: ql3-postgres-rw.qinglong3-system.svc
- name: QL3_POSTGRES_RUN_MANAGER_PORT
value: '5432'
- name: QL3_POSTGRES_RUN_MANAGER_DATABASE
value: qinglong
- name: QL3_POSTGRES_RUN_MANAGER_USER
valueFrom:
secretKeyRef:
name: ql3-postgres-run-manager-auth
key: username
- name: QL3_POSTGRES_RUN_MANAGER_PASSWORD
valueFrom:
secretKeyRef:
name: ql3-postgres-run-manager-auth
key: password
- name: QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME
value: ql3-postgres-rw.qinglong3-system.svc
- op: replace
path: /spec/template/spec/volumes/3/secret/secretName
value: ql3-postgres-ca
- op: replace
path: /spec/template/spec/volumes/3/secret/items/0/key
value: ca.crt
@@ -0,0 +1,25 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../base
patches:
- target:
group: apps
version: v1
kind: Deployment
labelSelector: app.kubernetes.io/name=ql3-run-management
path: deployment-patch.yaml
- target:
group: networking.k8s.io
version: v1
kind: NetworkPolicy
name: ql3-run-management
path: network-policy-patch.yaml
images:
- name: qinglong3-cluster-admin
newName: registry.example.com/qinglong/qinglong3-cluster-admin
# Fail closed until the independently verified release digest is supplied.
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,26 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-run-management
namespace: qinglong3-system
spec:
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
- to:
- podSelector:
matchLabels:
cnpg.io/cluster: ql3-postgres
ports:
- protocol: TCP
port: 5432
@@ -0,0 +1,72 @@
# Example only. Copy into a private overlay and replace every placeholder.
# This file is intentionally excluded from committed Kustomizations.
# Assertions require aud=qinglong3-run-management,
# typ=ql3-run-management+jwt and ql3_purpose=run-management.
apiVersion: v1
kind: Secret
metadata:
name: ql3-run-management-identity
namespace: qinglong3-system
type: Opaque
stringData:
keyset.json: |
{
"schemaVersion": 1,
"generation": 1,
"issuer": "https://identity.example.test/",
"audience": "qinglong3-run-management",
"keys": [
{
"alg": "EdDSA",
"crv": "Ed25519",
"kid": "REPLACE_WITH_KEY_ID",
"kty": "OKP",
"use": "sig",
"x": "REPLACE_WITH_ED25519_PUBLIC_JWK_X"
}
],
"revokedKids": [],
"assuranceMappings": [
{
"acr": "urn:example:mfa",
"assurance": "multi_factor",
"requiredAmr": ["pwd", "otp"]
},
{
"acr": "urn:example:hardware",
"assurance": "hardware",
"requiredAmr": ["hwk"]
}
],
"constraints": {
"maxAssertionBytes": 8192,
"maxLifetimeMs": 300000,
"maxAuthenticationAgeMs": 300000,
"clockSkewMs": 5000
}
}
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-run-management-tls
namespace: qinglong3-system
type: kubernetes.io/tls
stringData:
tls.crt: REPLACE_WITH_SERVER_CERTIFICATE_CHAIN
tls.key: REPLACE_WITH_SERVER_PRIVATE_KEY
ca.crt: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_AUTHORITIES
client.crl: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_REVOCATION_LISTS
---
# Non-CloudNativePG deployments only. The reviewed overlay uses
# ql3-postgres-run-manager-auth and ql3-postgres-ca instead.
apiVersion: v1
kind: Secret
metadata:
name: ql3-cluster-run-management-database
namespace: qinglong3-system
type: Opaque
stringData:
postgres-run-manager-url: REPLACE_WITH_RUN_MANAGER_DSN
postgres-tls-servername: REPLACE_WITH_POSTGRES_DNS_NAME
postgres-ca.crt: REPLACE_WITH_POSTGRES_CA_CERTIFICATE
@@ -122,6 +122,18 @@ stringData:
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-postgres-run-manager-auth
namespace: qinglong3-system
labels:
cnpg.io/reload: 'true'
type: kubernetes.io/basic-auth
stringData:
username: ql3_run_manager
password: REPLACE_WITH_SECRET_MANAGER_VALUE
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-postgres-worker-credential-manager-auth
namespace: qinglong3-system
@@ -240,6 +240,30 @@ spec:
---
apiVersion: postgresql.cnpg.io/v1
kind: DatabaseRole
metadata:
name: ql3-postgres-run-manager
labels:
app.kubernetes.io/name: ql3-postgres
app.kubernetes.io/component: database-role
app.kubernetes.io/part-of: qinglong3
spec:
cluster:
name: ql3-postgres
name: ql3_run_manager
comment: QingLong strongly authenticated human Run retry authority
login: true
superuser: false
createdb: false
createrole: false
replication: false
bypassrls: false
connectionLimit: 4
databaseRoleReclaimPolicy: retain
passwordSecret:
name: ql3-postgres-run-manager-auth
---
apiVersion: postgresql.cnpg.io/v1
kind: DatabaseRole
metadata:
name: ql3-postgres-worker-credential-manager
labels:
+16
View File
@@ -11,6 +11,22 @@
最新增量证据(2026-08-12):
- D-295/ADR-0383(已接受)
Cluster 手动 `run.retry` 已从“仅有 PostgreSQL authority”推进到独立强认证产品面:能力内聚在既有 `@qinglong/cluster-admin`
`run-management/` 目录,不新增 workspace package;只有 `QL3_PROFILE=cluster-admin` 且显式启用时才读取 mTLS/CRL、
purpose-bound OIDC keyset 和创建 PostgreSQL Pool/HTTPS listener。固定 route `/api/v3/runs/management` 同时要求 mTLS 与
`aud=qinglong3-run-management``typ=ql3-run-management+jwt``ql3_purpose=run-management` 的五分钟内
`multi_factor|hardware` User,服务端生成新 Run/Attempt/Event identitykeyset generation 使用独立 durable
`run-management` authority。PostgreSQL capability v55 / migration `pg-0056-run-management-boundary` 引入专用
`ql3_run_manager` 与最小 `SECURITY DEFINER` Project/RoleBinding lock function,产品进程不再使用宽泛的常驻 runtime role;精确
replay 改由 immutable created/queued Events 恢复最初创建事实,因此调度进展后仍返回原 durable identity。Kubernetes 部署作为
`operations/run-management` opt-in overlay 提供两副本/PDB/反亲和/私网 egress,默认 Edge、Standalone 与 Cluster base 均不引用,
低配设备继续保持零新增进程、连接、timer、listener、watcher、cache 或 sidecar。完整 18-package clean build/test 退出 0
backend 1,165 pass/2 conditional skip/0 failworkspace 保持 18 package/1,070 source/1,052 nested,且无 single-source
或 shallow package。PostgreSQL migration ledger 直属文件仍受 58 hard cap 约束,v55 migration 已归入既有 `run-management`
领域目录而非放宽阈值。真实 PostgreSQL 18.4 arm64 physical HA 以两个 `ql3_run_manager` Pool 验证 exact concurrent replay、
最后 quota slot、独立 keyset ledger、同步 WAL 与 promotion 后事实,共 119 gates、timeline `1→2`;报告 SHA-256
`6ca8ccfb48841589e10c6484f5c97ce72e24b123f3abb1066a639e63718e64c6`,离线证据审计无 finding。
- D-294/ADR-0382(已接受)
Cluster 已补齐共享 `qinglong/run-manual-retry@v1` 的 PostgreSQL 原子 authority,但在可信强认证 transport 完成前保持产品 route
关闭。adapter 只接受五分钟内的 `multi_factor|hardware` User,在单个 `SERIALIZABLE` 事务中取得 Project 行锁,重验
@@ -5,6 +5,8 @@
- 关联 RFCQL-RFC-0001 D-294
- 前置决策:ADR-0039、ADR-0047、ADR-0119、ADR-0361、ADR-0381
> 产品装配说明:ADR-0383 已 supersede 本文“复用 `ql3_runtime` 且不新增 role/migration”的决策。本文的共享 retry 语义、原子 repository 与 HA 证据继续有效;生产入口改由强认证的独立 Run Management Plane 和 `ql3_run_manager` 承载。
## 上下文
ADR-0381 已冻结手动 retry 的共享语义并完成 Local SQLite/CLI 纵向切片,但 Cluster 尚缺少可在多副本下工作的 PostgreSQL authority。不能把 Local 的进程内状态或单连接假设复制到 Cluster,也不能让每个 `cluster-control` 副本各自维护限流 bucket。
@@ -0,0 +1,74 @@
# ADR-0383:强认证的 Cluster Run Management Plane
- 状态:Accepted
- 日期:2026-08-12
- 关联 RFCQL-RFC-0001 D-295
- 前置决策:ADR-0039、ADR-0056、ADR-0356、ADR-0364、ADR-0366、ADR-0381、ADR-0382
- SupersedesADR-0382 中“复用 `ql3_runtime` 且不新增 role/migration”的产品装配决策;ADR-0382 的共享语义与 PostgreSQL 原子事务仍有效
## 上下文
ADR-0382 已证明 PostgreSQL 手动 Run retry 的原子语义与 HA 收敛,但现有 Cluster Control bearer 只能建立 `single_factor` User,不能承载会再次执行外部副作用的人工恢复操作。直接把 repository 接进通用 Cluster HTTP 会混合普通控制面与强人类认证 authority;继续使用 `ql3_runtime` 又会使常驻 runtime 持有本不需要的人工恢复权限。
QingLong 同时服务低资源路由设备与多节点集群。新能力不能让 Edge/Standalone 增加进程、连接、timer 或 Cluster 依赖,也不应为了一个内聚领域再制造单文件 workspace package。
## 决策
### 1. 能力归属既有 Cluster Admin package
Run management 作为 `@qinglong/cluster-admin` 内的 `run-management/` 领域目录发布 service、transport、HTTPS process、client 与两个 CLI,不新增 workspace package。PostgreSQL adapter 仍归属 `@qinglong/cluster-postgres/run-manager`。根目录只保留协议装配入口,领域源码不得重新平铺。
### 2. 独立、显式启用的强认证进程
管理端只在 `QL3_PROFILE=cluster-admin``QL3_RUN_MANAGEMENT_ENABLED=true` 时创建资源;关闭时不读取证书、keyset 或数据库配置,也不创建 Pool、listener、timer、watcher 或 cache。服务使用独立 HTTPS 端口 `8448` 和固定 route `/api/v3/runs/management`,同时要求:
1. 受信 client CA 与 CRL 校验的 mTLS
2. purpose-bound OIDC assertion`aud=qinglong3-run-management``typ=ql3-run-management+jwt``ql3_purpose=run-management`
3. 五分钟内的 `multi_factor|hardware` User
4. 精确的 `qinglong/run-manual-retry@v1` command envelope,不接受调用方指定新 Run/Attempt/Event identity
5. route admission 与数据库事务内分别重验 identity/Policy/audit fence。
Assertion keyset generation 使用 durable PostgreSQL ledger 的独立 `run-management` authority,不能与 Plugin、Worker、automation 或 Approval keyset 互换。client 是 caller-driven 的一次性命令,不引入常驻 agent。
### 3. 专用最小权限数据库角色
Migration `pg-0056-run-management-boundary` / capability v55 引入 `ql3_run_manager`。该角色只读 Project/RoleBinding/Task/execution revision,只对 Runs、Attempts、Events、security audit 和本 authority 的 identity ledger 取得精确所需权限;它没有 migration、admin、Worker、AI、Approval 或任意表 DELETE 权限,也没有更新既有 Run 的权限。
`SECURITY DEFINER` 函数 `ql3.lock_run_management_policy_fence` 只负责锁定 Project 并验证 active owner/admin/operator binding,固定 `search_path`,并仅向 `ql3_runtime``ql3_run_manager` 授予 EXECUTE。repository 不以 `FOR UPDATE` 读取源/重放 Run,因此不因 PostgreSQL 锁语法扩大 table UPDATE privilege。
精确 replay 从 immutable created/queued Events 恢复最初创建事实,而不是依赖被重试 Run 当前仍为 queued;调度进展后相同 mutation 仍返回原始 durable identity。
### 4. 部署按 Profile 付费
默认 Edge、Standalone 及 Cluster base overlay 都不引用 Run management manifests。需要该能力的集群显式应用 `operations/run-management`CloudNativePG overlay 使用专用 Secret、primary DNS 与最多两个数据库连接。生产模板为两副本、PDB、反亲和、只读根文件系统、无 ServiceAccount token、受限 ingress 与只到 DNS/PostgreSQL 的 egress。低配单机不承担这些资源。
## 验收
- service/transport/process/client/HTTPS 的单元与回环测试覆盖强认证、purpose isolation、错误映射、限流、drain 与 exact replay
- PostgreSQL package 全量测试覆盖 migration checksum、schema/readiness、role privilege 与调度后 replay
- CloudNativePG 静态审计必须固定十四个角色和非秘密 placeholderRun management deployment 必须证明 opt-in、mTLS、专用 role 与私网 egress
- 真实 PostgreSQL 18.4 HA 必须以迁移后角色完成 readiness、事务写入、同步复制与 promotion 后事实核验;
- 完整 workspace package/backend/dependency/package/Profile artifact 门全部通过,GitNexus staged scope 只包含本阶段预期 flow,才允许提交。
## 被否决的替代方案
1. **接入通用 Cluster Control bearer**:认证强度不足且会扩大普通控制面的恢复执行 authority。
2. **继续让 `ql3_runtime` 作为产品管理角色**:常驻 runtime 无需持有人工 retry 与身份 keyset 写权限。
3. **新增 `@qinglong/run-management` package**:没有独立依赖、制品或部署生命周期收益,会重新产生单文件/浅 package。
4. **每个 Cluster Control Pod 内置管理 listener**:副本与普通 runtime 同扩缩,增加攻击面和低配常驻资源。
5. **允许调用方提交新 aggregate identity**:会扩大 replay/conflict 表面并削弱 server-authored fact。
## 验收证据(2026-08-12
- `@qinglong/cluster-postgres`:309 pass、1 个外部数据库条件 skip、0 fail;`@qinglong/cluster-admin`:281 pass、2 个外部集成条件 skip、0 fail;`@qinglong/cluster-control`230 pass、2 skip、0 fail
- 完整 18-package clean build/test 全部退出 0backend 1,167 项中 1,165 pass、2 个环境条件 skip、0 fail
- workspace 保持 18 个 package,源码为 1,070 个(1,052 个 nested);`singleSourcePackages=[]``shallowSourcePackages=[]`。PostgreSQL ordered migration ledger 直属文件仍为 58v55 migration 归入既有 `run-management` 领域目录,没有放宽 dense-directory cap
- CloudNativePG 静态审计固定 14 个最小权限角色;Run Management deployment 静态门证明默认 overlay 不引用、mTLS/OIDC authority 私有、Pool 上限 2、无 ServiceAccount token、只读根和仅 DNS/PostgreSQL egress
- PostgreSQL 18.4 arm64 physical HA119 gates、timeline `1→2`,两个独立 `ql3_run_manager` Pool 完成 exact concurrent replay 与最后 quota slot 竞争,`run-management` keyset ledger 经双连接、重启、commit-response-loss 验证;报告 SHA-256 `6ca8ccfb48841589e10c6484f5c97ce72e24b123f3abb1066a639e63718e64c6`,离线审计 `compatible:true`
## 影响
- PostgreSQL schema contract 从 v54 升至 v55,生产部署在启用管理面前必须先创建 `ql3_run_manager` 并运行 migration
- Cluster Admin image 增加两个 opt-in binary,但默认 Profile 不启动它们;
- ADR-0382 的原子 repository 和既有 HA 证据继续成立,产品装配改由本 ADR 的强认证进程与专用角色承载;
- UI 仍需以同一 transport 展示 source/new Run linkage、终态原因和 retry preview,不得绕过该 authority。
+1
View File
@@ -386,6 +386,7 @@
| [ADR-0380](./ADR-0380-local-lost-run-retry-single-control-cadence.md) | Local lost Run retry 复用单一 execution-control cadence | Accepted |
| [ADR-0381](./ADR-0381-strong-local-manual-run-retry.md) | 强认证的 Local 手动 Run retry | Accepted |
| [ADR-0382](./ADR-0382-postgresql-cluster-manual-run-retry-authority.md) | PostgreSQL Cluster 手动 Run retry 原子 Authority | Accepted |
| [ADR-0383](./ADR-0383-strong-cluster-run-management-plane.md) | 强认证的 Cluster Run Management Plane 与专用数据库角色 | Accepted |
## 规则
+27
View File
@@ -20,6 +20,31 @@
"require": "./dist/approval-management/approvalDecisionManagement.js",
"default": "./dist/approval-management/approvalDecisionManagement.js"
},
"./run-management": {
"types": "./dist/run-management/runManagement.d.ts",
"require": "./dist/run-management/runManagement.js",
"default": "./dist/run-management/runManagement.js"
},
"./run-management-transport": {
"types": "./dist/run-management/runManagementTransport.d.ts",
"require": "./dist/run-management/runManagementTransport.js",
"default": "./dist/run-management/runManagementTransport.js"
},
"./run-management-http": {
"types": "./dist/run-management/runManagementHttp.d.ts",
"require": "./dist/run-management/runManagementHttp.js",
"default": "./dist/run-management/runManagementHttp.js"
},
"./run-management-process": {
"types": "./dist/run-management/runManagementProcess.d.ts",
"require": "./dist/run-management/runManagementProcess.js",
"default": "./dist/run-management/runManagementProcess.js"
},
"./run-management-client": {
"types": "./dist/run-management/runManagementClient.d.ts",
"require": "./dist/run-management/runManagementClient.js",
"default": "./dist/run-management/runManagementClient.js"
},
"./approval-management": {
"types": "./dist/approval-management/approvalManagement.d.ts",
"require": "./dist/approval-management/approvalManagement.js",
@@ -331,6 +356,8 @@
"ql3-worker-credential-client": "dist/worker-credential/workerCredentialManagementClientCli.js",
"ql3-approval-manage": "dist/approval-management/approvalManagementCli.js",
"ql3-approval-client": "dist/approval-management/approvalManagementClientCli.js",
"ql3-run-manage": "dist/run-management/runManagementCli.js",
"ql3-run-client": "dist/run-management/runManagementClientCli.js",
"ql3-automation-manage": "dist/automation-management/automationManagementCli.js",
"ql3-automation-client": "dist/automation-management/automationManagementClientCli.js",
"ql3-provider-credential-manage": "dist/model-provider-credential/modelProviderCredentialManagementCli.js",
@@ -80,6 +80,11 @@ export const CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PRO
purpose: 'model-provider-credential-management',
});
export const CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE = Object.freeze({
type: 'ql3-run-management+jwt',
purpose: 'run-management',
});
export interface ClusterPluginPackageIdentityAssertionVerifierOptions {
readonly issuer: string;
readonly audience: string;
@@ -9,6 +9,7 @@ import {
CLUSTER_AUTOMATION_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
CLUSTER_APPROVAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
createClusterPluginPackageIdentityAssertionVerifier,
type ClusterManagementIdentityAssertionProfile,
type ClusterPluginPackageIdentityAssertionAuthentication,
@@ -507,3 +508,12 @@ export function createClusterModelProviderCredentialIdentityKeysetFile(
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
});
}
export function createClusterRunIdentityKeysetFile(
options: ClusterWorkerCredentialIdentityKeysetFileOptions,
): Readonly<ClusterPluginPackageIdentityKeysetFile> {
return createClusterPluginPackageIdentityKeysetFile({
...options,
assertionProfile: CLUSTER_RUN_MANAGEMENT_IDENTITY_ASSERTION_PROFILE,
});
}
@@ -67,6 +67,19 @@ import {
ClusterModelProviderCredentialManagementTransportRequestError,
ClusterModelProviderCredentialManagementTransportUnavailableError,
} from '../model-provider-credential/modelProviderCredentialManagementTransport';
import {
ClusterRunManagementAuthorizationError,
ClusterRunManagementConflictError,
ClusterRunManagementRateLimitedError,
ClusterRunManagementRequestError,
ClusterRunManagementTargetUnavailableError,
ClusterRunManagementUnavailableError,
} from '../run-management/runManagement';
import {
ClusterRunManagementTransportAuthenticationError,
ClusterRunManagementTransportRequestError,
ClusterRunManagementTransportUnavailableError,
} from '../run-management/runManagementTransport';
export const CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH =
'/api/v3/plugin-packages/management';
@@ -77,18 +90,21 @@ export const CLUSTER_AUTOMATION_MANAGEMENT_PATH =
export const CLUSTER_APPROVAL_MANAGEMENT_PATH = '/api/v3/approvals/management';
export const CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH =
'/api/v3/provider-credentials/management';
export const CLUSTER_RUN_MANAGEMENT_PATH = '/api/v3/runs/management';
export type ClusterAuthenticatedManagementPath =
| typeof CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH
| typeof CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH
| typeof CLUSTER_AUTOMATION_MANAGEMENT_PATH
| typeof CLUSTER_APPROVAL_MANAGEMENT_PATH
| typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH;
| typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH
| typeof CLUSTER_RUN_MANAGEMENT_PATH;
const MANAGEMENT_PATHS = new Set<ClusterAuthenticatedManagementPath>([
CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH,
CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH,
CLUSTER_AUTOMATION_MANAGEMENT_PATH,
CLUSTER_APPROVAL_MANAGEMENT_PATH,
CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH,
CLUSTER_RUN_MANAGEMENT_PATH,
]);
const DEFAULT_MAX_BODY_BYTES = 64 * 1024;
const DEFAULT_MAX_CONNECTIONS = 64;
@@ -531,7 +547,8 @@ function responseError(error: unknown): HttpRequestError {
error instanceof ClusterApprovalManagementTransportAuthenticationError ||
error instanceof
ClusterModelProviderCredentialManagementTransportAuthenticationError ||
error instanceof ClusterModelProviderCredentialManagementAuthenticationError
error instanceof ClusterModelProviderCredentialManagementAuthenticationError ||
error instanceof ClusterRunManagementTransportAuthenticationError
) {
return new HttpRequestError(401, 'authentication_required');
}
@@ -544,6 +561,8 @@ function responseError(error: unknown): HttpRequestError {
error instanceof
ClusterModelProviderCredentialManagementTransportRequestError ||
error instanceof ClusterModelProviderCredentialManagementRequestError ||
error instanceof ClusterRunManagementTransportRequestError ||
error instanceof ClusterRunManagementRequestError ||
error instanceof PluginPackageManagementRequestError ||
error instanceof WorkerCredentialManagementRequestError
) {
@@ -554,7 +573,8 @@ function responseError(error: unknown): HttpRequestError {
error instanceof WorkerCredentialManagementAuthorizationError ||
error instanceof ClusterAutomationManagementAuthorizationError ||
error instanceof ClusterApprovalManagementTransportAuthorizationError ||
error instanceof ClusterModelProviderCredentialManagementAuthorizationError
error instanceof ClusterModelProviderCredentialManagementAuthorizationError ||
error instanceof ClusterRunManagementAuthorizationError
) {
return new HttpRequestError(403, 'forbidden');
}
@@ -563,7 +583,8 @@ function responseError(error: unknown): HttpRequestError {
error instanceof WorkerCredentialManagementConflictError ||
error instanceof ClusterAutomationManagementConflictError ||
error instanceof ClusterApprovalManagementTransportConflictError ||
error instanceof ClusterModelProviderCredentialManagementConflictError
error instanceof ClusterModelProviderCredentialManagementConflictError ||
error instanceof ClusterRunManagementConflictError
) {
return new HttpRequestError(409, 'conflict');
}
@@ -578,8 +599,12 @@ function responseError(error: unknown): HttpRequestError {
) {
return new HttpRequestError(429, 'quota_exceeded', error.retryAfterMs);
}
if (error instanceof ClusterRunManagementRateLimitedError) {
return new HttpRequestError(429, 'rate_limited', error.retryAfterMs);
}
if (
error instanceof ClusterApprovalManagementTransportTargetUnavailableError
error instanceof ClusterApprovalManagementTransportTargetUnavailableError ||
error instanceof ClusterRunManagementTargetUnavailableError
) {
return new HttpRequestError(404, 'not_found');
}
@@ -594,6 +619,8 @@ function responseError(error: unknown): HttpRequestError {
error instanceof
ClusterModelProviderCredentialManagementTransportUnavailableError ||
error instanceof ClusterModelProviderCredentialManagementUnavailableError ||
error instanceof ClusterRunManagementTransportUnavailableError ||
error instanceof ClusterRunManagementUnavailableError ||
error instanceof PluginPackageManagementUnavailableError ||
error instanceof WorkerCredentialManagementUnavailableError
) {
@@ -0,0 +1,273 @@
import { randomUUID } from 'node:crypto';
import {
PostgresProjectPolicyRepository,
PostgresRunManualRetryRepository,
PostgresSecurityAuditRepository,
} from '@qinglong/cluster-postgres/run-manager';
import type { PostgresPool } from '@qinglong/runtime-core';
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
import {
InvalidRunManualRetryError,
RunManualRetryFenceRejectedError,
RunManualRetryNotFoundError,
RunManualRetryRateLimitedError,
RunManualRetryUnavailableError,
type RunManualRetryResult,
type RunManualRetrySourceStatus,
} from '@qinglong/runtime-core/run-manual-retry';
import {
normalizeSecurityPrincipal,
type SecurityPolicyFence,
type SecurityPrincipal,
} from '@qinglong/runtime-core/security';
import { normalizeSecurityAuditRecord } from '@qinglong/runtime-core/security-audit';
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const UUID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
export interface ClusterRunManagementRetryRequest {
readonly projectId: string;
readonly sourceRunId: string;
readonly mutationId: string;
readonly expectedRunVersion: number;
readonly expectedRunStatus: RunManualRetrySourceStatus;
readonly requestId: string;
readonly auditEventId: string;
readonly failureAuditEventId: string;
readonly principal: Readonly<SecurityPrincipal>;
}
export interface ClusterRunManagementService {
retry(
request: Readonly<ClusterRunManagementRetryRequest>,
): Promise<Readonly<RunManualRetryResult>>;
}
export interface ClusterRunManagementOptions {
readonly pool: PostgresPool;
readonly now?: () => number;
readonly randomUuid?: () => string;
}
export class ClusterRunManagementConfigurationError extends TypeError {
readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFIGURATION_INVALID';
constructor() {
super('Cluster Run management configuration is invalid');
this.name = 'ClusterRunManagementConfigurationError';
}
}
export class ClusterRunManagementRequestError extends TypeError {
readonly code = 'CLUSTER_RUN_MANAGEMENT_REQUEST_INVALID';
constructor() {
super('Cluster Run management request is invalid');
this.name = 'ClusterRunManagementRequestError';
}
}
export class ClusterRunManagementAuthorizationError extends Error {
readonly code = 'CLUSTER_RUN_MANAGEMENT_FORBIDDEN';
constructor() {
super('Cluster Run management is forbidden');
this.name = 'ClusterRunManagementAuthorizationError';
}
}
export class ClusterRunManagementTargetUnavailableError extends Error {
readonly code = 'CLUSTER_RUN_MANAGEMENT_TARGET_UNAVAILABLE';
constructor() {
super('Cluster Run management target is unavailable');
this.name = 'ClusterRunManagementTargetUnavailableError';
}
}
export class ClusterRunManagementConflictError extends Error {
readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFLICT';
constructor() {
super('Cluster Run management conflicts with durable state');
this.name = 'ClusterRunManagementConflictError';
}
}
export class ClusterRunManagementRateLimitedError extends Error {
readonly code = 'CLUSTER_RUN_MANAGEMENT_RATE_LIMITED';
constructor(readonly retryAfterMs: number) {
super('Cluster Run management rate limit is exhausted');
this.name = 'ClusterRunManagementRateLimitedError';
}
}
export class ClusterRunManagementUnavailableError extends Error {
readonly code = 'CLUSTER_RUN_MANAGEMENT_UNAVAILABLE';
constructor(options?: ErrorOptions) {
super('Cluster Run management is unavailable', options);
this.name = 'ClusterRunManagementUnavailableError';
}
}
function exactRequest(
value: unknown,
): asserts value is Readonly<ClusterRunManagementRetryRequest> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
Object.keys(value).sort().join('\0') !==
[
'auditEventId',
'expectedRunStatus',
'expectedRunVersion',
'failureAuditEventId',
'mutationId',
'principal',
'projectId',
'requestId',
'sourceRunId',
]
.sort()
.join('\0')
) {
throw new ClusterRunManagementRequestError();
}
}
function validUuid(value: unknown): value is string {
return typeof value === 'string' && UUID_PATTERN.test(value);
}
function failureReason(error: unknown): string {
if (error instanceof ClusterRunManagementAuthorizationError) {
return 'authorization_rejected';
}
if (error instanceof RunManualRetryNotFoundError) return 'run_not_found';
if (error instanceof RunManualRetryRateLimitedError) return 'rate_limited';
if (error instanceof RunManualRetryFenceRejectedError) return error.reason;
return 'management_unavailable';
}
/** Strong OIDC Run management composition over one run-manager Pool. */
export function createClusterRunManagementService(
options: ClusterRunManagementOptions,
): Readonly<ClusterRunManagementService> {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
Object.keys(options).some(
(key) => key !== 'pool' && key !== 'now' && key !== 'randomUuid',
) ||
!options.pool ||
typeof options.pool.query !== 'function' ||
typeof options.pool.connect !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function') ||
(options.randomUuid !== undefined && typeof options.randomUuid !== 'function')
) {
throw new ClusterRunManagementConfigurationError();
}
const now = options.now ?? Date.now;
const createId = options.randomUuid ?? randomUUID;
const policy = new ProjectPolicyEngine(
new PostgresProjectPolicyRepository(options.pool),
);
const retries = new PostgresRunManualRetryRepository(options.pool);
const audit = new PostgresSecurityAuditRepository(options.pool);
return Object.freeze({
async retry(requestValue: Readonly<ClusterRunManagementRetryRequest>) {
exactRequest(requestValue);
const observedAtMs = now();
let principal: Readonly<SecurityPrincipal>;
if (
!Number.isSafeInteger(observedAtMs) ||
observedAtMs < 0 ||
!IDENTIFIER_PATTERN.test(requestValue.projectId) ||
!IDENTIFIER_PATTERN.test(requestValue.sourceRunId) ||
!IDENTIFIER_PATTERN.test(requestValue.requestId) ||
!validUuid(requestValue.mutationId) ||
!validUuid(requestValue.auditEventId) ||
!validUuid(requestValue.failureAuditEventId) ||
requestValue.auditEventId === requestValue.failureAuditEventId
) {
throw new ClusterRunManagementRequestError();
}
try {
principal = normalizeSecurityPrincipal(
requestValue.principal,
observedAtMs,
);
} catch {
throw new ClusterRunManagementRequestError();
}
let fence: Readonly<SecurityPolicyFence> | null = null;
try {
const decision = await policy.authorize(
principal,
requestValue.projectId,
'run.retry',
);
fence = decision.fence;
if (
decision.effect !== 'allow' ||
!fence ||
fence.bindingVersion === null
) {
throw new ClusterRunManagementAuthorizationError();
}
return await retries.retryRun({
projectId: requestValue.projectId,
sourceRunId: requestValue.sourceRunId,
mutationId: requestValue.mutationId,
expectedRunVersion: requestValue.expectedRunVersion,
expectedRunStatus: requestValue.expectedRunStatus,
runId: createId(),
attemptId: createId(),
createdEventId: createId(),
queuedEventId: createId(),
auditEventId: requestValue.auditEventId,
requestId: requestValue.requestId,
principal,
policyFence: fence,
});
} catch (error) {
try {
await audit.record(
normalizeSecurityAuditRecord({
eventId: requestValue.failureAuditEventId,
requestId: requestValue.requestId,
operationId: 'run.retry',
projectId: requestValue.projectId,
subject: principal.subject,
authenticationId: principal.authenticationId,
outcome: 'denied',
reasons: [failureReason(error)],
fence,
occurredAtMs: observedAtMs,
}),
);
} catch (auditError) {
throw new ClusterRunManagementUnavailableError({ cause: auditError });
}
if (error instanceof ClusterRunManagementAuthorizationError) throw error;
if (error instanceof InvalidRunManualRetryError) {
throw new ClusterRunManagementRequestError();
}
if (error instanceof RunManualRetryNotFoundError) {
throw new ClusterRunManagementTargetUnavailableError();
}
if (error instanceof RunManualRetryFenceRejectedError) {
throw new ClusterRunManagementConflictError();
}
if (error instanceof RunManualRetryRateLimitedError) {
throw new ClusterRunManagementRateLimitedError(error.retryAfterMs);
}
if (error instanceof RunManualRetryUnavailableError) {
throw new ClusterRunManagementUnavailableError({ cause: error });
}
throw new ClusterRunManagementUnavailableError({ cause: error });
}
},
});
}
@@ -0,0 +1,74 @@
#!/usr/bin/env node
import {
startClusterRunManagementProcess,
type ClusterRunManagementProcessRuntime,
} from './runManagementProcess';
const USAGE = 'Usage: ql3-run-manage';
function fact(error: unknown): Readonly<Record<string, unknown>> {
const candidate = error as { readonly name?: unknown; readonly code?: unknown };
return Object.freeze({
schemaVersion: 1,
component: 'qinglong3-run-management',
event: 'management_failed',
name: typeof candidate?.name === 'string' ? candidate.name : 'Error',
...(typeof candidate?.code === 'string' ? { code: candidate.code } : {}),
});
}
function emit(value: Readonly<Record<string, unknown>>): void {
process.stdout.write(`${JSON.stringify(value)}\n`);
}
async function run(argv: readonly string[]): Promise<void> {
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
process.stdout.write(`${USAGE}\n`);
return;
}
if (argv.length !== 0) {
process.stderr.write(`${JSON.stringify({ code: 'QL3_RUN_MANAGEMENT_CLI_USAGE_INVALID', message: USAGE })}\n`);
process.exitCode = 64;
return;
}
let runtime: Readonly<ClusterRunManagementProcessRuntime>;
try {
runtime = await startClusterRunManagementProcess({
environment: process.env,
onError: () => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_unavailable' }),
});
} catch (error) {
process.stderr.write(`${JSON.stringify(fact(error))}\n`);
process.exitCode = 1;
return;
}
if (runtime.status === 'disabled') {
emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_disabled' });
return;
}
emit({
schemaVersion: 1,
component: 'qinglong3-run-management',
event: 'management_started',
address: runtime.address,
identityGeneration: runtime.identity.generation,
databaseContractVersion: runtime.database.contractVersion,
databaseMigrationCount: runtime.database.migrationIds.length,
});
let stopping: Promise<void> | undefined;
const stop = (): Promise<void> => {
stopping ??= runtime.close().then(() => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_stopped' }));
return stopping;
};
for (const signal of ['SIGINT', 'SIGTERM'] as const) {
process.once(signal, () => {
void stop().then(
() => { process.exitCode = 0; },
(error) => { process.stderr.write(`${JSON.stringify(fact(error))}\n`); process.exitCode = 1; },
);
});
}
}
void run(process.argv.slice(2));
@@ -0,0 +1,106 @@
import {
RUN_MANUAL_RETRY_SCHEMA,
normalizeRunManualRetryResult,
} from '@qinglong/runtime-core/run-manual-retry';
import {
ClusterPluginPackageManagementClientRequestError,
executeClusterAuthenticatedManagementClient,
type ClusterAuthenticatedManagementClientResult,
type ClusterPluginPackageManagementClientConnectionOptions,
type ClusterPluginPackageManagementClientPaths,
} from '../management-support/pluginPackageManagementClient';
import {
normalizeClusterRunManagementCommand,
type ClusterRunManagementCommand,
type ClusterRunManagementTransportResult,
} from './runManagementTransport';
const MANAGEMENT_PATH = '/api/v3/runs/management';
export type ClusterRunManagementClientPaths =
ClusterPluginPackageManagementClientPaths;
export type ClusterRunManagementClientConnectionOptions =
ClusterPluginPackageManagementClientConnectionOptions;
export type ClusterRunManagementClientResult =
ClusterAuthenticatedManagementClientResult<ClusterRunManagementTransportResult>;
function invalid(): never {
throw new ClusterPluginPackageManagementClientRequestError();
}
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
const actual = Object.keys(value as object).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
invalid();
}
return value as Record<string, unknown>;
}
export function validateClusterRunManagementClientResult(
value: unknown,
command: Readonly<ClusterRunManagementCommand>,
): Readonly<ClusterRunManagementTransportResult> {
const envelope = exact(value, ['schemaVersion', 'operation', 'retry']);
if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
const retry = exact(envelope.retry, [
'schema',
'status',
'projectId',
'sourceRunId',
'sourceRunStatus',
'sourceRunVersion',
'runId',
'retryOfRunId',
'taskId',
'taskRevision',
'attemptId',
'runStatus',
'runVersion',
'eventSequence',
'executorType',
'executionRevisionDigest',
'createdAtMs',
]);
if (retry.schema !== RUN_MANUAL_RETRY_SCHEMA) invalid();
try {
const { schema: _schema, ...result } = retry;
const normalized = normalizeRunManualRetryResult(result as never);
if (
normalized.projectId !== command.request.projectId ||
normalized.sourceRunId !== command.request.sourceRunId ||
normalized.sourceRunVersion !== command.request.body.expectedRunVersion ||
normalized.sourceRunStatus !== command.request.body.expectedRunStatus ||
normalized.executorType !== 'remote_worker'
) {
invalid();
}
} catch {
invalid();
}
return Object.freeze(
envelope as unknown as ClusterRunManagementTransportResult,
);
}
const PROTOCOL = Object.freeze({
managementPath: MANAGEMENT_PATH,
clientCertificate: 'required' as const,
normalizeCommand: normalizeClusterRunManagementCommand,
validateResult: validateClusterRunManagementClientResult,
});
export function executeClusterRunManagementClient(
paths: ClusterRunManagementClientPaths,
connectionOptions?: ClusterRunManagementClientConnectionOptions,
): Promise<Readonly<ClusterRunManagementClientResult>> {
return executeClusterAuthenticatedManagementClient(
paths,
PROTOCOL,
connectionOptions,
);
}
@@ -0,0 +1,67 @@
#!/usr/bin/env node
import { ClusterPluginPackageManagementClientRemoteError } from '../management-support/pluginPackageManagementClient';
import { executeClusterRunManagementClient } from './runManagementClient';
const USAGE =
'Usage: ql3-run-client --config=/absolute/client.json --command=/absolute/command.json --assertion=/absolute/assertion.jwt';
function argumentsFrom(argv: readonly string[]): Readonly<{
configFile: string;
commandFile: string;
assertionFile: string;
}> | null {
if (argv.length !== 3) return null;
const values = new Map<string, string>();
for (const argument of argv) {
const match = /^--(config|command|assertion)=(\/.+)$/.exec(argument);
if (!match || values.has(match[1]!)) return null;
values.set(match[1]!, match[2]!);
}
if (!values.has('config') || !values.has('command') || !values.has('assertion')) return null;
return Object.freeze({
configFile: values.get('config')!,
commandFile: values.get('command')!,
assertionFile: values.get('assertion')!,
});
}
function failureFact(error: unknown): Readonly<Record<string, unknown>> {
const candidate = error as { readonly code?: unknown };
return Object.freeze({
schemaVersion: 1,
component: 'qinglong3-run-management-client',
event: 'command_failed',
code: typeof candidate?.code === 'string' ? candidate.code : 'QL3_RUN_MANAGEMENT_CLIENT_FAILED',
...(error instanceof ClusterPluginPackageManagementClientRemoteError
? {
statusCode: error.statusCode,
responseCode: error.responseCode,
requestId: error.requestId,
...(error.retryAfterSeconds === null ? {} : { retryAfterSeconds: error.retryAfterSeconds }),
}
: {}),
});
}
async function run(argv: readonly string[]): Promise<void> {
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
process.stdout.write(`${USAGE}\n`);
return;
}
const paths = argumentsFrom(argv);
if (!paths) {
process.stderr.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'usage_invalid', code: 'QL3_RUN_MANAGEMENT_CLIENT_USAGE_INVALID' })}\n`);
process.exitCode = 64;
return;
}
try {
const result = await executeClusterRunManagementClient(paths);
process.stdout.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'command_completed', requestId: result.requestId, result: result.result })}\n`);
} catch (error) {
process.stderr.write(`${JSON.stringify(failureFact(error))}\n`);
process.exitCode = 1;
}
}
void run(process.argv.slice(2));
@@ -0,0 +1,24 @@
import {
CLUSTER_RUN_MANAGEMENT_PATH,
startClusterPluginPackageManagementHttp,
type ClusterPluginPackageManagementHttpApplication,
type StartClusterPluginPackageManagementHttpOptions,
} from '../management-support/pluginPackageManagementHttp';
export type ClusterRunManagementHttpApplication =
ClusterPluginPackageManagementHttpApplication;
export type StartClusterRunManagementHttpOptions = Omit<
StartClusterPluginPackageManagementHttpOptions,
'managementPath'
>;
/** Starts the shared bounded OIDC/mTLS HTTPS adapter on the Run-only path. */
export function startClusterRunManagementHttp(
options: StartClusterRunManagementHttpOptions,
): Promise<Readonly<ClusterRunManagementHttpApplication>> {
return startClusterPluginPackageManagementHttp({
...options,
managementPath: CLUSTER_RUN_MANAGEMENT_PATH,
});
}
@@ -0,0 +1,467 @@
import type {
OpenPostgresDatabase,
PostgresDatabaseResource,
} from '@qinglong/runtime-core';
import {
PostgresRunManagementIdentityKeysetLedgerRepository,
assertPostgresRunManagerSchemaReady,
createPostgresDatabaseOpener,
isPostgresTlsDnsServername,
loadPostgresCertificateAuthorityFile,
loadPostgresConnectionEnvironment,
type PostgresConnectionOptions,
type PostgresPoolOptions,
type PostgresSchemaReadinessReport,
} from '@qinglong/cluster-postgres/run-manager';
import {
absoluteManagementEnvironmentFile,
booleanManagementEnvironmentValue,
boundedManagementEnvironmentValue,
integerManagementEnvironmentValue,
readManagementTlsFile,
} from '../management-support/managementProcessSupport';
import {
createClusterRunIdentityKeysetFile,
type ClusterPluginPackageIdentityKeysetFile,
type ClusterPluginPackageIdentityKeysetSnapshot,
} from '../management-support/pluginPackageIdentityKeyset';
import { validateClusterManagementClientTrust } from '../worker-credential/management-server/workerCredentialManagementMutualTls';
import { createClusterRunManagementService } from './runManagement';
import {
startClusterRunManagementHttp,
type ClusterRunManagementHttpApplication,
type StartClusterRunManagementHttpOptions,
} from './runManagementHttp';
import { createClusterRunManagementTransport } from './runManagementTransport';
const SAFE_HOST = /^[A-Za-z0-9][A-Za-z0-9.:-]{0,254}$/;
const SAFE_APPLICATION_NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,62}$/;
export type ClusterRunManagementProcessEnvironment = Readonly<
Record<string, string | undefined>
>;
export type ClusterRunManagementProcessConfig =
| Readonly<{ enabled: false }>
| Readonly<{
enabled: true;
profile: 'cluster-admin';
host: string;
port: number;
certificateFile: string;
privateKeyFile: string;
clientCertificateAuthorityFile: string;
clientCertificateRevocationListFile: string;
identityKeysetFile: string;
http: Readonly<{
maxBodyBytes: number;
maxConnections: number;
maxConcurrentRequests: number;
requestTimeoutMs: number;
drainTimeoutMs: number;
rateWindowMs: number;
peerRequestLimit: number;
globalRequestLimit: number;
maxRateLimitPeers: number;
}>;
database: Readonly<{
connection: PostgresConnectionOptions;
pool: PostgresPoolOptions;
}>;
}>;
export type ClusterRunManagementProcessRuntime =
| Readonly<{ status: 'disabled'; close(): Promise<void> }>
| Readonly<{
status: 'active';
address: Readonly<{ host: string; port: number }>;
database: PostgresSchemaReadinessReport;
identity: ClusterPluginPackageIdentityKeysetSnapshot;
availabilityStatus(): 'ready' | 'unavailable' | 'stopped';
close(): Promise<void>;
}>;
export interface StartClusterRunManagementProcessOptions {
readonly environment: ClusterRunManagementProcessEnvironment;
readonly openDatabase?: OpenPostgresDatabase;
readonly identities?: ClusterPluginPackageIdentityKeysetFile;
readonly assertReady?: (
pool: PostgresDatabaseResource['pool'],
) => Promise<PostgresSchemaReadinessReport>;
readonly startHttp?: (
options: StartClusterRunManagementHttpOptions,
) => Promise<Readonly<ClusterRunManagementHttpApplication>>;
readonly now?: () => number;
readonly randomUuid?: () => string;
readonly onError?: (error: unknown) => void;
}
export class ClusterRunManagementProcessConfigError extends TypeError {
readonly code = 'QL3_RUN_MANAGEMENT_PROCESS_CONFIG_INVALID';
constructor(message: string) {
super(`Run management process configuration is invalid: ${message}`);
this.name = 'ClusterRunManagementProcessConfigError';
}
}
function failure(message: string): ClusterRunManagementProcessConfigError {
return new ClusterRunManagementProcessConfigError(message);
}
function bounded(
environment: ClusterRunManagementProcessEnvironment,
name: string,
maximumLength: number,
required = false,
): string | undefined {
return boundedManagementEnvironmentValue(
environment,
name,
maximumLength,
failure,
required,
);
}
function bool(
environment: ClusterRunManagementProcessEnvironment,
name: string,
): boolean {
return booleanManagementEnvironmentValue(environment, name, failure);
}
function integer(
environment: ClusterRunManagementProcessEnvironment,
name: string,
fallback: number,
minimum: number,
maximum: number,
): number {
return integerManagementEnvironmentValue(
environment,
name,
fallback,
minimum,
maximum,
failure,
);
}
function absolute(
environment: ClusterRunManagementProcessEnvironment,
name: string,
): string {
return absoluteManagementEnvironmentFile(environment, name, failure);
}
function loadDatabase(
environment: ClusterRunManagementProcessEnvironment,
): Readonly<{
connection: PostgresConnectionOptions;
pool: PostgresPoolOptions;
}> {
let connection: PostgresConnectionOptions;
try {
connection = loadPostgresConnectionEnvironment(environment, {
connectionString: 'QL3_POSTGRES_RUN_MANAGER_URL',
host: 'QL3_POSTGRES_RUN_MANAGER_HOST',
port: 'QL3_POSTGRES_RUN_MANAGER_PORT',
database: 'QL3_POSTGRES_RUN_MANAGER_DATABASE',
user: 'QL3_POSTGRES_RUN_MANAGER_USER',
password: 'QL3_POSTGRES_RUN_MANAGER_PASSWORD',
});
} catch (error) {
throw failure(
error instanceof Error
? error.message
: 'PostgreSQL run manager connection is invalid',
);
}
const mode = environment.QL3_POSTGRES_RUN_MANAGER_TLS_MODE ?? 'verify-full';
if (mode !== 'verify-full' && mode !== 'disable') {
throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_MODE must be verify-full or disable');
}
if (
mode === 'disable' &&
!bool(environment, 'QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE')
) {
throw failure(
'disabling run manager PostgreSQL TLS requires QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE=true',
);
}
const servername = bounded(
environment,
'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME',
253,
);
if (mode === 'verify-full' && !isPostgresTlsDnsServername(servername)) {
throw failure(
'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME must be an explicit DNS name',
);
}
const caFile = bounded(
environment,
'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE',
4_096,
);
if (mode === 'disable' && caFile !== undefined) {
throw failure(
'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE cannot be used when TLS is disabled',
);
}
let ca: string | undefined;
if (caFile !== undefined) {
try {
ca = loadPostgresCertificateAuthorityFile(caFile);
} catch {
throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE is invalid');
}
}
const applicationName =
bounded(environment, 'QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME', 63) ??
'qinglong3-run-manager';
if (!SAFE_APPLICATION_NAME.test(applicationName)) {
throw failure('QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME is invalid');
}
return Object.freeze({
connection: Object.freeze({
...connection,
tls:
mode === 'disable'
? Object.freeze({ mode: 'disable' as const })
: Object.freeze({
mode: 'verify-full' as const,
servername: servername!,
...(ca === undefined ? {} : { ca }),
}),
}),
pool: Object.freeze({
applicationName,
maxConnections: integer(
environment,
'QL3_POSTGRES_RUN_MANAGER_POOL_MAX',
2,
1,
4,
),
idleTimeoutMs: integer(
environment,
'QL3_POSTGRES_RUN_MANAGER_IDLE_TIMEOUT_MS',
10_000,
1_000,
60_000,
),
connectionTimeoutMs: integer(
environment,
'QL3_POSTGRES_RUN_MANAGER_CONNECTION_TIMEOUT_MS',
5_000,
100,
60_000,
),
}),
});
}
export function loadClusterRunManagementProcessConfig(
environment: ClusterRunManagementProcessEnvironment,
): Readonly<ClusterRunManagementProcessConfig> {
if (!environment || typeof environment !== 'object') {
throw failure('environment is invalid');
}
if (!bool(environment, 'QL3_RUN_MANAGEMENT_ENABLED')) {
return Object.freeze({ enabled: false as const });
}
if (environment.QL3_PROFILE !== 'cluster-admin') {
throw failure('QL3_PROFILE must be cluster-admin when Run management is enabled');
}
const host = bounded(environment, 'QL3_RUN_MANAGEMENT_HOST', 255) ?? '0.0.0.0';
if (!SAFE_HOST.test(host)) throw failure('QL3_RUN_MANAGEMENT_HOST is invalid');
const http = Object.freeze({
maxBodyBytes: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_BODY_BYTES', 32 * 1024, 1_024, 256 * 1024),
maxConnections: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONNECTIONS', 32, 1, 512),
maxConcurrentRequests: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS', 16, 1, 256),
requestTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS', 10_000, 1_000, 60_000),
drainTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS', 5_000, 100, 60_000),
rateWindowMs: integer(environment, 'QL3_RUN_MANAGEMENT_RATE_WINDOW_MS', 60_000, 1_000, 5 * 60_000),
peerRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT', 30, 1, 10_000),
globalRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT', 300, 1, 100_000),
maxRateLimitPeers: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS', 1_024, 1, 16_384),
});
if (http.globalRequestLimit < http.peerRequestLimit) {
throw failure('global request limit cannot be below peer request limit');
}
return Object.freeze({
enabled: true as const,
profile: 'cluster-admin' as const,
host,
port: integer(environment, 'QL3_RUN_MANAGEMENT_PORT', 8_448, 1, 65_535),
certificateFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_CERT_FILE'),
privateKeyFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_KEY_FILE'),
clientCertificateAuthorityFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CA_FILE'),
clientCertificateRevocationListFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE'),
identityKeysetFile: absolute(environment, 'QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE'),
http,
database: loadDatabase(environment),
});
}
export async function startClusterRunManagementProcess(
options: StartClusterRunManagementProcessOptions,
): Promise<Readonly<ClusterRunManagementProcessRuntime>> {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
Object.keys(options).some(
(key) =>
![
'environment',
'openDatabase',
'identities',
'assertReady',
'startHttp',
'now',
'randomUuid',
'onError',
].includes(key),
) ||
!options.environment ||
typeof options.environment !== 'object' ||
(options.openDatabase !== undefined && typeof options.openDatabase !== 'function') ||
(options.identities !== undefined &&
(typeof options.identities.reload !== 'function' ||
typeof options.identities.bind !== 'function')) ||
(options.assertReady !== undefined && typeof options.assertReady !== 'function') ||
(options.startHttp !== undefined && typeof options.startHttp !== 'function') ||
(options.now !== undefined && typeof options.now !== 'function') ||
(options.randomUuid !== undefined && typeof options.randomUuid !== 'function') ||
(options.onError !== undefined && typeof options.onError !== 'function')
) {
throw failure('options are invalid');
}
const config = loadClusterRunManagementProcessConfig(options.environment);
if (!config.enabled) {
return Object.freeze({ status: 'disabled' as const, close: () => Promise.resolve() });
}
const now = options.now ?? Date.now;
let http: Readonly<ClusterRunManagementHttpApplication> | undefined;
let database: PostgresDatabaseResource | undefined;
let unavailableError: unknown;
let closePromise: Promise<void> | undefined;
const report = (error: unknown): void => {
try {
options.onError?.(error);
} catch {
// Diagnostics never own availability or cleanup.
}
};
const openDatabase =
options.openDatabase ??
createPostgresDatabaseOpener({
role: 'run-manager',
connection: config.database.connection,
pool: config.database.pool,
onPoolError(error) {
const first = unavailableError === undefined;
unavailableError ??= error;
http?.withdraw(error);
if (first) report(error);
},
});
try {
database = await openDatabase();
const evidence = await (options.assertReady ?? assertPostgresRunManagerSchemaReady)(database.pool);
if (unavailableError !== undefined) throw unavailableError;
const identities =
options.identities ??
createClusterRunIdentityKeysetFile({
filePath: config.identityKeysetFile,
now,
ledger: new PostgresRunManagementIdentityKeysetLedgerRepository(
database.pool,
'run-management',
),
});
const identity = await identities.reload();
const service = createClusterRunManagementService({
pool: database.pool,
now,
...(options.randomUuid === undefined ? {} : { randomUuid: options.randomUuid }),
});
const transport = createClusterRunManagementTransport({ service, now });
const privateKey = readManagementTlsFile(config.privateKeyFile, true, failure);
try {
const certificate = readManagementTlsFile(config.certificateFile, false, failure);
const clientCertificateAuthority = readManagementTlsFile(
config.clientCertificateAuthorityFile,
false,
failure,
);
const clientCertificateRevocationList = readManagementTlsFile(
config.clientCertificateRevocationListFile,
false,
failure,
);
validateClusterManagementClientTrust(
clientCertificateAuthority,
clientCertificateRevocationList,
now(),
failure,
);
http = await (options.startHttp ?? startClusterRunManagementHttp)({
host: config.host,
port: config.port,
tls: {
privateKey,
certificate,
clientCertificateAuthority,
clientCertificateRevocationList,
},
transport,
identities,
limits: config.http,
now,
onError: report,
});
} finally {
privateKey.fill(0);
}
if (unavailableError !== undefined) http.withdraw(unavailableError);
return Object.freeze({
status: 'active' as const,
address: http.address,
database: evidence,
identity,
availabilityStatus: () => http?.availabilityStatus() ?? 'stopped',
close(): Promise<void> {
closePromise ??= (async () => {
let primaryError: unknown;
try {
await http?.close();
} catch (error) {
primaryError = error;
}
try {
await database?.close();
} catch (error) {
primaryError ??= error;
}
if (primaryError) throw primaryError;
})();
return closePromise;
},
});
} catch (error) {
try {
await http?.close();
} catch {
// Preserve startup failure.
}
try {
await database?.close();
} catch {
// Preserve startup failure.
}
throw error;
}
}
@@ -0,0 +1,214 @@
import {
createRunManualRetryResponseBody,
parseRunManualRetryRequestBody,
type RunManualRetryResponseBody,
} from '@qinglong/runtime-core/run-manual-retry';
import {
normalizeSecurityPrincipal,
type SecurityPrincipal,
} from '@qinglong/runtime-core/security';
import type { ClusterRunManagementService } from './runManagement';
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const UUID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
export type ClusterRunManagementCommand = Readonly<{
schemaVersion: 1;
operation: 'run.retry';
request: Readonly<{
projectId: string;
sourceRunId: string;
requestId: string;
auditEventId: string;
failureAuditEventId: string;
body: Readonly<{
schema: 'qinglong/run-manual-retry@v1';
mutationId: string;
expectedRunVersion: number;
expectedRunStatus: 'failed' | 'cancelled' | 'timed_out';
}>;
}>;
}>;
export type ClusterRunManagementTransportResult = Readonly<{
schemaVersion: 1;
operation: 'run.retry';
retry: Readonly<RunManualRetryResponseBody>;
}>;
export interface ClusterRunManagementAuthentication {
authenticate(): Promise<Readonly<SecurityPrincipal> | null>;
}
export interface ClusterRunManagementTransport {
execute(
command: unknown,
authentication: ClusterRunManagementAuthentication,
): Promise<Readonly<ClusterRunManagementTransportResult>>;
}
export class ClusterRunManagementTransportConfigurationError extends TypeError {
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_CONFIGURATION_INVALID';
constructor() {
super('Cluster Run management transport configuration is invalid');
this.name = 'ClusterRunManagementTransportConfigurationError';
}
}
export class ClusterRunManagementTransportRequestError extends TypeError {
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_REQUEST_INVALID';
constructor() {
super('Cluster Run management transport request is invalid');
this.name = 'ClusterRunManagementTransportRequestError';
}
}
export class ClusterRunManagementTransportAuthenticationError extends Error {
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_AUTHENTICATION_REQUIRED';
constructor() {
super('Cluster Run management transport requires a strong User principal');
this.name = 'ClusterRunManagementTransportAuthenticationError';
}
}
export class ClusterRunManagementTransportUnavailableError extends Error {
readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_UNAVAILABLE';
constructor() {
super('Cluster Run management transport is unavailable');
this.name = 'ClusterRunManagementTransportUnavailableError';
}
}
function invalid(): never {
throw new ClusterRunManagementTransportRequestError();
}
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
const actual = Object.keys(value as object).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
invalid();
}
return value as Record<string, unknown>;
}
function identifier(value: unknown): string {
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) invalid();
return value;
}
function uuid(value: unknown): string {
if (typeof value !== 'string' || !UUID_PATTERN.test(value)) invalid();
return value;
}
export function normalizeClusterRunManagementCommand(
value: unknown,
): Readonly<ClusterRunManagementCommand> {
const envelope = exact(value, ['schemaVersion', 'operation', 'request']);
if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
const request = exact(envelope.request, [
'projectId',
'sourceRunId',
'requestId',
'auditEventId',
'failureAuditEventId',
'body',
]);
let body: ReturnType<typeof parseRunManualRetryRequestBody>;
try {
body = parseRunManualRetryRequestBody(request.body);
} catch {
invalid();
}
const auditEventId = uuid(request.auditEventId);
const failureAuditEventId = uuid(request.failureAuditEventId);
if (auditEventId === failureAuditEventId) invalid();
return Object.freeze({
schemaVersion: 1,
operation: 'run.retry',
request: Object.freeze({
projectId: identifier(request.projectId),
sourceRunId: identifier(request.sourceRunId),
requestId: identifier(request.requestId),
auditEventId,
failureAuditEventId,
body,
}),
});
}
export function createClusterRunManagementTransport(options: Readonly<{
service: ClusterRunManagementService;
now?: () => number;
}>): Readonly<ClusterRunManagementTransport> {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
Object.keys(options).some((key) => key !== 'service' && key !== 'now') ||
!options.service ||
typeof options.service.retry !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function')
) {
throw new ClusterRunManagementTransportConfigurationError();
}
const now = options.now ?? Date.now;
return Object.freeze({
async execute(
commandValue: unknown,
authentication: ClusterRunManagementAuthentication,
) {
const command = normalizeClusterRunManagementCommand(commandValue);
if (
!authentication ||
typeof authentication !== 'object' ||
Array.isArray(authentication) ||
Object.keys(authentication).length !== 1 ||
typeof authentication.authenticate !== 'function'
) {
throw new ClusterRunManagementTransportConfigurationError();
}
let candidate: Readonly<SecurityPrincipal> | null;
try {
candidate = await authentication.authenticate();
} catch {
throw new ClusterRunManagementTransportUnavailableError();
}
let principal: Readonly<SecurityPrincipal>;
try {
principal = normalizeSecurityPrincipal(candidate as SecurityPrincipal, now());
} catch {
throw new ClusterRunManagementTransportAuthenticationError();
}
if (
principal.subject.type !== 'user' ||
!STRONG_ASSURANCES.has(principal.assurance)
) {
throw new ClusterRunManagementTransportAuthenticationError();
}
const result = await options.service.retry({
projectId: command.request.projectId,
sourceRunId: command.request.sourceRunId,
mutationId: command.request.body.mutationId,
expectedRunVersion: command.request.body.expectedRunVersion,
expectedRunStatus: command.request.body.expectedRunStatus,
requestId: command.request.requestId,
auditEventId: command.request.auditEventId,
failureAuditEventId: command.request.failureAuditEventId,
principal,
});
return Object.freeze({
schemaVersion: 1,
operation: 'run.retry',
retry: createRunManualRetryResponseBody(result),
});
},
});
}
@@ -163,6 +163,7 @@ function database(serverVersionNum = '160014') {
'enforce_plugin_package_stage_provenance',
'lock_active_plugin_package_project',
'lock_approval_policy_fence',
'lock_run_management_policy_fence',
'plugin_package_lifecycle_blocking_runs',
'plugin_package_automation_start_allowed',
'plugin_package_run_start_allowed',
@@ -12,6 +12,7 @@ const {
createClusterAutomationIdentityKeysetFile,
createClusterApprovalIdentityKeysetFile,
createClusterModelProviderCredentialIdentityKeysetFile,
createClusterRunIdentityKeysetFile,
} = require('@qinglong/cluster-admin/plugin-package-identity-keyset');
const NOW_MS = 1_700_000_000_000;
@@ -216,6 +217,38 @@ function providerCredentialAssertion(key, overrides = {}) {
).toString('base64url')}`;
}
function runAssertion(key, overrides = {}) {
const header = Buffer.from(
JSON.stringify({
alg: 'EdDSA',
kid: key.kid,
typ: 'ql3-run-management+jwt',
}),
).toString('base64url');
const now = Math.floor(NOW_MS / 1000);
const payload = Buffer.from(
JSON.stringify({
acr: 'urn:ql3:mfa',
amr: ['pwd', 'otp'],
aud: 'qinglong3-run-management',
auth_time: now - 10,
exp: now + 120,
iat: now,
iss: ISSUER,
jti: `run-assertion-${key.kid}`,
ql3_purpose: 'run-management',
sub: 'run-operator-1',
...overrides,
}),
).toString('base64url');
const signed = `${header}.${payload}`;
return `${signed}.${sign(
null,
Buffer.from(signed, 'ascii'),
key.privateKey,
).toString('base64url')}`;
}
async function atomicWrite(filePath, document) {
const nextPath = `${filePath}.next`;
await writeFile(nextPath, `${JSON.stringify(document)}\n`, { mode: 0o644 });
@@ -374,6 +407,34 @@ test('loads a provider credential keyset isolated by type, purpose and audience'
});
});
test('loads a Run keyset isolated from every other management purpose', async () => {
await fixture(async ({ filePath }) => {
const key = reviewedKey('run-identity-key-1');
await atomicWrite(filePath, {
...keyset(1, [key]),
audience: 'qinglong3-run-management',
});
const provider = createClusterRunIdentityKeysetFile({
filePath,
now: () => NOW_MS,
});
const principal = await provider.bind(runAssertion(key)).authenticate();
assert.deepEqual(principal.subject, {
type: 'user',
id: 'run-operator-1',
});
await assert.rejects(provider.bind(approvalAssertion(key)).authenticate(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID',
});
await assert.rejects(
provider
.bind(runAssertion(key, { ql3_purpose: 'approval-management' }))
.authenticate(),
{ code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
);
});
});
test('supports overlap rotation then immediately revokes the previous key', async () => {
await fixture(async ({ filePath }) => {
const first = reviewedKey('issuer-key-1');
@@ -197,6 +197,7 @@ function database(serverVersionNum = '160014') {
'plugin_package_tool_start_allowed',
'plugin_package_workflow_admission_snapshot',
'plugin_package_workflow_task_attempt_snapshot',
'lock_run_management_policy_fence',
].includes(functionName),
isOwner: false,
})),
@@ -0,0 +1,164 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ClusterRunManagementAuthorizationError,
createClusterRunManagementService,
} = require('@qinglong/cluster-admin/run-management');
const NOW = 1_000_000;
const SOURCE_DIGEST = 'a'.repeat(64);
const EXECUTION_DIGEST = 'b'.repeat(64);
const TASK_REVISION = `qltd:v1:7:${SOURCE_DIGEST}`;
const GENERATED = [
'019f9500-0000-4000-8000-000000000010',
'019f9500-0000-4000-8000-000000000011',
'019f9500-0000-4000-8000-000000000012',
'019f9500-0000-4000-8000-000000000013',
];
function request() {
return {
projectId: 'project-1',
sourceRunId: 'source-run-1',
mutationId: '019f9500-0000-4000-8000-000000000001',
expectedRunVersion: 7,
expectedRunStatus: 'failed',
requestId: 'request-1',
auditEventId: '019f9500-0000-4000-8000-000000000002',
failureAuditEventId: '019f9500-0000-4000-8000-000000000003',
principal: {
subject: { type: 'user', id: 'operator-1' },
authenticationId: 'oidc:run-management-1',
authenticatedAtMs: 999_000,
expiresAtMs: 1_100_000,
assurance: 'multi_factor',
},
};
}
function policyRow(role = 'operator') {
return {
projectId: 'project-1',
projectName: 'Project 1',
projectSlug: 'project-1',
projectStatus: 'active',
projectVersion: 2,
projectCreatedAtMs: '1',
projectUpdatedAtMs: '2',
bindingProjectId: 'project-1',
bindingSubjectType: 'user',
bindingSubjectId: 'operator-1',
bindingVersion: 3,
bindingState: 'active',
bindingRole: role,
bindingMutationId: 'binding-3',
bindingChangedByType: 'user',
bindingChangedById: 'owner-1',
bindingCreatedAtMs: '3',
};
}
function fixture(role = 'operator') {
const calls = [];
const pool = {
async query(sql, params = []) {
const text = sql.replace(/\s+/g, ' ').trim();
calls.push({ scope: 'pool', sql: text, params });
if (text.includes('LEFT JOIN LATERAL')) return { rows: [policyRow(role)] };
if (text.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
return { rows: [], rowCount: 1 };
}
throw new Error(`unexpected pool query: ${text}`);
},
async connect() {
return {
async query(sql, params = []) {
const text = sql.replace(/\s+/g, ' ').trim();
calls.push({ scope: 'client', sql: text, params });
if (
text === 'BEGIN ISOLATION LEVEL SERIALIZABLE' ||
text === 'COMMIT' ||
text === 'ROLLBACK' ||
text.startsWith('SELECT set_config')
) return { rows: [], rowCount: 0 };
if (text.includes('statement_timestamp()')) {
return { rows: [{ nowMs: NOW }], rowCount: 1 };
}
if (text.includes('lock_run_management_policy_fence')) {
return { rows: [{ matches: true }], rowCount: 1 };
}
if (text.includes('idempotency_key = $2')) return { rows: [] };
if (text.includes('WHERE run.id = $1')) {
return {
rows: [{
projectId: 'project-1',
taskId: 'task-1',
taskRevision: TASK_REVISION,
taskName: 'Task 1',
taskSnapshotRef: TASK_REVISION,
parentRunId: null,
triggerType: 'task_start',
executionOwner: 'runtime',
inputRef: null,
priority: 1,
runStatus: 'failed',
runVersion: 7,
attemptExecutorType: 'remote_worker',
}],
};
}
if (text.includes('FROM "ql3"."task_definitions"')) {
return { rows: [{ enabled: true }] };
}
if (text.includes('task_execution_revisions')) {
return { rows: [{ sourceContentDigest: SOURCE_DIGEST, contentDigest: EXECUTION_DIGEST }] };
}
if (text.startsWith('SELECT') && text.includes("trigger_type = 'run_manual_retry'")) {
return { rows: [] };
}
if (text.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
throw new Error(`unexpected client query: ${text}`);
},
release() {},
};
},
};
let index = 0;
return {
calls,
service: createClusterRunManagementService({
pool,
now: () => NOW,
randomUuid: () => GENERATED[index++],
}),
};
}
test('authorizes run.retry and keeps all generated aggregate identities server-side', async () => {
const { calls, service } = fixture();
const result = await service.retry(request());
assert.equal(result.status, 'accepted');
assert.equal(result.runId, GENERATED[0]);
assert.equal(result.attemptId, GENERATED[1]);
const runInsert = calls.find(({ sql }) => sql.startsWith('INSERT INTO "ql3"."runs"'));
assert.equal(runInsert.params[0], GENERATED[0]);
assert.equal(runInsert.params.includes(GENERATED[2]), false);
assert.equal(
calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')),
true,
);
});
test('denied policy writes only the caller-supplied failure audit', async () => {
const { calls, service } = fixture('viewer');
await assert.rejects(service.retry(request()), ClusterRunManagementAuthorizationError);
const audits = calls.filter(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
);
assert.equal(audits.length, 1);
assert.equal(audits[0].params[0], request().failureAuditEventId);
assert.equal(calls.some(({ scope }) => scope === 'client'), false);
});
@@ -0,0 +1,77 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
validateClusterRunManagementClientResult,
} = require('@qinglong/cluster-admin/run-management-client');
const {
normalizeClusterRunManagementCommand,
} = require('@qinglong/cluster-admin/run-management-transport');
const {
ClusterPluginPackageManagementClientRequestError,
} = require('@qinglong/cluster-admin/plugin-package-management-client');
const command = normalizeClusterRunManagementCommand({
schemaVersion: 1,
operation: 'run.retry',
request: {
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'request-1',
auditEventId: '019f9400-0000-4000-8000-000000000001',
failureAuditEventId: '019f9400-0000-4000-8000-000000000002',
body: {
schema: 'qinglong/run-manual-retry@v1',
mutationId: '019f9400-0000-4000-8000-000000000003',
expectedRunVersion: 7,
expectedRunStatus: 'failed',
},
},
});
function response(overrides = {}) {
return {
schemaVersion: 1,
operation: 'run.retry',
retry: {
schema: 'qinglong/run-manual-retry@v1',
status: 'accepted',
projectId: 'project-1',
sourceRunId: 'source-run-1',
sourceRunStatus: 'failed',
sourceRunVersion: 7,
runId: '019f9400-0000-4000-8000-000000000010',
retryOfRunId: 'source-run-1',
taskId: 'task-1',
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
attemptId: '019f9400-0000-4000-8000-000000000011',
runStatus: 'queued',
runVersion: 2,
eventSequence: 2,
executorType: 'remote_worker',
executionRevisionDigest: 'b'.repeat(64),
createdAtMs: 1_000_000,
...overrides,
},
};
}
test('validates one low-sensitive retry response against the request fence', () => {
assert.deepEqual(validateClusterRunManagementClientResult(response(), command), response());
});
test('rejects response target, execution placement and shape drift', () => {
for (const candidate of [
response({ projectId: 'project-2' }),
response({ executorType: 'local_process' }),
response({ sourceRunVersion: 8 }),
{ ...response(), principal: { type: 'user', id: 'operator-1' } },
]) {
assert.throws(
() => validateClusterRunManagementClientResult(candidate, command),
ClusterPluginPackageManagementClientRequestError,
);
}
});
@@ -0,0 +1,108 @@
'use strict';
const assert = require('node:assert/strict');
const { readFileSync } = require('node:fs');
const { request: httpsRequest } = require('node:https');
const { resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterRunManagementRateLimitedError,
} = require('@qinglong/cluster-admin/run-management');
const {
startClusterRunManagementHttp,
} = require('@qinglong/cluster-admin/run-management-http');
const SERVER_KEY = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-key.pem');
const SERVER_CERT = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem');
const PATH = '/api/v3/runs/management';
function post(port, path = PATH) {
const body = Buffer.from(JSON.stringify({
schemaVersion: 1,
operation: 'run.retry',
request: {
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'request-1',
auditEventId: '019f9600-0000-4000-8000-000000000001',
failureAuditEventId: '019f9600-0000-4000-8000-000000000002',
body: {
schema: 'qinglong/run-manual-retry@v1',
mutationId: '019f9600-0000-4000-8000-000000000003',
expectedRunVersion: 7,
expectedRunStatus: 'failed',
},
},
}));
return new Promise((resolvePromise, reject) => {
const outgoing = httpsRequest({
host: '127.0.0.1',
port,
path,
method: 'POST',
rejectUnauthorized: false,
agent: false,
headers: {
authorization: 'Bearer assertion',
'content-type': 'application/json',
'content-length': String(body.length),
},
}, (incoming) => {
const chunks = [];
incoming.on('data', (chunk) => chunks.push(chunk));
incoming.once('end', () => {
const bytes = Buffer.concat(chunks);
resolvePromise({
statusCode: incoming.statusCode,
headers: incoming.headers,
body: bytes.length ? JSON.parse(bytes.toString('utf8')) : null,
});
});
});
outgoing.once('error', reject);
outgoing.end(body);
});
}
test('serves only the Run path and maps durable quota to bounded HTTP facts', async () => {
const application = await startClusterRunManagementHttp({
host: '127.0.0.1',
port: 0,
tls: {
privateKey: Buffer.from(readFileSync(SERVER_KEY)),
certificate: Buffer.from(readFileSync(SERVER_CERT)),
},
identities: {
async reload() { throw new Error('not used'); },
bind() {
return { authenticate: async () => ({
subject: { type: 'user', id: 'operator-1' },
authenticationId: 'oidc:run-management-1',
authenticatedAtMs: 900,
expiresAtMs: 10_000,
assurance: 'hardware',
}) };
},
},
transport: {
async execute(_command, authentication) {
await authentication.authenticate();
throw new ClusterRunManagementRateLimitedError(1_500);
},
},
now: () => 1_000,
});
try {
const limited = await post(application.address.port);
assert.equal(limited.statusCode, 429);
assert.equal(limited.headers['retry-after'], '2');
assert.equal(limited.body.error.code, 'rate_limited');
assert.match(limited.body.requestId, /^[0-9a-f-]{36}$/);
const absent = await post(application.address.port, '/api/v3/approvals/management');
assert.equal(absent.statusCode, 404);
assert.equal(absent.body.error.code, 'not_found');
} finally {
await application.close();
}
});
@@ -0,0 +1,72 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ClusterRunManagementProcessConfigError,
loadClusterRunManagementProcessConfig,
startClusterRunManagementProcess,
} = require('@qinglong/cluster-admin/run-management-process');
function enabled(overrides = {}) {
return {
QL3_RUN_MANAGEMENT_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
QL3_RUN_MANAGEMENT_TLS_CERT_FILE: '/run/ql3/run/tls.crt',
QL3_RUN_MANAGEMENT_TLS_KEY_FILE: '/run/ql3/run/tls.key',
QL3_RUN_MANAGEMENT_CLIENT_CA_FILE: '/run/ql3/run/client-ca.crt',
QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE: '/run/ql3/run/client.crl',
QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE: '/run/ql3/run/identity.json',
QL3_POSTGRES_RUN_MANAGER_HOST: 'postgres.qinglong3-system.svc',
QL3_POSTGRES_RUN_MANAGER_DATABASE: 'qinglong3',
QL3_POSTGRES_RUN_MANAGER_USER: 'ql3_run_manager',
QL3_POSTGRES_RUN_MANAGER_PASSWORD: 'secret',
QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: 'postgres.qinglong3-system.svc',
...overrides,
};
}
test('disabled Run manager acquires no PostgreSQL or file authority', async () => {
let opened = false;
const runtime = await startClusterRunManagementProcess({
environment: {},
openDatabase: async () => {
opened = true;
throw new Error('must not open');
},
});
assert.equal(runtime.status, 'disabled');
assert.equal(opened, false);
await runtime.close();
});
test('loads a bounded opt-in Run-only process configuration', () => {
const config = loadClusterRunManagementProcessConfig(enabled());
assert.equal(config.enabled, true);
assert.equal(config.port, 8448);
assert.equal(config.http.maxConcurrentRequests, 16);
assert.equal(config.database.pool.maxConnections, 2);
assert.equal(config.database.connection.user, 'ql3_run_manager');
assert.deepEqual(config.database.connection.tls, {
mode: 'verify-full',
servername: 'postgres.qinglong3-system.svc',
});
});
test('rejects profile drift and implicit insecure PostgreSQL', () => {
assert.throws(
() => loadClusterRunManagementProcessConfig(enabled({ QL3_PROFILE: 'cluster-control' })),
ClusterRunManagementProcessConfigError,
);
assert.throws(
() =>
loadClusterRunManagementProcessConfig(
enabled({
QL3_POSTGRES_RUN_MANAGER_TLS_MODE: 'disable',
QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: undefined,
}),
),
ClusterRunManagementProcessConfigError,
);
});
@@ -0,0 +1,141 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ClusterRunManagementTransportAuthenticationError,
ClusterRunManagementTransportRequestError,
createClusterRunManagementTransport,
normalizeClusterRunManagementCommand,
} = require('@qinglong/cluster-admin/run-management-transport');
const NOW = 1_000_000;
function principal(overrides = {}) {
return {
subject: { type: 'user', id: 'operator-1' },
authenticationId: 'oidc:run-management-1',
authenticatedAtMs: 999_000,
expiresAtMs: 1_100_000,
assurance: 'multi_factor',
...overrides,
};
}
function command(overrides = {}) {
return {
schemaVersion: 1,
operation: 'run.retry',
request: {
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'request-1',
auditEventId: '019f9300-0000-4000-8000-000000000001',
failureAuditEventId: '019f9300-0000-4000-8000-000000000002',
body: {
schema: 'qinglong/run-manual-retry@v1',
mutationId: '019f9300-0000-4000-8000-000000000003',
expectedRunVersion: 7,
expectedRunStatus: 'failed',
},
...overrides,
},
};
}
function retryResult() {
return {
status: 'accepted',
projectId: 'project-1',
sourceRunId: 'source-run-1',
sourceRunStatus: 'failed',
sourceRunVersion: 7,
runId: '019f9300-0000-4000-8000-000000000010',
retryOfRunId: 'source-run-1',
taskId: 'task-1',
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
attemptId: '019f9300-0000-4000-8000-000000000011',
runStatus: 'queued',
runVersion: 2,
eventSequence: 2,
executorType: 'remote_worker',
executionRevisionDigest: 'b'.repeat(64),
createdAtMs: NOW,
};
}
test('routes one exact strong User retry and emits the shared response', async () => {
const calls = [];
const transport = createClusterRunManagementTransport({
now: () => NOW,
service: {
async retry(request) {
calls.push(request);
return retryResult();
},
},
});
const result = await transport.execute(command(), {
authenticate: async () => principal(),
});
assert.equal(calls.length, 1);
assert.equal(calls[0].mutationId, command().request.body.mutationId);
assert.equal(calls[0].principal.assurance, 'multi_factor');
assert.deepEqual(result, {
schemaVersion: 1,
operation: 'run.retry',
retry: {
schema: 'qinglong/run-manual-retry@v1',
...retryResult(),
},
});
});
test('rejects weak or non-User identity before service authority', async () => {
let called = false;
const transport = createClusterRunManagementTransport({
now: () => NOW,
service: {
async retry() {
called = true;
return retryResult();
},
},
});
await assert.rejects(
transport.execute(command(), {
authenticate: async () => principal({ assurance: 'single_factor' }),
}),
ClusterRunManagementTransportAuthenticationError,
);
await assert.rejects(
transport.execute(command(), {
authenticate: async () =>
principal({ subject: { type: 'agent', id: 'agent-1' } }),
}),
ClusterRunManagementTransportAuthenticationError,
);
assert.equal(called, false);
});
test('rejects widened commands and ambiguous audit identity', () => {
assert.throws(
() => normalizeClusterRunManagementCommand({ ...command(), principal: principal() }),
ClusterRunManagementTransportRequestError,
);
assert.throws(
() =>
normalizeClusterRunManagementCommand(
command({ failureAuditEventId: command().request.auditEventId }),
),
ClusterRunManagementTransportRequestError,
);
assert.throws(
() =>
normalizeClusterRunManagementCommand(
command({ body: { ...command().request.body, expectedRunStatus: 'lost' } }),
),
ClusterRunManagementTransportRequestError,
);
});
@@ -334,6 +334,7 @@ function databaseResource(events, options = {}) {
'plugin_package_workflow_task_attempt_snapshot',
'plugin_package_run_start_allowed',
'plugin_package_tool_start_allowed',
'lock_run_management_policy_fence',
].includes(functionName),
isOwner: false,
})),
@@ -243,6 +243,7 @@ function databaseResource(events, overrides = {}) {
'plugin_package_workflow_task_attempt_snapshot',
'plugin_package_run_start_allowed',
'plugin_package_tool_start_allowed',
'lock_run_management_policy_fence',
].includes(functionName),
isOwner: false,
})),
@@ -75,6 +75,11 @@
"require": "./dist/run-management/runManualRetryRepository.js",
"default": "./dist/run-management/runManualRetryRepository.js"
},
"./run-manager": {
"types": "./dist/entrypoints/runManager.d.ts",
"require": "./dist/entrypoints/runManager.js",
"default": "./dist/entrypoints/runManager.js"
},
"./approval-manager": {
"types": "./dist/approval-management/index.d.ts",
"require": "./dist/approval-management/index.js",
@@ -19,6 +19,7 @@ const DEFAULT_AI_CREDENTIAL_TESTER_APPLICATION_NAME =
const DEFAULT_AUTOMATION_MANAGER_APPLICATION_NAME =
'qinglong-automation-manager';
const DEFAULT_APPROVAL_MANAGER_APPLICATION_NAME = 'qinglong-approval-manager';
const DEFAULT_RUN_MANAGER_APPLICATION_NAME = 'qinglong-run-manager';
const DEFAULT_PACKAGE_MANAGER_APPLICATION_NAME = 'qinglong-package-manager';
const DEFAULT_PACKAGE_EXECUTOR_APPLICATION_NAME = 'qinglong-package-executor';
const DEFAULT_WORKER_CREDENTIAL_MANAGER_APPLICATION_NAME =
@@ -89,6 +90,7 @@ export type PostgresDatabaseRole =
| 'admin'
| 'automation-manager'
| 'approval-manager'
| 'run-manager'
| 'package-manager'
| 'package-executor'
| 'worker-credential-manager'
@@ -255,6 +257,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
'admin',
'automation-manager',
'approval-manager',
'run-manager',
'package-manager',
'package-executor',
'worker-credential-manager',
@@ -296,6 +299,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
const isAdmin = options.role === 'admin';
const isAutomationManager = options.role === 'automation-manager';
const isApprovalManager = options.role === 'approval-manager';
const isRunManager = options.role === 'run-manager';
const isPackageManager = options.role === 'package-manager';
const isPackageExecutor = options.role === 'package-executor';
const isWorkerCredentialManager =
@@ -309,6 +313,7 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
isAdmin ||
isAutomationManager ||
isApprovalManager ||
isRunManager ||
isPackageManager ||
isPackageExecutor ||
isWorkerCredentialManager ||
@@ -370,6 +375,8 @@ function buildPoolConfig(options: OpenPostgresDatabaseOptions): PoolConfig {
? DEFAULT_AUTOMATION_MANAGER_APPLICATION_NAME
: isApprovalManager
? DEFAULT_APPROVAL_MANAGER_APPLICATION_NAME
: isRunManager
? DEFAULT_RUN_MANAGER_APPLICATION_NAME
: isPackageManager
? DEFAULT_PACKAGE_MANAGER_APPLICATION_NAME
: isPackageExecutor
@@ -0,0 +1,31 @@
export { PostgresRunManualRetryRepository } from '../run-management/runManualRetryRepository';
export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository';
export { PostgresSecurityAuditRepository } from '../security/securityAuditRepository';
export {
PostgresPluginPackageIdentityKeysetLedgerRepository as PostgresRunManagementIdentityKeysetLedgerRepository,
type ClusterManagementIdentityAuthority,
} from '../management/pluginPackageIdentityKeysetLedgerRepository';
export {
PgPoolBinding,
createPostgresDatabaseOpener,
isPostgresTlsDnsServername,
type OpenPostgresDatabaseOptions,
type PostgresConnectionOptions,
type PostgresDatabaseRole,
type PostgresPoolOptions,
} from '../connection/pool';
export {
PostgresConnectionEnvironmentError,
loadPostgresConnectionEnvironment,
type PostgresConnectionEnvironment,
type PostgresConnectionEnvironmentKeys,
} from '../connection/connectionEnvironment';
export {
loadPostgresCertificateAuthorityFile,
type PostgresCertificateAuthorityFileInspection,
} from '../connection/certificateAuthority';
export {
PostgresSchemaReadinessError,
assertPostgresRunManagerSchemaReady,
type PostgresSchemaReadinessReport,
} from '../schema/schemaReadiness';
@@ -5,7 +5,8 @@ export type ClusterManagementIdentityAuthority =
| 'plugin-package-management'
| 'worker-credential-management'
| 'automation-management'
| 'approval-management';
| 'approval-management'
| 'run-management';
const DIGEST_PATTERN = /^[A-Za-z0-9_-]{43}$/;
const KEY_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
@@ -189,7 +190,8 @@ export class PostgresPluginPackageIdentityKeysetLedgerRepository
authority !== 'plugin-package-management' &&
authority !== 'worker-credential-management' &&
authority !== 'automation-management' &&
authority !== 'approval-management'
authority !== 'approval-management' &&
authority !== 'run-management'
) {
throw new TypeError(
'PostgreSQL management identity keyset authority is invalid',
@@ -283,5 +283,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest =
checksum:
'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30',
}),
Object.freeze({
id: 'pg-0056-run-management-boundary',
checksum:
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
}),
]),
});
@@ -58,6 +58,7 @@ import { pg0052AutomationManagementIdentityKeysetLedgerMigration } from './pg-00
import { pg0053PluginPackageWorkflowRunListIndexMigration } from './pg-0053-plugin-package-workflow-run-list-index';
import { pg0054ApprovalManagementBoundaryMigration } from './pg-0054-approval-management-boundary';
import { pg0055RunAttemptLogRetentionMigration } from './pg-0055-run-attempt-log-retention';
import { pg0056RunManagementBoundaryMigration } from '../run-management/pg-0056-run-management-boundary';
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
Object.freeze({
@@ -121,5 +122,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMi
pg0053PluginPackageWorkflowRunListIndexMigration,
pg0054ApprovalManagementBoundaryMigration,
pg0055RunAttemptLogRetentionMigration,
pg0056RunManagementBoundaryMigration,
]),
});
@@ -0,0 +1,122 @@
import { CAPABILITIES_V54 } from '../migrations/pg-0055-run-attempt-log-retention';
import { definePostgresSqlMigration } from '../migrations/sqlMigration';
export const CAPABILITIES_V55 = CAPABILITIES_V54.replace(
'"run_core":1,',
'"run_management_boundary":1,"run_core":1,',
);
export const pg0056RunManagementBoundaryMigration =
definePostgresSqlMigration({
id: 'pg-0056-run-management-boundary',
statements: [
`
DO $ql3$
DECLARE
role_invalid boolean;
BEGIN
SELECT roles.rolname IS NULL
OR roles.rolcanlogin IS NOT TRUE
OR roles.rolsuper IS NOT FALSE
OR roles.rolcreatedb IS NOT FALSE
OR roles.rolcreaterole IS NOT FALSE
OR roles.rolreplication IS NOT FALSE
OR roles.rolbypassrls IS NOT FALSE
INTO role_invalid
FROM (SELECT 1) AS expected
LEFT JOIN pg_catalog.pg_roles AS roles
ON roles.rolname = 'ql3_run_manager';
IF role_invalid IS NOT FALSE THEN
RAISE EXCEPTION
'required QingLong run manager role is missing or privileged'
USING ERRCODE = 'insufficient_privilege';
END IF;
END
$ql3$
`.trim(),
`
DO $ql3$
BEGIN
EXECUTE format(
'GRANT CONNECT ON DATABASE %I TO ql3_run_manager',
current_database()
);
END
$ql3$
`.trim(),
`GRANT USAGE ON SCHEMA "ql3" TO ql3_run_manager`,
`REVOKE ALL ON ALL TABLES IN SCHEMA "ql3" FROM ql3_run_manager`,
`REVOKE ALL ON ALL FUNCTIONS IN SCHEMA "ql3" FROM ql3_run_manager`,
`
CREATE FUNCTION "ql3"."lock_run_management_policy_fence"(
varchar,
varchar,
varchar,
integer,
integer
)
RETURNS boolean
LANGUAGE plpgsql
VOLATILE
SECURITY DEFINER
SET search_path = pg_catalog, ql3
AS $ql3$
DECLARE
project_status varchar;
project_version integer;
binding_state varchar;
binding_role varchar;
binding_version integer;
BEGIN
SELECT project.status, project.version
INTO project_status, project_version
FROM "ql3"."projects" AS project
WHERE project.id = $1
FOR UPDATE;
SELECT binding.state, binding.role, binding.version
INTO binding_state, binding_role, binding_version
FROM "ql3"."project_role_bindings" AS binding
WHERE binding.project_id = $1
AND binding.subject_type = $2
AND binding.subject_id = $3
ORDER BY binding.version DESC
LIMIT 1;
RETURN project_status = 'active'
AND project_version = $4
AND binding_state = 'active'
AND binding_role IN ('owner', 'admin', 'operator')
AND binding_version = $5;
END
$ql3$
`.trim(),
`REVOKE ALL ON FUNCTION "ql3"."lock_run_management_policy_fence"(varchar, varchar, varchar, integer, integer) FROM PUBLIC`,
`GRANT EXECUTE ON FUNCTION "ql3"."lock_run_management_policy_fence"(varchar, varchar, varchar, integer, integer) TO ql3_runtime, ql3_run_manager`,
`GRANT SELECT ON "ql3"."schema_migrations", "ql3"."schema_capabilities", "ql3"."projects", "ql3"."project_role_bindings", "ql3"."task_definitions", "ql3"."task_definition_revisions", "ql3"."task_execution_revisions" TO ql3_run_manager`,
`GRANT SELECT, INSERT ON "ql3"."runs", "ql3"."run_attempts", "ql3"."run_events", "ql3"."security_audit_events" TO ql3_run_manager`,
`GRANT SELECT, INSERT, UPDATE ON "ql3"."plugin_package_identity_keyset_ledger" TO ql3_run_manager`,
`ALTER TABLE "ql3"."plugin_package_identity_keyset_ledger" DROP CONSTRAINT ql3_plugin_package_identity_keyset_authority_check`,
`ALTER TABLE "ql3"."plugin_package_identity_keyset_ledger" ADD CONSTRAINT ql3_plugin_package_identity_keyset_authority_check CHECK (authority IN ('plugin-package-management', 'worker-credential-management', 'automation-management', 'approval-management', 'run-management'))`,
`
DO $ql3$
BEGIN
UPDATE "ql3"."schema_capabilities"
SET contract_version = 55,
migration_id = 'pg-0056-run-management-boundary',
capabilities = '${CAPABILITIES_V55}'::jsonb,
updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint
WHERE contract_name = 'control-core'
AND contract_version = 54
AND migration_id = 'pg-0055-run-attempt-log-retention'
AND capabilities = '${CAPABILITIES_V54}'::jsonb;
IF NOT FOUND THEN
RAISE EXCEPTION 'control-core capability is not at version 54'
USING ERRCODE = 'check_violation';
END IF;
END
$ql3$
`.trim(),
],
});
@@ -9,7 +9,6 @@ import {
RunManualRetryUnavailableError,
normalizeRunManualRetryCommand,
normalizeRunManualRetryResult,
type RunManualRetryAllowedRole,
type RunManualRetryCommand,
type RunManualRetryRepository,
type RunManualRetryResult,
@@ -33,11 +32,6 @@ type Row = Record<string, unknown>;
export const CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS = 60_000;
export const CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT = 64;
const ALLOWED_ROLES = new Set<RunManualRetryAllowedRole>([
'owner',
'admin',
'operator',
]);
const CLUSTER_STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
const TASK_REVISION_PATTERN = /^qltd:v1:([1-9]\d*):([0-9a-f]{64})$/;
@@ -128,45 +122,23 @@ async function confirmAuthorization(
client: PostgresClient,
command: Readonly<RunManualRetryCommand>,
): Promise<void> {
const project = await client.query<Row>(
const result = await client.query<Row>(
`
SELECT status AS "projectStatus", version AS "projectVersion"
FROM "ql3"."projects" WHERE id = $1 FOR UPDATE
`,
[command.projectId],
);
if (project.rows.length === 0) throw new RunManualRetryNotFoundError();
if (project.rows.length !== 1) throw unavailable();
// Authorized management mutations take the same Project lock. Keeping this
// append-only RoleBinding read lock-free avoids granting UPDATE authority to
// the runtime role merely to use PostgreSQL row-lock syntax.
const binding = await client.query<Row>(
`
SELECT version AS "bindingVersion", state AS "bindingState",
role AS "bindingRole"
FROM "ql3"."project_role_bindings"
WHERE project_id = $1 AND subject_type = $2 AND subject_id = $3
ORDER BY version DESC LIMIT 1
SELECT "ql3"."lock_run_management_policy_fence"(
$1::varchar, $2::varchar, $3::varchar, $4::integer, $5::integer
) AS "matches"
`,
[
command.projectId,
command.principal.subject.type,
command.principal.subject.id,
command.policyFence.projectVersion,
command.policyFence.bindingVersion,
],
);
const currentProject = project.rows[0]!;
const currentBinding = binding.rows[0];
if (
text(currentProject, 'projectStatus') !== 'active' ||
integer(currentProject, 'projectVersion') !==
command.policyFence.projectVersion ||
!currentBinding ||
integer(currentBinding, 'bindingVersion') !==
command.policyFence.bindingVersion ||
text(currentBinding, 'bindingState') !== 'active' ||
!ALLOWED_ROLES.has(
text(currentBinding, 'bindingRole') as RunManualRetryAllowedRole,
)
result.rows.length !== 1 ||
!postgresRequiredBoolean(result.rows[0]!.matches, unavailable)
) {
throw new RunManualRetryFenceRejectedError('authorization_changed');
}
@@ -185,8 +157,6 @@ async function findReplay(
run.execution_origin AS "executionOrigin",
run.execution_owner AS "executionOwner",
run.triggered_by AS "triggeredBy", run.request_id AS "requestId",
run.status AS "runStatus", run.version AS "runVersion",
run.event_sequence AS "eventSequence",
run.created_at_ms AS "createdAtMs",
attempt.id AS "attemptId", attempt.executor_type AS "executorType",
created.actor_type AS "createdActorType",
@@ -205,7 +175,6 @@ async function findReplay(
ON queued.run_id = run.id AND queued.sequence = 2
AND queued.type = 'run.queued'
WHERE run.project_id = $1 AND run.idempotency_key = $2
FOR UPDATE OF run
`,
[command.projectId, `ql3:run-manual-retry:v1:${command.mutationId}`],
);
@@ -227,9 +196,6 @@ function replayResult(
text(row, 'executionOwner') !== 'runtime' ||
text(row, 'triggeredBy') !== command.principal.subject.id ||
text(row, 'requestId') !== command.mutationId ||
text(row, 'runStatus') !== 'queued' ||
integer(row, 'runVersion') !== 2 ||
integer(row, 'eventSequence') !== 2 ||
text(row, 'executorType') !== 'remote_worker' ||
text(row, 'createdActorType') !== command.principal.subject.type ||
text(row, 'createdActorId') !== command.principal.subject.id ||
@@ -295,7 +261,6 @@ async function findSource(
LIMIT 1
) AS attempt ON true
WHERE run.id = $1
FOR UPDATE OF run
`,
[command.sourceRunId],
);
@@ -15,13 +15,14 @@ export interface PostgresSchemaContractFunction {
export interface PostgresSchemaContract {
readonly schema: 'ql3';
readonly contractName: 'control-core';
readonly contractVersion: 54;
readonly migrationId: 'pg-0055-run-attempt-log-retention';
readonly contractVersion: 55;
readonly migrationId: 'pg-0056-run-management-boundary';
readonly minimumServerMajor: 16;
readonly maximumServerMajor: 18;
readonly capabilities: Readonly<{
run_core: 1;
run_attempt_log_retention: 1;
run_management_boundary: 1;
run_dispatch_lease: 1;
run_retry_policy: 1;
project_policy: 1;
@@ -101,8 +102,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
Object.freeze({
schema: 'ql3',
contractName: 'control-core',
contractVersion: 54,
migrationId: 'pg-0055-run-attempt-log-retention',
contractVersion: 55,
migrationId: 'pg-0056-run-management-boundary',
minimumServerMajor: 16,
maximumServerMajor: 18,
capabilities: Object.freeze({
@@ -143,6 +144,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
project_tool_definition_snapshot: 1,
run_core: 1,
run_attempt_log_retention: 1,
run_management_boundary: 1,
run_dispatch_lease: 1,
run_retry_policy: 1,
security_audit: 1,
@@ -2323,6 +2325,15 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
volatility: 'volatile',
configuration: Object.freeze(['search_path=pg_catalog, ql3']),
}),
Object.freeze({
name: 'lock_run_management_policy_fence',
identityArguments:
'character varying, character varying, character varying, integer, integer',
owner: 'ql3_migration',
securityDefiner: true,
volatility: 'volatile',
configuration: Object.freeze(['search_path=pg_catalog, ql3']),
}),
Object.freeze({
name: 'commit_plugin_package_task_reconciliation',
identityArguments:
@@ -19,6 +19,7 @@ export const POSTGRES_SCHEMA_READINESS_ERROR_CODES = [
'admin_role_invalid',
'automation_manager_role_invalid',
'approval_manager_role_invalid',
'run_manager_role_invalid',
'package_manager_role_invalid',
'package_executor_role_invalid',
'worker_credential_manager_role_invalid',
@@ -1361,6 +1362,37 @@ const REQUIRED_APPROVAL_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze(
),
);
const REQUIRED_RUN_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze(
Object.fromEntries(
postgresqlControlSchemaContract.tables.map(({ name }) => [
name,
Object.freeze(
name === 'schema_migrations' ||
name === 'schema_capabilities' ||
name === 'projects' ||
name === 'project_role_bindings' ||
name === 'task_definitions' ||
name === 'task_definition_revisions' ||
name === 'task_execution_revisions'
? { ...NO_TABLE_PRIVILEGES, select: true }
: name === 'runs' ||
name === 'run_attempts' ||
name === 'run_events' ||
name === 'security_audit_events'
? { ...NO_TABLE_PRIVILEGES, select: true, insert: true }
: name === 'plugin_package_identity_keyset_ledger'
? {
...NO_TABLE_PRIVILEGES,
select: true,
insert: true,
update: true,
}
: NO_TABLE_PRIVILEGES,
),
]),
),
);
const REQUIRED_WORKER_CREDENTIAL_MANAGER_PRIVILEGES: RequiredPrivileges =
Object.freeze(
Object.fromEntries(
@@ -1448,6 +1480,7 @@ const REQUIRED_RUNTIME_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
enforce_plugin_package_stage_provenance: false,
lock_active_plugin_package_project: false,
lock_approval_policy_fence: false,
lock_run_management_policy_fence: true,
plugin_package_automation_start_allowed: true,
plugin_package_workflow_admission_snapshot: true,
plugin_package_workflow_task_attempt_snapshot: true,
@@ -1464,6 +1497,7 @@ const REQUIRED_PACKAGE_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
enforce_plugin_package_stage_provenance: false,
lock_active_plugin_package_project: false,
lock_approval_policy_fence: true,
lock_run_management_policy_fence: false,
plugin_package_automation_start_allowed: false,
plugin_package_workflow_admission_snapshot: false,
plugin_package_workflow_task_attempt_snapshot: false,
@@ -1480,6 +1514,7 @@ const REQUIRED_PACKAGE_EXECUTOR_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges
enforce_plugin_package_stage_provenance: false,
lock_active_plugin_package_project: true,
lock_approval_policy_fence: true,
lock_run_management_policy_fence: false,
plugin_package_automation_start_allowed: false,
plugin_package_workflow_admission_snapshot: false,
plugin_package_workflow_task_attempt_snapshot: false,
@@ -1500,6 +1535,12 @@ const REQUIRED_APPROVAL_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges
lock_approval_policy_fence: true,
});
const REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
Object.freeze({
...NO_FUNCTION_PRIVILEGES,
lock_run_management_policy_fence: true,
});
function safeInteger(value: unknown): number | null {
if (typeof value === 'number' && Number.isSafeInteger(value)) return value;
if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) {
@@ -1864,6 +1905,7 @@ async function assertRole(
| 'admin_role_invalid'
| 'automation_manager_role_invalid'
| 'approval_manager_role_invalid'
| 'run_manager_role_invalid'
| 'package_manager_role_invalid'
| 'package_executor_role_invalid'
| 'worker_credential_manager_role_invalid'
@@ -2120,6 +2162,30 @@ export async function assertPostgresApprovalManagerSchemaReady(
});
}
export async function assertPostgresRunManagerSchemaReady(
queryable: PostgresMigrationQueryable,
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
): Promise<PostgresSchemaReadinessReport> {
const server = await readServer(queryable, contract);
const migrationIds = await assertHistory(queryable);
await assertCapability(queryable, contract);
await assertSchemaContract(queryable, contract);
await assertRole(
queryable,
contract,
REQUIRED_RUN_MANAGER_PRIVILEGES,
REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES,
'run_manager_role_invalid',
);
return Object.freeze({
ready: true,
...server,
contractName: contract.contractName,
contractVersion: contract.contractVersion,
migrationIds,
});
}
export async function assertPostgresPackageManagerSchemaReady(
queryable: PostgresMigrationQueryable,
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
@@ -108,7 +108,7 @@ test('serializes first observation, exact replay and append-only rotation', asyn
assert.equal(value.releases(), 3);
});
test('isolates Plugin, Worker, automation and Approval generations by authority key', async () => {
test('isolates Plugin, Worker, automation, Approval and Run generations by authority key', async () => {
const value = fixture('worker-credential-management');
await value.repository.observe(
snapshot(1, { audience: 'qinglong3-worker-credential-management' }),
@@ -131,6 +131,14 @@ test('isolates Plugin, Worker, automation and Approval generations by authority
approval.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
'approval-management',
);
const run = fixture('run-management');
await run.repository.observe(
snapshot(1, { audience: 'qinglong3-run-management' }),
);
assert.equal(
run.queries.find(({ text }) => text.startsWith('INSERT')).values[0],
'run-management',
);
assert.throws(
() => fixture('worker-credential-executor'),
TypeError,
@@ -104,6 +104,7 @@ test('enforces role-specific bounded pool sizes', () => {
'ai-credential-tester',
'automation-manager',
'approval-manager',
'run-manager',
'worker-credential-manager',
'worker-credential-executor',
]) {
@@ -104,6 +104,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
'pg-0053-plugin-package-workflow-run-list-index',
'pg-0054-approval-management-boundary',
'pg-0055-run-attempt-log-retention',
'pg-0056-run-management-boundary',
],
);
for (const migration of postgresqlMainMigrationStream.migrations) {
@@ -514,6 +515,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
checksum:
'c775c65ec03ae3a1606f899064d2d38fa63fd136ce52cbd1b1172c3a51e6bf30',
},
{
id: 'pg-0056-run-management-boundary',
checksum:
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
},
];
assert.deepEqual(
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
@@ -1904,3 +1910,30 @@ test('advances capability v54 with durable Cluster log retention authority', asy
/migration_id = 'pg-0054-approval-management-boundary'/,
);
});
test('advances capability v55 with isolated strong Run management authority', async () => {
const migration = migrationById('pg-0056-run-management-boundary');
const statements = [];
await migration.up({
async query(statement) {
statements.push(statement);
return { rows: [] };
},
});
const sql = statements.join('\n');
assert.match(sql, /ql3_run_manager/);
assert.match(sql, /lock_run_management_policy_fence/);
assert.match(
sql,
/GRANT SELECT, INSERT ON "ql3"\."runs", "ql3"\."run_attempts", "ql3"\."run_events", "ql3"\."security_audit_events" TO ql3_run_manager/,
);
assert.doesNotMatch(sql, /GRANT UPDATE ON "ql3"\."runs"/);
assert.match(sql, /'run-management'/);
assert.match(sql, /contract_version = 55/);
assert.match(sql, /"run_management_boundary":1/);
assert.match(sql, /contract_version = 54/);
assert.match(
sql,
/migration_id = 'pg-0055-run-attempt-log-retention'/,
);
});
@@ -4,6 +4,7 @@ const {
PostgresSchemaReadinessError,
assertPostgresAdminSchemaReady,
assertPostgresApprovalManagerSchemaReady,
assertPostgresRunManagerSchemaReady,
assertPostgresAutomationManagerSchemaReady,
assertPostgresPackageExecutorSchemaReady,
assertPostgresPackageManagerSchemaReady,
@@ -474,6 +475,37 @@ function approvalManagerPrivileges() {
}));
}
function runManagerPrivileges() {
const readable = new Set([
'schema_migrations',
'schema_capabilities',
'projects',
'project_role_bindings',
'task_definitions',
'task_definition_revisions',
'task_execution_revisions',
'runs',
'run_attempts',
'run_events',
'security_audit_events',
'plugin_package_identity_keyset_ledger',
]);
return postgresqlControlSchemaContract.tables.map(({ name: tableName }) => ({
tableName,
selectAllowed: readable.has(tableName),
insertAllowed: [
'runs',
'run_attempts',
'run_events',
'security_audit_events',
'plugin_package_identity_keyset_ledger',
].includes(tableName),
updateAllowed: tableName === 'plugin_package_identity_keyset_ledger',
deleteAllowed: false,
isOwner: false,
}));
}
function workerCredentialPrivileges(kind) {
const manager = kind === 'manager';
const readable = new Set([
@@ -634,6 +666,8 @@ function queryable(overrides = {}) {
executeAllowed:
overrides.functionMode === 'manager'
? functionName === 'lock_approval_policy_fence'
: overrides.functionMode === 'run-manager'
? functionName === 'lock_run_management_policy_fence'
: overrides.functionMode === 'executor'
? [
'commit_plugin_package_lifecycle',
@@ -651,6 +685,7 @@ function queryable(overrides = {}) {
'plugin_package_workflow_task_attempt_snapshot',
'plugin_package_run_start_allowed',
'plugin_package_tool_start_allowed',
'lock_run_management_policy_fence',
].includes(functionName),
isOwner: false,
})),
@@ -696,7 +731,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
serverMajor: 16,
currentUser: 'ql3_runtime',
contractName: 'control-core',
contractVersion: 54,
contractVersion: 55,
migrationIds: [
'pg-0001-schema-capability',
'pg-0002-run-core',
@@ -753,6 +788,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
'pg-0053-plugin-package-workflow-run-list-index',
'pg-0054-approval-management-boundary',
'pg-0055-run-attempt-log-retention',
'pg-0056-run-management-boundary',
],
});
});
@@ -783,10 +819,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
}),
);
assert.equal(report.currentUser, 'ql3_admin');
assert.equal(report.contractVersion, 54);
assert.equal(report.contractVersion, 55);
assert.equal(
report.migrationIds.at(-1),
'pg-0055-run-attempt-log-retention',
'pg-0056-run-management-boundary',
);
});
@@ -799,10 +835,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
}),
);
assert.equal(report.currentUser, 'ql3_automation_manager');
assert.equal(report.contractVersion, 54);
assert.equal(report.contractVersion, 55);
assert.equal(
report.migrationIds.at(-1),
'pg-0055-run-attempt-log-retention',
'pg-0056-run-management-boundary',
);
const widened = automationManagerPrivileges();
@@ -831,10 +867,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
}),
);
assert.equal(report.currentUser, 'ql3_approval_manager');
assert.equal(report.contractVersion, 54);
assert.equal(report.contractVersion, 55);
assert.equal(
report.migrationIds.at(-1),
'pg-0055-run-attempt-log-retention',
'pg-0056-run-management-boundary',
);
const widened = approvalManagerPrivileges();
@@ -856,6 +892,35 @@ test('accepts the isolated least-privilege human Approval manager role', async (
);
});
test('accepts the isolated least-privilege Run manager role', async () => {
const report = await assertPostgresRunManagerSchemaReady(
queryable({
currentUser: 'ql3_run_manager',
privileges: runManagerPrivileges(),
functionMode: 'run-manager',
}),
);
assert.equal(report.currentUser, 'ql3_run_manager');
assert.equal(report.contractVersion, 55);
assert.equal(report.migrationIds.at(-1), 'pg-0056-run-management-boundary');
const widened = runManagerPrivileges();
widened.find(({ tableName }) => tableName === 'runs').updateAllowed = true;
await assert.rejects(
assertPostgresRunManagerSchemaReady(
queryable({
currentUser: 'ql3_run_manager',
privileges: widened,
functionMode: 'run-manager',
}),
),
(error) =>
error instanceof PostgresSchemaReadinessError &&
error.code === 'run_manager_role_invalid' &&
error.facts.includes('table-privileges:runs'),
);
});
test('accepts isolated Package manager and executor roles', async () => {
const manager = await assertPostgresPackageManagerSchemaReady(
queryable({
@@ -941,10 +1006,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_worker_ingress');
assert.equal(report.contractVersion, 54);
assert.equal(report.contractVersion, 55);
assert.equal(
report.migrationIds.at(-1),
'pg-0055-run-attempt-log-retention',
'pg-0056-run-management-boundary',
);
});
@@ -138,21 +138,11 @@ function fixture(options = {}) {
if (normalized.includes('statement_timestamp()')) {
return { rows: [{ nowMs: 1_000_000 }], rowCount: 1 };
}
if (normalized.includes('FROM "ql3"."projects"')) {
const rows = options.projectRows ?? [
{ projectStatus: 'active', projectVersion: 2 },
];
return { rows, rowCount: rows.length };
}
if (normalized.includes('project_role_bindings')) {
const rows = options.bindingRows ?? [
{
bindingVersion: 3,
bindingState: 'active',
bindingRole: 'operator',
},
];
return { rows, rowCount: rows.length };
if (normalized.includes('lock_run_management_policy_fence')) {
return {
rows: [{ matches: options.authorizationMatches ?? true }],
rowCount: 1,
};
}
if (normalized.includes('idempotency_key = $2')) {
const rows = options.replayRows ?? [];
@@ -245,7 +235,11 @@ test('atomically appends a linked queued Run, remote Attempt, events and allowed
});
test('returns durable identities for an exact replay without appending again', async () => {
const { calls, repository } = fixture({ replayRows: [replayRow()] });
const { calls, repository } = fixture({
replayRows: [
replayRow({ runStatus: 'running', runVersion: 4, eventSequence: 4 }),
],
});
const result = await repository.retryRun(
command({
runId: '019f9200-0000-4000-8000-000000000102',
@@ -257,6 +251,7 @@ test('returns durable identities for an exact replay without appending again', a
assert.equal(result.status, 'existing');
assert.equal(result.runId, IDS.runId);
assert.equal(result.attemptId, IDS.attemptId);
assert.equal(result.runStatus, 'queued');
assert.equal(
calls.some(({ sql }) => sql.startsWith('INSERT INTO')),
false,
@@ -283,15 +278,11 @@ test('rejects stale authentication and changed authorization inside the transact
true,
);
assert.equal(
stale.calls.some(({ sql }) => sql.includes('FROM "ql3"."projects"')),
stale.calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')),
false,
);
const changed = fixture({
bindingRows: [
{ bindingVersion: 4, bindingState: 'active', bindingRole: 'operator' },
],
});
const changed = fixture({ authorizationMatches: false });
await assert.rejects(
changed.repository.retryRun(command()),
(error) =>
@@ -65,6 +65,10 @@ const EXPECTED_ROLES = Object.freeze({
connectionLimit: 32,
secret: 'ql3-postgres-runtime-auth',
}),
ql3_run_manager: Object.freeze({
connectionLimit: 4,
secret: 'ql3-postgres-run-manager-auth',
}),
ql3_worker_credential_executor: Object.freeze({
connectionLimit: 4,
secret: 'ql3-postgres-worker-credential-executor-auth',
@@ -258,7 +262,7 @@ function assertRolesAndDatabase(readFile, root, findings) {
findings.push(
finding(
'QL3_CNPG_DATABASE_ROLE',
'DatabaseRole resources must match the thirteen fixed least-privilege identities',
'DatabaseRole resources must match the fourteen fixed least-privilege identities',
),
);
break;
@@ -272,7 +276,7 @@ function assertRolesAndDatabase(readFile, root, findings) {
findings.push(
finding(
'QL3_CNPG_DATABASE_ROLE_SET',
'exactly the thirteen reviewed migration, runtime, AI maintenance, AI credential management/testing, admin, automation, Approval, package and Worker roles are required',
'exactly the fourteen reviewed migration, runtime, AI maintenance, AI credential management/testing, admin, automation, Approval, Run, package and Worker roles are required',
),
);
}
@@ -332,6 +336,7 @@ function assertSecretBoundary(readFile, root, findings) {
['ql3-postgres-admin-auth', 'ql3_admin'],
['ql3-postgres-automation-manager-auth', 'ql3_automation_manager'],
['ql3-postgres-approval-manager-auth', 'ql3_approval_manager'],
['ql3-postgres-run-manager-auth', 'ql3_run_manager'],
['ql3-postgres-worker-ingress-auth', 'ql3_worker_ingress'],
['ql3-postgres-package-manager-auth', 'ql3_package_manager'],
['ql3-postgres-package-executor-auth', 'ql3_package_executor'],
@@ -417,7 +422,7 @@ function assertSecretBoundary(readFile, root, findings) {
findings.push(
finding(
'QL3_CNPG_SECRET_EXAMPLE_SET',
'the example must describe all thirteen database credentials and the reviewed workload Secrets',
'the example must describe all fourteen database credentials and the reviewed workload Secrets',
),
);
}
+5
View File
@@ -2366,6 +2366,11 @@ function auditSourceImports(root, packagePath, findings) {
'src/approval-management/approvalManagementProcess.ts',
].includes(path.relative(packageDirectory, filePath)) &&
specifier === '@qinglong/cluster-postgres/approval-manager') ||
([
'src/run-management/runManagement.ts',
'src/run-management/runManagementProcess.ts',
].includes(path.relative(packageDirectory, filePath)) &&
specifier === '@qinglong/cluster-postgres/run-manager') ||
([
'src/prompt-output/retention/promptOutputGcCli.ts',
'src/prompt-output/retention/promptOutputGcProcess.ts',
+51 -12
View File
@@ -201,6 +201,9 @@ const {
const {
assertPostgresApprovalManagerSchemaReady,
} = require('../packages/ql3-cluster-postgres/dist/approval-management/index.js');
const {
assertPostgresRunManagerSchemaReady,
} = require('../packages/ql3-cluster-postgres/dist/entrypoints/runManager.js');
const {
assertPostgresPackageExecutorSchemaReady,
PostgresPluginPackageMaterializedRevisionRepository,
@@ -341,6 +344,8 @@ const AUTOMATION_MANAGER_USER = 'ql3_automation_manager';
const AUTOMATION_MANAGER_PASSWORD = 'ql3_automation_manager_test';
const APPROVAL_MANAGER_USER = 'ql3_approval_manager';
const APPROVAL_MANAGER_PASSWORD = 'ql3_approval_manager_test';
const RUN_MANAGER_USER = 'ql3_run_manager';
const RUN_MANAGER_PASSWORD = 'ql3_run_manager_test';
const PACKAGE_MANAGER_USER = 'ql3_package_manager';
const PACKAGE_MANAGER_PASSWORD = 'ql3_package_manager_test';
const PACKAGE_EXECUTOR_USER = 'ql3_package_executor';
@@ -3721,6 +3726,9 @@ async function provisionCredentialRoles(database) {
await database.pool.query(
`CREATE ROLE ${APPROVAL_MANAGER_USER} LOGIN PASSWORD '${APPROVAL_MANAGER_PASSWORD}'`,
);
await database.pool.query(
`CREATE ROLE ${RUN_MANAGER_USER} LOGIN PASSWORD '${RUN_MANAGER_PASSWORD}'`,
);
await database.pool.query(
`CREATE ROLE ${PACKAGE_MANAGER_USER} LOGIN PASSWORD '${PACKAGE_MANAGER_PASSWORD}'`,
);
@@ -5010,54 +5018,70 @@ async function runIdentityKeysetLedgerMatrix(options) {
authority === 'plugin-package-management' ||
authority === 'worker-credential-management' ||
authority === 'automation-management' ||
authority === 'approval-management',
authority === 'approval-management' ||
authority === 'run-management',
);
const workerAuthority = authority === 'worker-credential-management';
const automationAuthority = authority === 'automation-management';
const approvalAuthority = authority === 'approval-management';
const user = approvalAuthority
const runAuthority = authority === 'run-management';
const user = runAuthority
? RUN_MANAGER_USER
: approvalAuthority
? APPROVAL_MANAGER_USER
: automationAuthority
? AUTOMATION_MANAGER_USER
: workerAuthority
? WORKER_CREDENTIAL_MANAGER_USER
: PACKAGE_MANAGER_USER;
const password = approvalAuthority
const password = runAuthority
? RUN_MANAGER_PASSWORD
: approvalAuthority
? APPROVAL_MANAGER_PASSWORD
: automationAuthority
? AUTOMATION_MANAGER_PASSWORD
: workerAuthority
? WORKER_CREDENTIAL_MANAGER_PASSWORD
: PACKAGE_MANAGER_PASSWORD;
const role = approvalAuthority
const role = runAuthority
? 'run-manager'
: approvalAuthority
? 'approval-manager'
: automationAuthority
? 'automation-manager'
: workerAuthority
? 'worker-credential-manager'
: 'package-manager';
const applicationPrefix = approvalAuthority
const applicationPrefix = runAuthority
? 'ql3-ha-run-keyset-ledger'
: approvalAuthority
? 'ql3-ha-approval-keyset-ledger'
: automationAuthority
? 'ql3-ha-automation-keyset-ledger'
: workerAuthority
? 'ql3-ha-worker-keyset-ledger'
: 'ql3-ha-keyset-ledger';
const keyPrefix = approvalAuthority
const keyPrefix = runAuthority
? 'ha-run-identity-key'
: approvalAuthority
? 'ha-approval-identity-key'
: automationAuthority
? 'ha-automation-identity-key'
: workerAuthority
? 'ha-worker-identity-key'
: 'ha-identity-key';
const issuer = approvalAuthority
const issuer = runAuthority
? 'https://run-identity.ha.example.test/'
: approvalAuthority
? 'https://approval-identity.ha.example.test/'
: automationAuthority
? 'https://automation-identity.ha.example.test/'
: workerAuthority
? 'https://worker-identity.ha.example.test/'
: 'https://identity.ha.example.test/';
const audience = approvalAuthority
const audience = runAuthority
? 'qinglong3-run-management'
: approvalAuthority
? 'qinglong3-approval-management'
: automationAuthority
? 'qinglong3-automation-management'
@@ -9446,13 +9470,13 @@ async function runManualRunRetryHaEvidence(options) {
})
).definition;
const firstRuntime = await databaseOpener(
'runtime',
databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, primaryPort),
'run-manager',
databaseUrl(RUN_MANAGER_USER, RUN_MANAGER_PASSWORD, primaryPort),
'ql3-ha-manual-run-retry-a',
)();
const secondRuntime = await databaseOpener(
'runtime',
databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, primaryPort),
'run-manager',
databaseUrl(RUN_MANAGER_USER, RUN_MANAGER_PASSWORD, primaryPort),
'ql3-ha-manual-run-retry-b',
)();
try {
@@ -9707,6 +9731,7 @@ async function main(argv = process.argv.slice(2)) {
let workerCredentialIdentityKeysetLedger;
let automationIdentityKeysetLedger;
let approvalIdentityKeysetLedger;
let runManagementIdentityKeysetLedger;
let automationManagementInspection;
let pluginPackageLifecycle;
let pluginPackageQuarantine;
@@ -9933,6 +9958,11 @@ async function main(argv = process.argv.slice(2)) {
),
'ql3-ha-approval-manager-readiness',
)();
const runManagerReadinessDatabase = await databaseOpener(
'run-manager',
databaseUrl(RUN_MANAGER_USER, RUN_MANAGER_PASSWORD, primaryPort),
'ql3-ha-run-manager-readiness',
)();
const packageManagerReadinessDatabase = await databaseOpener(
'package-manager',
databaseUrl(PACKAGE_MANAGER_USER, PACKAGE_MANAGER_PASSWORD, primaryPort),
@@ -9974,6 +10004,9 @@ async function main(argv = process.argv.slice(2)) {
await assertPostgresApprovalManagerSchemaReady(
approvalManagerReadinessDatabase.pool,
);
await assertPostgresRunManagerSchemaReady(
runManagerReadinessDatabase.pool,
);
await assertPostgresPackageManagerSchemaReady(
packageManagerReadinessDatabase.pool,
);
@@ -9992,6 +10025,7 @@ async function main(argv = process.argv.slice(2)) {
adminReadinessDatabase.close(),
automationManagerReadinessDatabase.close(),
approvalManagerReadinessDatabase.close(),
runManagerReadinessDatabase.close(),
packageManagerReadinessDatabase.close(),
packageExecutorReadinessDatabase.close(),
workerManagerReadinessDatabase.close(),
@@ -10031,6 +10065,10 @@ async function main(argv = process.argv.slice(2)) {
primaryPort,
authority: 'approval-management',
});
runManagementIdentityKeysetLedger = await runIdentityKeysetLedgerMatrix({
primaryPort,
authority: 'run-management',
});
timeline.push({
state: 'durable_identity_keyset_ledger_verified',
atMs: Number((performance.now() - startedAt).toFixed(3)),
@@ -11968,6 +12006,7 @@ async function main(argv = process.argv.slice(2)) {
workerCredentialIdentityKeysetLedger,
automationIdentityKeysetLedger,
approvalIdentityKeysetLedger,
runManagementIdentityKeysetLedger,
automationManagementInspection,
pluginPackageLifecycle: pluginPackageLifecycle.report,
pluginPackageQuarantine: pluginPackageQuarantine.report,
@@ -32,6 +32,7 @@ test('accepts the locked CloudNativePG HA and authority profile', () => {
'ql3_migration',
'ql3_package_executor',
'ql3_package_manager',
'ql3_run_manager',
'ql3_runtime',
'ql3_worker_credential_executor',
'ql3_worker_credential_manager',
+4 -4
View File
@@ -340,10 +340,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', (
rootSourceFileRoles: clusterAdmin.rootSourceFileRoles,
},
{
sourceFiles: 87,
sourceFiles: 94,
rootSourceFiles: 1,
rootSourceLines: 61,
nestedSourceFiles: 86,
nestedSourceFiles: 93,
rootSourceFileRoles: {
'modelInvocationMigrationCli.ts': 'binary_entry',
},
@@ -421,10 +421,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', (
rootSourceFileRoles: clusterPostgres.rootSourceFileRoles,
},
{
sourceFiles: 150,
sourceFiles: 152,
rootSourceFiles: 1,
rootSourceLines: 125,
nestedSourceFiles: 149,
nestedSourceFiles: 151,
rootSourceFileRoles: { 'index.ts': 'public_export' },
},
);
@@ -0,0 +1,85 @@
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const { test } = require('node:test');
const yaml = require('js-yaml');
const ROOT = path.resolve(__dirname, '../..');
function document(relativePath) {
return yaml.load(fs.readFileSync(path.join(ROOT, relativePath), 'utf8'));
}
test('keeps strong Run management opt-in, private and least-resource', () => {
const directory = 'deploy/kubernetes/ql3-cluster/operations/run-management/base';
const deployment = document(`${directory}/deployment.yaml`);
const service = document(`${directory}/service.yaml`);
const policy = document(`${directory}/network-policy.yaml`);
const kustomization = document(`${directory}/kustomization.yaml`);
const container = deployment.spec.template.spec.containers[0];
const environment = new Map(container.env.map((entry) => [entry.name, entry]));
assert.equal(deployment.metadata.name, 'ql3-run-management');
assert.equal(deployment.spec.replicas, 2);
assert.equal(deployment.spec.template.spec.automountServiceAccountToken, false);
assert.equal(container.securityContext.readOnlyRootFilesystem, true);
assert.equal(container.securityContext.allowPrivilegeEscalation, false);
assert.deepEqual(container.securityContext.capabilities.drop, ['ALL']);
assert.deepEqual(container.command, [
'node',
'/opt/qinglong/node_modules/@qinglong/cluster-admin/dist/run-management/runManagementCli.js',
]);
assert.equal(environment.get('QL3_RUN_MANAGEMENT_ENABLED').value, 'true');
assert.equal(environment.get('QL3_RUN_MANAGEMENT_PORT').value, '8448');
assert.equal(environment.get('QL3_POSTGRES_RUN_MANAGER_POOL_MAX').value, '2');
assert.equal(container.resources.requests.memory, '96Mi');
assert.equal(service.spec.type, 'ClusterIP');
assert.equal(service.spec.ports[0].port, 8448);
assert.equal(policy.spec.ingress[0].from[0].podSelector.matchLabels['qinglong.io/run-management-client'], 'true');
assert.equal(policy.spec.egress.length, 1);
assert.deepEqual(kustomization.resources, [
'service-account.yaml',
'service.yaml',
'deployment.yaml',
'pod-disruption-budget.yaml',
'network-policy.yaml',
]);
for (const defaultPath of [
'deploy/kubernetes/ql3-cluster/base/kustomization.yaml',
'deploy/kubernetes/ql3-cluster/overlays/cloudnative-pg/kustomization.yaml',
]) {
assert.doesNotMatch(fs.readFileSync(path.join(ROOT, defaultPath), 'utf8'), /run-management/);
}
});
test('binds the CloudNativePG overlay only to the dedicated Run manager role', () => {
const directory = 'deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg';
const patch = yaml.load(fs.readFileSync(path.join(ROOT, directory, 'deployment-patch.yaml'), 'utf8'));
const environment = new Map(patch[0].value.map((entry) => [entry.name, entry]));
const policy = document(`${directory}/network-policy-patch.yaml`);
const manifest = JSON.parse(
fs.readFileSync(path.join(ROOT, 'packages/ql3-cluster-admin/package.json'), 'utf8'),
);
assert.equal(environment.has('QL3_POSTGRES_RUN_MANAGER_URL'), false);
assert.equal(
environment.get('QL3_POSTGRES_RUN_MANAGER_USER').valueFrom.secretKeyRef.name,
'ql3-postgres-run-manager-auth',
);
assert.equal(
environment.get('QL3_POSTGRES_RUN_MANAGER_PASSWORD').valueFrom.secretKeyRef.name,
'ql3-postgres-run-manager-auth',
);
assert.equal(environment.get('QL3_POSTGRES_RUN_MANAGER_HOST').value, 'ql3-postgres-rw.qinglong3-system.svc');
assert.equal(policy.spec.egress.length, 2);
assert.deepEqual(policy.spec.egress[1].to[0].podSelector.matchLabels, {
'cnpg.io/cluster': 'ql3-postgres',
});
assert.equal(manifest.bin['ql3-run-manage'], 'dist/run-management/runManagementCli.js');
assert.equal(manifest.bin['ql3-run-client'], 'dist/run-management/runManagementClientCli.js');
assert.equal(
manifest.exports['./run-management-process'].require,
'./dist/run-management/runManagementProcess.js',
);
});