feat(ql3): establish 3.0 incubation baseline

This commit is contained in:
whyour
2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,275 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ql3-cluster-control
labels:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane
app.kubernetes.io/part-of: qinglong3
spec:
replicas: 2
minReadySeconds: 10
revisionHistoryLimit: 3
progressDeadlineSeconds: 600
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane
template:
metadata:
labels:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane
app.kubernetes.io/part-of: qinglong3
spec:
serviceAccountName: ql3-cluster-control
automountServiceAccountToken: false
terminationGracePeriodSeconds: 30
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- topologyKey: kubernetes.io/hostname
labelSelector:
matchLabels:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane
containers:
- name: cluster-control
image: qinglong3-cluster-control:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: QL_DEPLOYMENT_PROFILE
value: cluster-control
- name: QL3_CLUSTER_CONTROL_ENABLED
value: "true"
- name: QL3_CLUSTER_HTTP_HOST
value: 0.0.0.0
- name: QL3_CLUSTER_HTTP_PORT
value: "5800"
- name: QL3_CLUSTER_HTTP_DRAIN_TIMEOUT_MS
value: "10000"
- name: QL3_WORKER_INGRESS_ENABLED
value: "true"
- name: QL3_WORKER_INGRESS_HOST
value: 0.0.0.0
- name: QL3_WORKER_INGRESS_PORT
value: "5801"
- name: QL3_WORKER_INGRESS_POSTGRES_TLS_MODE
value: verify-full
- name: QL3_WORKER_INGRESS_POSTGRES_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-worker-ingress/ca.crt
- name: QL3_WORKER_INGRESS_POSTGRES_MAX_CONNECTIONS
value: "4"
- name: QL3_WORKER_INGRESS_POSTGRES_APPLICATION_NAME
value: qinglong3-worker-ingress
- name: QL3_WORKER_INGRESS_TLS_PRIVATE_KEY_FILE
value: /var/run/secrets/qinglong3/worker-ingress-tls/tls.key
- name: QL3_WORKER_INGRESS_TLS_CERTIFICATE_FILE
value: /var/run/secrets/qinglong3/worker-ingress-tls/tls.crt
- name: QL3_WORKER_INGRESS_TLS_CLIENT_CA_FILE
value: /var/run/secrets/qinglong3/worker-ingress-tls/client-ca.crt
- name: QL3_WORKER_SECRET_PROVIDER
value: mounted-files
- name: QL3_WORKER_SECRET_ROOT_DIRECTORY
value: /var/run/secrets/qinglong3/worker-values
- name: QL3_POSTGRES_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-runtime/ca.crt
- name: QL3_POSTGRES_MAX_CONNECTIONS
value: "8"
- name: QL3_POSTGRES_APPLICATION_NAME
value: qinglong3-cluster-control
- name: QL3_CLUSTER_REPLICA_ID
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: QL3_POSTGRES_RUNTIME_URL
valueFrom:
secretKeyRef:
name: ql3-cluster-control-runtime
key: postgres-runtime-url
- name: QL3_POSTGRES_TLS_SERVERNAME
valueFrom:
secretKeyRef:
name: ql3-cluster-control-runtime
key: postgres-tls-servername
- name: QL3_API_CREDENTIAL_PEPPER
valueFrom:
secretKeyRef:
name: ql3-cluster-control-runtime
key: api-credential-pepper
- name: QL3_POSTGRES_WORKER_INGRESS_URL
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: postgres-worker-ingress-url
- name: QL3_WORKER_INGRESS_POSTGRES_TLS_SERVERNAME
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: postgres-tls-servername
- name: QL3_WORKER_CREDENTIAL_PEPPER
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: worker-credential-pepper
- name: QL3_WORKER_ARTIFACT_S3_BUCKET
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: artifact-s3-bucket
- name: QL3_WORKER_ARTIFACT_S3_REGION
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: artifact-s3-region
- name: QL3_WORKER_ARTIFACT_S3_ENDPOINT
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: artifact-s3-endpoint
optional: true
- name: QL3_WORKER_ARTIFACT_S3_ALLOW_INSECURE
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: artifact-s3-allow-insecure
optional: true
- name: QL3_WORKER_ARTIFACT_S3_FORCE_PATH_STYLE
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: artifact-s3-force-path-style
optional: true
- name: QL3_WORKER_ARTIFACT_S3_ENCRYPTION
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: artifact-s3-encryption
optional: true
- name: QL3_WORKER_ARTIFACT_S3_KMS_KEY_ID
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: artifact-s3-kms-key-id
optional: true
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: artifact-s3-access-key-id
optional: true
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: ql3-cluster-worker-ingress
key: artifact-s3-secret-access-key
optional: true
ports:
- name: http
containerPort: 5800
protocol: TCP
- name: worker-mtls
containerPort: 5801
protocol: TCP
startupProbe:
httpGet:
path: /livez
port: http
scheme: HTTP
periodSeconds: 2
timeoutSeconds: 1
failureThreshold: 30
readinessProbe:
httpGet:
path: /readyz
port: http
scheme: HTTP
periodSeconds: 5
timeoutSeconds: 2
failureThreshold: 2
livenessProbe:
httpGet:
path: /livez
port: http
scheme: HTTP
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: "1"
memory: 512Mi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: postgres-runtime-ca
mountPath: /var/run/secrets/qinglong3/postgres-runtime
readOnly: true
- name: postgres-worker-ingress-ca
mountPath: /var/run/secrets/qinglong3/postgres-worker-ingress
readOnly: true
- name: worker-ingress-tls
mountPath: /var/run/secrets/qinglong3/worker-ingress-tls
readOnly: true
- name: worker-secret-values
mountPath: /var/run/secrets/qinglong3/worker-values
readOnly: true
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 16Mi
- name: postgres-runtime-ca
secret:
secretName: ql3-cluster-control-runtime
defaultMode: 292
items:
- key: postgres-ca.crt
path: ca.crt
- name: postgres-worker-ingress-ca
secret:
secretName: ql3-cluster-worker-ingress
defaultMode: 288
items:
- key: postgres-ca.crt
path: ca.crt
- name: worker-ingress-tls
secret:
secretName: ql3-cluster-worker-ingress
defaultMode: 288
items:
- key: tls.key
path: tls.key
- key: tls.crt
path: tls.crt
- key: client-ca.crt
path: client-ca.crt
- name: worker-secret-values
secret:
secretName: ql3-cluster-worker-values
optional: true
defaultMode: 288
@@ -0,0 +1,11 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: qinglong3-system
resources:
- namespace.yaml
- service-account.yaml
- service.yaml
- deployment.yaml
- pod-disruption-budget.yaml
@@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
name: qinglong3-system
labels:
app.kubernetes.io/part-of: qinglong3
@@ -0,0 +1,13 @@
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: ql3-cluster-control
labels:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane
spec:
minAvailable: 1
selector:
matchLabels:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane
@@ -0,0 +1,8 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: ql3-cluster-control
labels:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane
automountServiceAccountToken: false
@@ -0,0 +1,21 @@
apiVersion: v1
kind: Service
metadata:
name: ql3-cluster-control
labels:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane
ports:
- name: http
port: 5800
targetPort: http
protocol: TCP
- name: worker-mtls
port: 5801
targetPort: worker-mtls
protocol: TCP
@@ -0,0 +1,10 @@
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: ql3-postgres
spec:
plugins:
- name: barman-cloud.cloudnative-pg.io
isWALArchiver: true
parameters:
barmanObjectName: ql3-postgres-backup
@@ -0,0 +1,8 @@
apiVersion: kustomize.config.k8s.io/v1alpha1
kind: Component
resources:
- scheduled-backup.yaml
patches:
- path: cluster-plugin-patch.yaml
@@ -0,0 +1,30 @@
# Example only. Copy into a private overlay and replace every placeholder.
# This resource is intentionally excluded from the Component.
apiVersion: barmancloud.cnpg.io/v1
kind: ObjectStore
metadata:
name: ql3-postgres-backup
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-postgres
app.kubernetes.io/component: database-backup
app.kubernetes.io/part-of: qinglong3
spec:
retentionPolicy: 30d
configuration:
destinationPath: s3://REPLACE_WITH_VERSIONED_LOCKED_BUCKET/qinglong3/ql3-postgres
endpointURL: https://REPLACE_WITH_OBJECT_STORE_ENDPOINT
s3Credentials:
accessKeyId:
name: ql3-postgres-backup-object-store
key: ACCESS_KEY_ID
secretAccessKey:
name: ql3-postgres-backup-object-store
key: ACCESS_SECRET_KEY
wal:
compression: lz4
encryption: AES256
maxParallel: 2
data:
compression: lz4
encryption: AES256
@@ -0,0 +1,11 @@
# Example only. Save as kustomization.yaml in a private overlay beside a
# populated object-store.yaml. Do not apply this file directly.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../operators/cloudnative-pg
- object-store.yaml
components:
- ../../components/barman-cloud-backup
@@ -0,0 +1,20 @@
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: ql3-postgres-daily
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-postgres
app.kubernetes.io/component: database-backup
app.kubernetes.io/part-of: qinglong3
spec:
schedule: '0 0 0 * * *'
backupOwnerReference: self
immediate: false
suspend: false
target: prefer-standby
cluster:
name: ql3-postgres
method: plugin
pluginConfiguration:
name: barman-cloud.cloudnative-pg.io
@@ -0,0 +1,30 @@
# Optional Cluster AI durable Prompt output
This component opts the existing Cluster AI runtime into encrypted, durable
Prompt output. Apply it only together with `../cluster-ai`; the default Cluster
and default Cluster AI profiles remain live-output-only.
The runtime receives only a read-only `0440` projection of
`ql3-prompt-output-keyring/keyring.json`. It does not receive a ServiceAccount
token, Kubernetes API authority, or permission to provision, rotate, or retire
keys. The projected-keyring adapter reopens the manifest for each operation, so
Kubernetes atomic projection updates become visible without a process restart.
Before applying it:
1. Provision the namespaced `ql3-prompt-output-keyring` Secret through the
deployment platform or Secret manager. Do not commit key material or a
deployable Secret manifest to this repository.
2. Store the canonical
`qinglong/plugin-package-prompt-output-keyring@v1` document under the exact
`keyring.json` data key. Keep the active key and bounded decrypt-only history
in that one document.
3. Apply both components from a private overlay and pin the independent Cluster
AI image digest, as shown by
`../../overlays/cluster-ai-prompt-output-example/kustomization.yaml`.
4. Use the reviewed management operation for retirement. Provisioning and
active-key rotation remain deployment-plane responsibilities; the runtime
must never be granted Secret mutation authority.
The Secret must remain non-optional. A missing, malformed, writable, escaped,
or rotating-during-read keyring fails startup or the affected operation closed.
@@ -0,0 +1,26 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ql3-cluster-control
spec:
template:
spec:
containers:
- name: cluster-control
env:
- name: QL3_CLUSTER_AI_PROMPT_OUTPUT_ENABLED
value: 'true'
- name: QL3_CLUSTER_AI_PROMPT_OUTPUT_KEYRING_ROOT
value: /var/run/secrets/qinglong3/ai/prompt-output-keyring
volumeMounts:
- name: cluster-ai-prompt-output-keyring
mountPath: /var/run/secrets/qinglong3/ai/prompt-output-keyring
readOnly: true
volumes:
- name: cluster-ai-prompt-output-keyring
secret:
secretName: ql3-prompt-output-keyring
defaultMode: 288
items:
- key: keyring.json
path: keyring.json
@@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1alpha1
kind: Component
patches:
- path: deployment-patch.yaml
@@ -0,0 +1,26 @@
# Optional Cluster AI component
This component replaces only the Cluster Control image with the explicit
`runtime-ai` target. The default Cluster deployment and image remain AI-free.
Before applying it:
1. Replace the example Project, provider URL, model and policy revision in
`provider-authority-configmap.yaml`; keep `authority.json` canonical,
one-line JSON with one trailing newline.
2. Bind the same Project/provider to a canonical SecretRef through the
append-only model-provider credential catalog.
3. Project each provider authorization value under the lowercase SHA-256 of
its canonical SecretRef. Use `provider-secrets.example.yaml` only as a
shape reference and provision the real Secret through a Secret manager.
4. Build and publish the Docker `runtime-ai` target, then pin its independent
digest in a private overlay based on `private-overlay.example.yaml`.
The component mounts ConfigMap and Secret volumes read-only with mode `0440`,
does not mount a ServiceAccount token, and grants no Kubernetes API access.
It remains live-output-only by default. Compose the separate
`../cluster-ai-prompt-output` component only when encrypted durable Prompt
output and its externally provisioned keyring are required.
Each replica adds a separate, bounded PostgreSQL runtime pool of four
connections by default; tune `QL3_CLUSTER_AI_DATABASE_MAX_CONNECTIONS` and
`QL3_CLUSTER_AI_MAX_CONCURRENT` together for the cluster's resource budget.
@@ -0,0 +1,42 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ql3-cluster-control
spec:
template:
spec:
containers:
- name: cluster-control
image: qinglong3-cluster-control-ai:3.0.0-alpha.0
env:
- name: QL3_CLUSTER_AI_ENABLED
value: "true"
- name: QL3_CLUSTER_AI_PROVIDER_AUTHORITY_FILE
value: /var/run/qinglong3/ai/provider-authority/authority.json
- name: QL3_CLUSTER_AI_SECRET_ROOT
value: /var/run/secrets/qinglong3/ai/provider-secrets
- name: QL3_CLUSTER_AI_MAX_CONCURRENT
value: "4"
- name: QL3_CLUSTER_AI_RECOVERY_LIMIT
value: "32"
- name: QL3_CLUSTER_AI_DATABASE_MAX_CONNECTIONS
value: "4"
volumeMounts:
- name: cluster-ai-provider-authority
mountPath: /var/run/qinglong3/ai/provider-authority
readOnly: true
- name: cluster-ai-provider-secrets
mountPath: /var/run/secrets/qinglong3/ai/provider-secrets
readOnly: true
volumes:
- name: cluster-ai-provider-authority
configMap:
name: ql3-cluster-ai-provider-authority
defaultMode: 288
items:
- key: authority.json
path: authority.json
- name: cluster-ai-provider-secrets
secret:
secretName: ql3-cluster-ai-provider-secrets
defaultMode: 288
@@ -0,0 +1,8 @@
apiVersion: kustomize.config.k8s.io/v1alpha1
kind: Component
resources:
- provider-authority-configmap.yaml
patches:
- path: deployment-patch.yaml
@@ -0,0 +1,15 @@
# Example only. Save as kustomization.yaml in a private overlay and replace the
# digest with the independently verified runtime-ai image digest.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../base
components:
- ../../components/cluster-ai
images:
- name: qinglong3-cluster-control-ai
newName: registry.example.com/qinglong/qinglong3-cluster-control-ai
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,11 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: ql3-cluster-ai-provider-authority
labels:
app.kubernetes.io/name: ql3-cluster-control
app.kubernetes.io/component: control-plane-ai
app.kubernetes.io/part-of: qinglong3
data:
authority.json: |
{"schema":"qinglong/projected-model-gateway-authority@v1","providers":[{"type":"openai-compatible","baseUrl":"https://models.example.invalid/v1/","allowPlaintextLoopback":false,"maxResponseBytes":1048576}],"projects":[{"projectId":"replace-project-id","policy":{"revision":"replace-policy-v1","allowedProviders":["openai-compatible"],"allowedModels":["replace-model-id"],"maxInputBytes":65536,"maxOutputBytes":1048576,"maxOutputTokens":4096,"maxTotalTokens":32768,"maxCostMicros":null,"priceRevision":null}}]}
@@ -0,0 +1,11 @@
# Example only. Populate through a Secret manager and never commit real values.
# The data key is sha256(canonical SecretRef); the matching catalog binding is:
# qlsecret:v1:eyJwcm9qZWN0SWQiOiJyZXBsYWNlLXByb2plY3QtaWQiLCJuYW1lIjoib3BlbmFpLWNvbXBhdGlibGUtdG9rZW4ifQ
apiVersion: v1
kind: Secret
metadata:
name: ql3-cluster-ai-provider-secrets
namespace: qinglong3-system
type: Opaque
stringData:
2ac34db919c697fd198d1a4fdfd90f69d9fa58c363c7ce5c8e4ca113413fbc6b: REPLACE_WITH_PROVIDER_AUTHORIZATION_VALUE
@@ -0,0 +1,196 @@
apiVersion: batch/v1
kind: Job
metadata:
name: ql3-approval-management-client
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-approval-management-client
app.kubernetes.io/component: approval-management-client
app.kubernetes.io/part-of: qinglong3
qinglong.io/execution-model: caller-driven
spec:
backoffLimit: 0
activeDeadlineSeconds: 120
ttlSecondsAfterFinished: 600
template:
metadata:
labels:
app.kubernetes.io/name: ql3-approval-management-client
app.kubernetes.io/component: approval-management-client
app.kubernetes.io/part-of: qinglong3
qinglong.io/execution-model: caller-driven
qinglong.io/approval-management-client: 'true'
spec:
serviceAccountName: ql3-approval-management-client
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
initContainers:
- name: wait-for-manager
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- -e
args:
- |
const fs = require('node:fs');
const https = require('node:https');
const host = 'ql3-approval-management.qinglong3-system.svc';
const ca = fs.readFileSync('/var/run/qinglong3/approval-management-trust/ca.crt');
const cert = fs.readFileSync('/var/run/secrets/qinglong3/approval-management-client-identity/tls.crt');
const key = fs.readFileSync('/var/run/secrets/qinglong3/approval-management-client-identity/tls.key');
const probe = () => new Promise((resolve, reject) => {
const request = https.request({
host,
port: 8447,
path: '/readyz',
method: 'GET',
servername: host,
ca,
cert,
key,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
rejectUnauthorized: true,
agent: false,
headers: { connection: 'close' },
}, (response) => {
response.resume();
response.once('end', () =>
response.statusCode === 200
? resolve()
: reject(new Error('manager not ready')));
});
request.setTimeout(3000, () => request.destroy(new Error('timeout')));
request.once('error', reject);
request.end();
});
(async () => {
for (let attempt = 1; attempt <= 30; attempt += 1) {
try {
await probe();
return;
} catch {
if (attempt === 30) throw new Error('manager readiness unavailable');
await new Promise((resolve) => setTimeout(resolve, 1000));
}
}
})().catch((error) => {
process.stderr.write(`${error.message}\n`);
process.exitCode = 1;
});
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 5m
memory: 16Mi
limits:
cpu: 100m
memory: 64Mi
volumeMounts:
- name: trust
mountPath: /var/run/qinglong3/approval-management-trust
readOnly: true
- name: client-identity
mountPath: /var/run/secrets/qinglong3/approval-management-client-identity
readOnly: true
containers:
- name: client
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- /bin/sh
- -c
args:
- |
set -eu
umask 077
cp /var/run/secrets/qinglong3/approval-management-request/client.json /tmp/client.json
cp /var/run/secrets/qinglong3/approval-management-request/command.json /tmp/command.json
cp /var/run/secrets/qinglong3/approval-management-assertion/assertion.jwt /tmp/assertion.jwt
cp /var/run/qinglong3/approval-management-trust/ca.crt /tmp/ca.crt
cp /var/run/secrets/qinglong3/approval-management-client-identity/tls.crt /tmp/client.crt
cp /var/run/secrets/qinglong3/approval-management-client-identity/tls.key /tmp/client.key
chmod 600 /tmp/client.json /tmp/command.json /tmp/assertion.jwt /tmp/ca.crt /tmp/client.crt /tmp/client.key
exec node /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/approval-management/approvalManagementClientCli.js \
--config=/tmp/client.json \
--command=/tmp/command.json \
--assertion=/tmp/assertion.jwt
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 25m
memory: 48Mi
limits:
cpu: 250m
memory: 128Mi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: request
mountPath: /var/run/secrets/qinglong3/approval-management-request
readOnly: true
- name: assertion
mountPath: /var/run/secrets/qinglong3/approval-management-assertion
readOnly: true
- name: trust
mountPath: /var/run/qinglong3/approval-management-trust
readOnly: true
- name: client-identity
mountPath: /var/run/secrets/qinglong3/approval-management-client-identity
readOnly: true
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 4Mi
- name: request
secret:
secretName: ql3-approval-management-request
defaultMode: 288
items:
- key: client.json
path: client.json
- key: command.json
path: command.json
- name: assertion
secret:
secretName: ql3-approval-management-assertion
defaultMode: 288
items:
- key: assertion.jwt
path: assertion.jwt
- name: trust
configMap:
name: ql3-approval-management-client-trust
defaultMode: 292
items:
- key: ca.crt
path: ca.crt
- name: client-identity
secret:
secretName: ql3-approval-management-client-identity
defaultMode: 288
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
@@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- service-account.yaml
- job.yaml
- network-policy.yaml
@@ -0,0 +1,39 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-approval-management-client
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-approval-management-client
app.kubernetes.io/component: approval-management-client
app.kubernetes.io/part-of: qinglong3
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: ql3-approval-management-client
app.kubernetes.io/component: approval-management-client
policyTypes:
- Ingress
- Egress
ingress: []
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
ports:
- protocol: TCP
port: 8447
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: ql3-approval-management-client
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-approval-management-client
app.kubernetes.io/component: approval-management-client
app.kubernetes.io/part-of: qinglong3
automountServiceAccountToken: false
@@ -0,0 +1,45 @@
# Example only. Copy these objects into a private, per-command overlay.
# Replace every placeholder. Never commit an assertion, private key or review detail.
apiVersion: v1
kind: Secret
metadata:
name: ql3-approval-management-request
namespace: qinglong3-system
immutable: true
type: Opaque
stringData:
client.json: |
{"schemaVersion":1,"endpoint":"https://ql3-approval-management.qinglong3-system.svc:8447/api/v3/approvals/management","servername":"ql3-approval-management.qinglong3-system.svc","caFile":"/tmp/ca.crt","clientCertificateFile":"/tmp/client.crt","clientPrivateKeyFile":"/tmp/client.key","requestTimeoutMs":15000}
command.json: |
{"schemaVersion":1,"operation":"approval.inspect","request":{"projectId":"REPLACE_WITH_PROJECT_ID","approvalRequestId":"REPLACE_WITH_APPROVAL_REQUEST_ID","requestId":"REPLACE_WITH_UNIQUE_REQUEST_ID","auditEventId":"REPLACE_WITH_UUID_V4","failureAuditEventId":"REPLACE_WITH_DIFFERENT_UUID_V4"}}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: ql3-approval-management-client-trust
namespace: qinglong3-system
immutable: true
data:
ca.crt: |
REPLACE_WITH_REVIEWED_MANAGER_CA_CERTIFICATE
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-approval-management-assertion
namespace: qinglong3-system
immutable: true
type: Opaque
stringData:
assertion.jwt: REPLACE_WITH_SHORT_LIVED_STRONG_USER_ASSERTION
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-approval-management-client-identity
namespace: qinglong3-system
immutable: true
type: kubernetes.io/tls
stringData:
tls.crt: REPLACE_WITH_SHORT_LIVED_CLIENT_CERTIFICATE_CHAIN
tls.key: REPLACE_WITH_CLIENT_PRIVATE_KEY
@@ -0,0 +1,11 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- base
images:
- name: qinglong3-cluster-admin
newName: registry.example.com/qinglong/qinglong3-cluster-admin
# Fail closed until the independently verified release digest is supplied.
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,198 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ql3-approval-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
app.kubernetes.io/part-of: qinglong3
spec:
replicas: 2
minReadySeconds: 10
revisionHistoryLimit: 3
progressDeadlineSeconds: 600
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
template:
metadata:
annotations:
qinglong.io/approval-management-client-ca-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
qinglong.io/approval-management-client-crl-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
labels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
app.kubernetes.io/part-of: qinglong3
spec:
serviceAccountName: ql3-approval-management
automountServiceAccountToken: false
terminationGracePeriodSeconds: 30
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- topologyKey: kubernetes.io/hostname
labelSelector:
matchLabels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
containers:
- name: management
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/approval-management/approvalManagementCli.js
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: QL3_PROFILE
value: cluster-admin
- name: QL3_APPROVAL_MANAGEMENT_ENABLED
value: 'true'
- name: QL3_APPROVAL_MANAGEMENT_HOST
value: 0.0.0.0
- name: QL3_APPROVAL_MANAGEMENT_PORT
value: '8447'
- name: QL3_APPROVAL_MANAGEMENT_TLS_CERT_FILE
value: /var/run/secrets/qinglong3/approval-management-tls/tls.crt
- name: QL3_APPROVAL_MANAGEMENT_TLS_KEY_FILE
value: /var/run/secrets/qinglong3/approval-management-tls/tls.key
- name: QL3_APPROVAL_MANAGEMENT_CLIENT_CA_FILE
value: /var/run/secrets/qinglong3/approval-management-tls/ca.crt
- name: QL3_APPROVAL_MANAGEMENT_CLIENT_CRL_FILE
value: /var/run/secrets/qinglong3/approval-management-tls/client.crl
- name: QL3_APPROVAL_MANAGEMENT_IDENTITY_KEYSET_FILE
value: /var/run/qinglong3/approval-management-identity/keyset.json
- name: QL3_APPROVAL_MANAGEMENT_MAX_BODY_BYTES
value: '65536'
- name: QL3_APPROVAL_MANAGEMENT_MAX_CONNECTIONS
value: '32'
- name: QL3_APPROVAL_MANAGEMENT_MAX_CONCURRENT_REQUESTS
value: '16'
- name: QL3_APPROVAL_MANAGEMENT_REQUEST_TIMEOUT_MS
value: '10000'
- name: QL3_APPROVAL_MANAGEMENT_DRAIN_TIMEOUT_MS
value: '5000'
- name: QL3_APPROVAL_MANAGEMENT_RATE_WINDOW_MS
value: '60000'
- name: QL3_APPROVAL_MANAGEMENT_PEER_REQUEST_LIMIT
value: '30'
- name: QL3_APPROVAL_MANAGEMENT_GLOBAL_REQUEST_LIMIT
value: '300'
- name: QL3_APPROVAL_MANAGEMENT_MAX_RATE_LIMIT_PEERS
value: '1024'
- name: QL3_POSTGRES_APPROVAL_MANAGER_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_APPROVAL_MANAGER_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-approval-manager/ca.crt
- name: QL3_POSTGRES_APPROVAL_MANAGER_APPLICATION_NAME
value: qinglong3-approval-manager
- name: QL3_POSTGRES_APPROVAL_MANAGER_POOL_MAX
value: '2'
- name: QL3_POSTGRES_APPROVAL_MANAGER_URL
valueFrom:
secretKeyRef:
name: ql3-cluster-approval-management-database
key: postgres-approval-manager-url
- name: QL3_POSTGRES_APPROVAL_MANAGER_TLS_SERVERNAME
valueFrom:
secretKeyRef:
name: ql3-cluster-approval-management-database
key: postgres-tls-servername
ports:
- name: https
containerPort: 8447
protocol: TCP
startupProbe:
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 2
timeoutSeconds: 1
failureThreshold: 30
readinessProbe:
httpGet:
path: /readyz
port: https
scheme: HTTPS
periodSeconds: 5
timeoutSeconds: 2
failureThreshold: 2
livenessProbe:
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
resources:
requests:
cpu: 50m
memory: 96Mi
limits:
cpu: '1'
memory: 384Mi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: management-tls
mountPath: /var/run/secrets/qinglong3/approval-management-tls
readOnly: true
- name: management-identity
mountPath: /var/run/qinglong3/approval-management-identity
readOnly: true
- name: postgres-approval-manager-ca
mountPath: /var/run/secrets/qinglong3/postgres-approval-manager
readOnly: true
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 16Mi
- name: management-tls
secret:
secretName: ql3-approval-management-tls
defaultMode: 288
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
- key: ca.crt
path: ca.crt
- key: client.crl
path: client.crl
- name: management-identity
secret:
secretName: ql3-approval-management-identity
defaultMode: 292
items:
- key: keyset.json
path: keyset.json
- name: postgres-approval-manager-ca
secret:
secretName: ql3-cluster-approval-management-database
defaultMode: 292
items:
- key: postgres-ca.crt
path: ca.crt
@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- service-account.yaml
- service.yaml
- deployment.yaml
- pod-disruption-budget.yaml
- network-policy.yaml
@@ -0,0 +1,38 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-approval-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
app.kubernetes.io/part-of: qinglong3
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
qinglong.io/approval-management-client: 'true'
ports:
- protocol: TCP
port: 8447
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
@@ -0,0 +1,15 @@
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: ql3-approval-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
app.kubernetes.io/part-of: qinglong3
spec:
minAvailable: 1
selector:
matchLabels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: ql3-approval-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
app.kubernetes.io/part-of: qinglong3
automountServiceAccountToken: false
@@ -0,0 +1,19 @@
apiVersion: v1
kind: Service
metadata:
name: ql3-approval-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
app.kubernetes.io/part-of: qinglong3
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: ql3-approval-management
app.kubernetes.io/component: approval-management
ports:
- name: https
port: 8447
targetPort: https
protocol: TCP
@@ -0,0 +1,71 @@
- op: replace
path: /spec/template/spec/containers/0/env
value:
- name: QL3_PROFILE
value: cluster-admin
- name: QL3_APPROVAL_MANAGEMENT_ENABLED
value: 'true'
- name: QL3_APPROVAL_MANAGEMENT_HOST
value: 0.0.0.0
- name: QL3_APPROVAL_MANAGEMENT_PORT
value: '8447'
- name: QL3_APPROVAL_MANAGEMENT_TLS_CERT_FILE
value: /var/run/secrets/qinglong3/approval-management-tls/tls.crt
- name: QL3_APPROVAL_MANAGEMENT_TLS_KEY_FILE
value: /var/run/secrets/qinglong3/approval-management-tls/tls.key
- name: QL3_APPROVAL_MANAGEMENT_CLIENT_CA_FILE
value: /var/run/secrets/qinglong3/approval-management-tls/ca.crt
- name: QL3_APPROVAL_MANAGEMENT_CLIENT_CRL_FILE
value: /var/run/secrets/qinglong3/approval-management-tls/client.crl
- name: QL3_APPROVAL_MANAGEMENT_IDENTITY_KEYSET_FILE
value: /var/run/qinglong3/approval-management-identity/keyset.json
- name: QL3_APPROVAL_MANAGEMENT_MAX_BODY_BYTES
value: '65536'
- name: QL3_APPROVAL_MANAGEMENT_MAX_CONNECTIONS
value: '32'
- name: QL3_APPROVAL_MANAGEMENT_MAX_CONCURRENT_REQUESTS
value: '16'
- name: QL3_APPROVAL_MANAGEMENT_REQUEST_TIMEOUT_MS
value: '10000'
- name: QL3_APPROVAL_MANAGEMENT_DRAIN_TIMEOUT_MS
value: '5000'
- name: QL3_APPROVAL_MANAGEMENT_RATE_WINDOW_MS
value: '60000'
- name: QL3_APPROVAL_MANAGEMENT_PEER_REQUEST_LIMIT
value: '30'
- name: QL3_APPROVAL_MANAGEMENT_GLOBAL_REQUEST_LIMIT
value: '300'
- name: QL3_APPROVAL_MANAGEMENT_MAX_RATE_LIMIT_PEERS
value: '1024'
- name: QL3_POSTGRES_APPROVAL_MANAGER_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_APPROVAL_MANAGER_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-approval-manager/ca.crt
- name: QL3_POSTGRES_APPROVAL_MANAGER_APPLICATION_NAME
value: qinglong3-approval-manager
- name: QL3_POSTGRES_APPROVAL_MANAGER_POOL_MAX
value: '2'
- name: QL3_POSTGRES_APPROVAL_MANAGER_HOST
value: ql3-postgres-rw.qinglong3-system.svc
- name: QL3_POSTGRES_APPROVAL_MANAGER_PORT
value: '5432'
- name: QL3_POSTGRES_APPROVAL_MANAGER_DATABASE
value: qinglong
- name: QL3_POSTGRES_APPROVAL_MANAGER_USER
valueFrom:
secretKeyRef:
name: ql3-postgres-approval-manager-auth
key: username
- name: QL3_POSTGRES_APPROVAL_MANAGER_PASSWORD
valueFrom:
secretKeyRef:
name: ql3-postgres-approval-manager-auth
key: password
- name: QL3_POSTGRES_APPROVAL_MANAGER_TLS_SERVERNAME
value: ql3-postgres-rw.qinglong3-system.svc
- op: replace
path: /spec/template/spec/volumes/3/secret/secretName
value: ql3-postgres-ca
- op: replace
path: /spec/template/spec/volumes/3/secret/items/0/key
value: ca.crt
@@ -0,0 +1,25 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../base
patches:
- target:
group: apps
version: v1
kind: Deployment
labelSelector: app.kubernetes.io/name=ql3-approval-management
path: deployment-patch.yaml
- target:
group: networking.k8s.io
version: v1
kind: NetworkPolicy
name: ql3-approval-management
path: network-policy-patch.yaml
images:
- name: qinglong3-cluster-admin
newName: registry.example.com/qinglong/qinglong3-cluster-admin
# Fail closed until the independently verified release digest is supplied.
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,26 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-approval-management
namespace: qinglong3-system
spec:
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
- to:
- podSelector:
matchLabels:
cnpg.io/cluster: ql3-postgres
ports:
- protocol: TCP
port: 5432
@@ -0,0 +1,72 @@
# Example only. Copy into a private overlay and replace every placeholder.
# This file is intentionally excluded from committed Kustomizations.
# Assertions require aud=qinglong3-approval-management,
# typ=ql3-approval-management+jwt and ql3_purpose=approval-management.
apiVersion: v1
kind: Secret
metadata:
name: ql3-approval-management-identity
namespace: qinglong3-system
type: Opaque
stringData:
keyset.json: |
{
"schemaVersion": 1,
"generation": 1,
"issuer": "https://identity.example.test/",
"audience": "qinglong3-approval-management",
"keys": [
{
"alg": "EdDSA",
"crv": "Ed25519",
"kid": "REPLACE_WITH_KEY_ID",
"kty": "OKP",
"use": "sig",
"x": "REPLACE_WITH_ED25519_PUBLIC_JWK_X"
}
],
"revokedKids": [],
"assuranceMappings": [
{
"acr": "urn:example:mfa",
"assurance": "multi_factor",
"requiredAmr": ["pwd", "otp"]
},
{
"acr": "urn:example:hardware",
"assurance": "hardware",
"requiredAmr": ["hwk"]
}
],
"constraints": {
"maxAssertionBytes": 8192,
"maxLifetimeMs": 300000,
"maxAuthenticationAgeMs": 300000,
"clockSkewMs": 5000
}
}
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-approval-management-tls
namespace: qinglong3-system
type: kubernetes.io/tls
stringData:
tls.crt: REPLACE_WITH_SERVER_CERTIFICATE_CHAIN
tls.key: REPLACE_WITH_SERVER_PRIVATE_KEY
ca.crt: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_AUTHORITIES
client.crl: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_REVOCATION_LISTS
---
# Non-CloudNativePG deployments only. The reviewed overlay uses
# ql3-postgres-approval-manager-auth and ql3-postgres-ca instead.
apiVersion: v1
kind: Secret
metadata:
name: ql3-cluster-approval-management-database
namespace: qinglong3-system
type: Opaque
stringData:
postgres-approval-manager-url: REPLACE_WITH_APPROVAL_MANAGER_DSN
postgres-tls-servername: REPLACE_WITH_POSTGRES_DNS_NAME
postgres-ca.crt: REPLACE_WITH_POSTGRES_CA_CERTIFICATE
@@ -0,0 +1,196 @@
apiVersion: batch/v1
kind: Job
metadata:
name: ql3-automation-management-client
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-automation-management-client
app.kubernetes.io/component: automation-management-client
app.kubernetes.io/part-of: qinglong3
qinglong.io/execution-model: caller-driven
spec:
backoffLimit: 0
activeDeadlineSeconds: 120
ttlSecondsAfterFinished: 600
template:
metadata:
labels:
app.kubernetes.io/name: ql3-automation-management-client
app.kubernetes.io/component: automation-management-client
app.kubernetes.io/part-of: qinglong3
qinglong.io/execution-model: caller-driven
qinglong.io/automation-management-client: 'true'
spec:
serviceAccountName: ql3-automation-management-client
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
initContainers:
- name: wait-for-manager
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- -e
args:
- |
const fs = require('node:fs');
const https = require('node:https');
const host = 'ql3-automation-management.qinglong3-system.svc';
const ca = fs.readFileSync('/var/run/qinglong3/automation-management-trust/ca.crt');
const cert = fs.readFileSync('/var/run/secrets/qinglong3/automation-management-client-identity/tls.crt');
const key = fs.readFileSync('/var/run/secrets/qinglong3/automation-management-client-identity/tls.key');
const probe = () => new Promise((resolve, reject) => {
const request = https.request({
host,
port: 8445,
path: '/readyz',
method: 'GET',
servername: host,
ca,
cert,
key,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
rejectUnauthorized: true,
agent: false,
headers: { connection: 'close' },
}, (response) => {
response.resume();
response.once('end', () =>
response.statusCode === 200
? resolve()
: reject(new Error('manager not ready')));
});
request.setTimeout(3000, () => request.destroy(new Error('timeout')));
request.once('error', reject);
request.end();
});
(async () => {
for (let attempt = 1; attempt <= 30; attempt += 1) {
try {
await probe();
return;
} catch {
if (attempt === 30) throw new Error('manager readiness unavailable');
await new Promise((resolve) => setTimeout(resolve, 1000));
}
}
})().catch((error) => {
process.stderr.write(`${error.message}\n`);
process.exitCode = 1;
});
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 5m
memory: 16Mi
limits:
cpu: 100m
memory: 64Mi
volumeMounts:
- name: trust
mountPath: /var/run/qinglong3/automation-management-trust
readOnly: true
- name: client-identity
mountPath: /var/run/secrets/qinglong3/automation-management-client-identity
readOnly: true
containers:
- name: client
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- /bin/sh
- -c
args:
- |
set -eu
umask 077
cp /var/run/secrets/qinglong3/automation-management-request/client.json /tmp/client.json
cp /var/run/secrets/qinglong3/automation-management-request/command.json /tmp/command.json
cp /var/run/secrets/qinglong3/automation-management-assertion/assertion.jwt /tmp/assertion.jwt
cp /var/run/qinglong3/automation-management-trust/ca.crt /tmp/ca.crt
cp /var/run/secrets/qinglong3/automation-management-client-identity/tls.crt /tmp/client.crt
cp /var/run/secrets/qinglong3/automation-management-client-identity/tls.key /tmp/client.key
chmod 600 /tmp/client.json /tmp/command.json /tmp/assertion.jwt /tmp/ca.crt /tmp/client.crt /tmp/client.key
exec node /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/automation-management/automationManagementClientCli.js \
--config=/tmp/client.json \
--command=/tmp/command.json \
--assertion=/tmp/assertion.jwt
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 25m
memory: 48Mi
limits:
cpu: 250m
memory: 128Mi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: request
mountPath: /var/run/secrets/qinglong3/automation-management-request
readOnly: true
- name: assertion
mountPath: /var/run/secrets/qinglong3/automation-management-assertion
readOnly: true
- name: trust
mountPath: /var/run/qinglong3/automation-management-trust
readOnly: true
- name: client-identity
mountPath: /var/run/secrets/qinglong3/automation-management-client-identity
readOnly: true
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 4Mi
- name: request
secret:
secretName: ql3-automation-management-request
defaultMode: 288
items:
- key: client.json
path: client.json
- key: command.json
path: command.json
- name: assertion
secret:
secretName: ql3-automation-management-assertion
defaultMode: 288
items:
- key: assertion.jwt
path: assertion.jwt
- name: trust
configMap:
name: ql3-automation-management-client-trust
defaultMode: 292
items:
- key: ca.crt
path: ca.crt
- name: client-identity
secret:
secretName: ql3-automation-management-client-identity
defaultMode: 288
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
@@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- service-account.yaml
- job.yaml
- network-policy.yaml
@@ -0,0 +1,39 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-automation-management-client
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-automation-management-client
app.kubernetes.io/component: automation-management-client
app.kubernetes.io/part-of: qinglong3
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: ql3-automation-management-client
app.kubernetes.io/component: automation-management-client
policyTypes:
- Ingress
- Egress
ingress: []
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
ports:
- protocol: TCP
port: 8445
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: ql3-automation-management-client
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-automation-management-client
app.kubernetes.io/component: automation-management-client
app.kubernetes.io/part-of: qinglong3
automountServiceAccountToken: false
@@ -0,0 +1,45 @@
# Example only. Copy these objects into a private, per-command overlay.
# Replace every placeholder. Never commit a Task body, assertion or private key.
apiVersion: v1
kind: Secret
metadata:
name: ql3-automation-management-request
namespace: qinglong3-system
immutable: true
type: Opaque
stringData:
client.json: |
{"schemaVersion":1,"endpoint":"https://ql3-automation-management.qinglong3-system.svc:8445/api/v3/automations/management","servername":"ql3-automation-management.qinglong3-system.svc","caFile":"/tmp/ca.crt","clientCertificateFile":"/tmp/client.crt","clientPrivateKeyFile":"/tmp/client.key","requestTimeoutMs":15000}
command.json: |
{"schemaVersion":1,"operation":"task.publish","request":{"requestId":"REPLACE_WITH_UNIQUE_REQUEST_ID","command":{"projectId":"REPLACE_WITH_PROJECT_ID","taskId":"REPLACE_WITH_TASK_ID","expectedRevision":null,"mutationId":"REPLACE_WITH_UUID_V4","name":"REPLACE_WITH_TASK_NAME","kind":"script","spec":{"schema":"qinglong/script@v1","config":{"source":"REPLACE_WITH_REVIEWED_TASK_SOURCE"}},"labels":{},"enabled":true,"occurredAtMs":0}}}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: ql3-automation-management-client-trust
namespace: qinglong3-system
immutable: true
data:
ca.crt: |
REPLACE_WITH_REVIEWED_MANAGER_CA_CERTIFICATE
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-automation-management-assertion
namespace: qinglong3-system
immutable: true
type: Opaque
stringData:
assertion.jwt: REPLACE_WITH_SHORT_LIVED_STRONG_USER_ASSERTION
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-automation-management-client-identity
namespace: qinglong3-system
immutable: true
type: kubernetes.io/tls
stringData:
tls.crt: REPLACE_WITH_SHORT_LIVED_CLIENT_CERTIFICATE_CHAIN
tls.key: REPLACE_WITH_CLIENT_PRIVATE_KEY
@@ -0,0 +1,11 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- base
images:
- name: qinglong3-cluster-admin
newName: registry.example.com/qinglong/qinglong3-cluster-admin
# Fail closed until the independently verified release digest is supplied.
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,200 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ql3-automation-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
app.kubernetes.io/part-of: qinglong3
spec:
replicas: 2
minReadySeconds: 10
revisionHistoryLimit: 3
progressDeadlineSeconds: 600
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
template:
metadata:
annotations:
# Private production overlays must replace these with the exact bundle
# digests before changing the client trust generation.
qinglong.io/automation-management-client-ca-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
qinglong.io/automation-management-client-crl-sha256: sha256:0000000000000000000000000000000000000000000000000000000000000000
labels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
app.kubernetes.io/part-of: qinglong3
spec:
serviceAccountName: ql3-automation-management
automountServiceAccountToken: false
terminationGracePeriodSeconds: 30
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- topologyKey: kubernetes.io/hostname
labelSelector:
matchLabels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
containers:
- name: management
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/automation-management/automationManagementCli.js
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: QL3_PROFILE
value: cluster-admin
- name: QL3_AUTOMATION_MANAGEMENT_ENABLED
value: 'true'
- name: QL3_AUTOMATION_MANAGEMENT_HOST
value: 0.0.0.0
- name: QL3_AUTOMATION_MANAGEMENT_PORT
value: '8445'
- name: QL3_AUTOMATION_MANAGEMENT_TLS_CERT_FILE
value: /var/run/secrets/qinglong3/automation-management-tls/tls.crt
- name: QL3_AUTOMATION_MANAGEMENT_TLS_KEY_FILE
value: /var/run/secrets/qinglong3/automation-management-tls/tls.key
- name: QL3_AUTOMATION_MANAGEMENT_CLIENT_CA_FILE
value: /var/run/secrets/qinglong3/automation-management-tls/ca.crt
- name: QL3_AUTOMATION_MANAGEMENT_CLIENT_CRL_FILE
value: /var/run/secrets/qinglong3/automation-management-tls/client.crl
- name: QL3_AUTOMATION_MANAGEMENT_IDENTITY_KEYSET_FILE
value: /var/run/qinglong3/automation-management-identity/keyset.json
- name: QL3_AUTOMATION_MANAGEMENT_MAX_BODY_BYTES
value: '65536'
- name: QL3_AUTOMATION_MANAGEMENT_MAX_CONNECTIONS
value: '32'
- name: QL3_AUTOMATION_MANAGEMENT_MAX_CONCURRENT_REQUESTS
value: '16'
- name: QL3_AUTOMATION_MANAGEMENT_REQUEST_TIMEOUT_MS
value: '10000'
- name: QL3_AUTOMATION_MANAGEMENT_DRAIN_TIMEOUT_MS
value: '5000'
- name: QL3_AUTOMATION_MANAGEMENT_RATE_WINDOW_MS
value: '60000'
- name: QL3_AUTOMATION_MANAGEMENT_PEER_REQUEST_LIMIT
value: '60'
- name: QL3_AUTOMATION_MANAGEMENT_GLOBAL_REQUEST_LIMIT
value: '600'
- name: QL3_AUTOMATION_MANAGEMENT_MAX_RATE_LIMIT_PEERS
value: '1024'
- name: QL3_POSTGRES_AUTOMATION_MANAGER_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_AUTOMATION_MANAGER_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-automation-manager/ca.crt
- name: QL3_POSTGRES_AUTOMATION_MANAGER_APPLICATION_NAME
value: qinglong3-automation-manager
- name: QL3_POSTGRES_AUTOMATION_MANAGER_POOL_MAX
value: '2'
- name: QL3_POSTGRES_AUTOMATION_MANAGER_URL
valueFrom:
secretKeyRef:
name: ql3-cluster-automation-management-database
key: postgres-automation-manager-url
- name: QL3_POSTGRES_AUTOMATION_MANAGER_TLS_SERVERNAME
valueFrom:
secretKeyRef:
name: ql3-cluster-automation-management-database
key: postgres-tls-servername
ports:
- name: https
containerPort: 8445
protocol: TCP
startupProbe:
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 2
timeoutSeconds: 1
failureThreshold: 30
readinessProbe:
httpGet:
path: /readyz
port: https
scheme: HTTPS
periodSeconds: 5
timeoutSeconds: 2
failureThreshold: 2
livenessProbe:
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: '1'
memory: 512Mi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: management-tls
mountPath: /var/run/secrets/qinglong3/automation-management-tls
readOnly: true
- name: management-identity
mountPath: /var/run/qinglong3/automation-management-identity
readOnly: true
- name: postgres-automation-manager-ca
mountPath: /var/run/secrets/qinglong3/postgres-automation-manager
readOnly: true
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 16Mi
- name: management-tls
secret:
secretName: ql3-automation-management-tls
defaultMode: 288
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
- key: ca.crt
path: ca.crt
- key: client.crl
path: client.crl
- name: management-identity
secret:
secretName: ql3-automation-management-identity
defaultMode: 292
items:
- key: keyset.json
path: keyset.json
- name: postgres-automation-manager-ca
secret:
secretName: ql3-cluster-automation-management-database
defaultMode: 292
items:
- key: postgres-ca.crt
path: ca.crt
@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- service-account.yaml
- service.yaml
- deployment.yaml
- pod-disruption-budget.yaml
- network-policy.yaml
@@ -0,0 +1,38 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-automation-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
app.kubernetes.io/part-of: qinglong3
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
qinglong.io/automation-management-client: 'true'
ports:
- protocol: TCP
port: 8445
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
@@ -0,0 +1,15 @@
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: ql3-automation-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
app.kubernetes.io/part-of: qinglong3
spec:
minAvailable: 1
selector:
matchLabels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: ql3-automation-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
app.kubernetes.io/part-of: qinglong3
automountServiceAccountToken: false
@@ -0,0 +1,19 @@
apiVersion: v1
kind: Service
metadata:
name: ql3-automation-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
app.kubernetes.io/part-of: qinglong3
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: ql3-automation-management
app.kubernetes.io/component: automation-management
ports:
- name: https
port: 8445
targetPort: https
protocol: TCP
@@ -0,0 +1,71 @@
- op: replace
path: /spec/template/spec/containers/0/env
value:
- name: QL3_PROFILE
value: cluster-admin
- name: QL3_AUTOMATION_MANAGEMENT_ENABLED
value: 'true'
- name: QL3_AUTOMATION_MANAGEMENT_HOST
value: 0.0.0.0
- name: QL3_AUTOMATION_MANAGEMENT_PORT
value: '8445'
- name: QL3_AUTOMATION_MANAGEMENT_TLS_CERT_FILE
value: /var/run/secrets/qinglong3/automation-management-tls/tls.crt
- name: QL3_AUTOMATION_MANAGEMENT_TLS_KEY_FILE
value: /var/run/secrets/qinglong3/automation-management-tls/tls.key
- name: QL3_AUTOMATION_MANAGEMENT_CLIENT_CA_FILE
value: /var/run/secrets/qinglong3/automation-management-tls/ca.crt
- name: QL3_AUTOMATION_MANAGEMENT_CLIENT_CRL_FILE
value: /var/run/secrets/qinglong3/automation-management-tls/client.crl
- name: QL3_AUTOMATION_MANAGEMENT_IDENTITY_KEYSET_FILE
value: /var/run/qinglong3/automation-management-identity/keyset.json
- name: QL3_AUTOMATION_MANAGEMENT_MAX_BODY_BYTES
value: '65536'
- name: QL3_AUTOMATION_MANAGEMENT_MAX_CONNECTIONS
value: '32'
- name: QL3_AUTOMATION_MANAGEMENT_MAX_CONCURRENT_REQUESTS
value: '16'
- name: QL3_AUTOMATION_MANAGEMENT_REQUEST_TIMEOUT_MS
value: '10000'
- name: QL3_AUTOMATION_MANAGEMENT_DRAIN_TIMEOUT_MS
value: '5000'
- name: QL3_AUTOMATION_MANAGEMENT_RATE_WINDOW_MS
value: '60000'
- name: QL3_AUTOMATION_MANAGEMENT_PEER_REQUEST_LIMIT
value: '60'
- name: QL3_AUTOMATION_MANAGEMENT_GLOBAL_REQUEST_LIMIT
value: '600'
- name: QL3_AUTOMATION_MANAGEMENT_MAX_RATE_LIMIT_PEERS
value: '1024'
- name: QL3_POSTGRES_AUTOMATION_MANAGER_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_AUTOMATION_MANAGER_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-automation-manager/ca.crt
- name: QL3_POSTGRES_AUTOMATION_MANAGER_APPLICATION_NAME
value: qinglong3-automation-manager
- name: QL3_POSTGRES_AUTOMATION_MANAGER_POOL_MAX
value: '2'
- name: QL3_POSTGRES_AUTOMATION_MANAGER_HOST
value: ql3-postgres-rw.qinglong3-system.svc
- name: QL3_POSTGRES_AUTOMATION_MANAGER_PORT
value: '5432'
- name: QL3_POSTGRES_AUTOMATION_MANAGER_DATABASE
value: qinglong
- name: QL3_POSTGRES_AUTOMATION_MANAGER_USER
valueFrom:
secretKeyRef:
name: ql3-postgres-automation-manager-auth
key: username
- name: QL3_POSTGRES_AUTOMATION_MANAGER_PASSWORD
valueFrom:
secretKeyRef:
name: ql3-postgres-automation-manager-auth
key: password
- name: QL3_POSTGRES_AUTOMATION_MANAGER_TLS_SERVERNAME
value: ql3-postgres-rw.qinglong3-system.svc
- op: replace
path: /spec/template/spec/volumes/3/secret/secretName
value: ql3-postgres-ca
- op: replace
path: /spec/template/spec/volumes/3/secret/items/0/key
value: ca.crt
@@ -0,0 +1,25 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../base
patches:
- target:
group: apps
version: v1
kind: Deployment
labelSelector: app.kubernetes.io/name=ql3-automation-management
path: deployment-patch.yaml
- target:
group: networking.k8s.io
version: v1
kind: NetworkPolicy
name: ql3-automation-management
path: network-policy-patch.yaml
images:
- name: qinglong3-cluster-admin
newName: registry.example.com/qinglong/qinglong3-cluster-admin
# Fail closed until the independently verified release digest is supplied.
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,26 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-automation-management
namespace: qinglong3-system
spec:
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
- to:
- podSelector:
matchLabels:
cnpg.io/cluster: ql3-postgres
ports:
- protocol: TCP
port: 5432
@@ -0,0 +1,75 @@
# Example only. Copy into a private overlay and replace every placeholder.
# This file is intentionally excluded from all committed Kustomizations.
# Assertions signed by this keyset must use all three automation-specific values:
# aud=qinglong3-automation-management, typ=ql3-automation-management+jwt, and
# ql3_purpose=automation-management. Plugin Package and Worker assertions are
# rejected even when they use the same issuer and signing key.
apiVersion: v1
kind: Secret
metadata:
name: ql3-automation-management-identity
namespace: qinglong3-system
type: Opaque
stringData:
keyset.json: |
{
"schemaVersion": 1,
"generation": 1,
"issuer": "https://identity.example.test/",
"audience": "qinglong3-automation-management",
"keys": [
{
"alg": "EdDSA",
"crv": "Ed25519",
"kid": "REPLACE_WITH_KEY_ID",
"kty": "OKP",
"use": "sig",
"x": "REPLACE_WITH_ED25519_PUBLIC_JWK_X"
}
],
"revokedKids": [],
"assuranceMappings": [
{
"acr": "urn:example:mfa",
"assurance": "multi_factor",
"requiredAmr": ["pwd", "otp"]
},
{
"acr": "urn:example:hardware",
"assurance": "hardware",
"requiredAmr": ["hwk"]
}
],
"constraints": {
"maxAssertionBytes": 8192,
"maxLifetimeMs": 300000,
"maxAuthenticationAgeMs": 300000,
"clockSkewMs": 5000
}
}
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-automation-management-tls
namespace: qinglong3-system
type: kubernetes.io/tls
stringData:
tls.crt: REPLACE_WITH_SERVER_CERTIFICATE_CHAIN
tls.key: REPLACE_WITH_SERVER_PRIVATE_KEY
ca.crt: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_AUTHORITIES
client.crl: REPLACE_WITH_1_TO_16_CLIENT_CERTIFICATE_REVOCATION_LISTS
---
# Non-CloudNativePG deployments only. The reviewed CloudNativePG overlay uses
# the operator-managed ql3-postgres-automation-manager-auth and ql3-postgres-ca
# Secrets instead of this resource.
apiVersion: v1
kind: Secret
metadata:
name: ql3-cluster-automation-management-database
namespace: qinglong3-system
type: Opaque
stringData:
postgres-automation-manager-url: REPLACE_WITH_AUTOMATION_MANAGER_DSN
postgres-tls-servername: REPLACE_WITH_POSTGRES_DNS_NAME
postgres-ca.crt: REPLACE_WITH_POSTGRES_CA_CERTIFICATE
@@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- migrate-job.yaml
@@ -0,0 +1,85 @@
apiVersion: batch/v1
kind: Job
metadata:
name: ql3-cluster-migration
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-cluster-migration
app.kubernetes.io/component: database-migration
app.kubernetes.io/part-of: qinglong3
spec:
backoffLimit: 0
activeDeadlineSeconds: 600
ttlSecondsAfterFinished: 600
template:
metadata:
labels:
app.kubernetes.io/name: ql3-cluster-migration
app.kubernetes.io/component: database-migration
app.kubernetes.io/part-of: qinglong3
spec:
serviceAccountName: ql3-cluster-control
automountServiceAccountToken: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: migration
image: qinglong3-cluster-control:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- /opt/qinglong/node_modules/@qinglong/cluster-postgres/dist/migration/migrationCli.js
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: QL3_POSTGRES_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-migration/ca.crt
- name: QL3_POSTGRES_APPLICATION_NAME
value: qinglong3-cluster-migration
- name: QL3_POSTGRES_MIGRATION_URL
valueFrom:
secretKeyRef:
name: ql3-cluster-migration
key: postgres-migration-url
- name: QL3_POSTGRES_TLS_SERVERNAME
valueFrom:
secretKeyRef:
name: ql3-cluster-migration
key: postgres-tls-servername
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: '1'
memory: 256Mi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: postgres-migration-ca
mountPath: /var/run/secrets/qinglong3/postgres-migration
readOnly: true
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 16Mi
- name: postgres-migration-ca
secret:
secretName: ql3-cluster-migration
defaultMode: 292
items:
- key: postgres-ca.crt
path: ca.crt
@@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: qinglong3-system
resources:
- restore-cluster.yaml
@@ -0,0 +1,27 @@
# Example only. Recovery credentials should be read-only wherever the provider
# supports separate read/list and write/delete authorities. This resource is
# intentionally excluded from Kustomize.
apiVersion: barmancloud.cnpg.io/v1
kind: ObjectStore
metadata:
name: ql3-postgres-recovery-source
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-postgres-restore
app.kubernetes.io/component: database-restore-source
app.kubernetes.io/part-of: qinglong3
spec:
configuration:
destinationPath: s3://REPLACE_WITH_VERSIONED_LOCKED_BUCKET/qinglong3/ql3-postgres
endpointURL: https://REPLACE_WITH_OBJECT_STORE_ENDPOINT
s3Credentials:
accessKeyId:
name: ql3-postgres-restore-object-store
key: ACCESS_KEY_ID
secretAccessKey:
name: ql3-postgres-restore-object-store
key: ACCESS_SECRET_KEY
wal:
compression: lz4
encryption: AES256
maxParallel: 4
@@ -0,0 +1,59 @@
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: ql3-postgres-restore
labels:
app.kubernetes.io/name: ql3-postgres-restore
app.kubernetes.io/component: database-restore-drill
app.kubernetes.io/part-of: qinglong3
app.kubernetes.io/managed-by: cloudnative-pg
spec:
instances: 3
imageName: ghcr.io/cloudnative-pg/postgresql:18.4-minimal-trixie@sha256:24d229d801663f95b584416f8ebdfad4849b1a3fa4cfcf95a7f026df7aa6e22d
enableSuperuserAccess: false
primaryUpdateMethod: switchover
failoverDelay: 0
switchoverDelay: 60
smartShutdownTimeout: 60
stopDelay: 300
bootstrap:
recovery:
source: ql3-postgres-origin
externalClusters:
- name: ql3-postgres-origin
plugin:
name: barman-cloud.cloudnative-pg.io
parameters:
barmanObjectName: ql3-postgres-recovery-source
serverName: ql3-postgres
postgresql:
parameters:
max_connections: '100'
password_encryption: scram-sha-256
shared_buffers: 128MB
synchronous_commit: remote_apply
synchronous:
method: any
number: 1
dataDurability: required
failoverQuorum: true
affinity:
enablePodAntiAffinity: true
podAntiAffinityType: required
topologyKey: kubernetes.io/hostname
managed:
services:
disabledDefaultServices:
- r
- ro
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: '2'
memory: 1Gi
storage:
size: 20Gi
walStorage:
size: 5Gi
@@ -0,0 +1,19 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../base
images:
- name: qinglong3-cluster-control
newName: registry.example.com/qinglong/qinglong3-cluster-control
# Fail closed until the independently verified release digest is supplied.
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
patches:
- target:
group: batch
version: v1
kind: Job
labelSelector: app.kubernetes.io/name=ql3-cluster-migration
path: migrate-job-patch.yaml
@@ -0,0 +1,33 @@
- op: replace
path: /spec/template/spec/containers/0/env
value:
- name: QL3_POSTGRES_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-migration/ca.crt
- name: QL3_POSTGRES_APPLICATION_NAME
value: qinglong3-cluster-migration
- name: QL3_POSTGRES_MIGRATION_HOST
value: ql3-postgres-rw.qinglong3-system.svc
- name: QL3_POSTGRES_MIGRATION_PORT
value: '5432'
- name: QL3_POSTGRES_MIGRATION_DATABASE
value: qinglong
- name: QL3_POSTGRES_MIGRATION_USER
valueFrom:
secretKeyRef:
name: ql3-postgres-migration-auth
key: username
- name: QL3_POSTGRES_MIGRATION_PASSWORD
valueFrom:
secretKeyRef:
name: ql3-postgres-migration-auth
key: password
- name: QL3_POSTGRES_TLS_SERVERNAME
value: ql3-postgres-rw.qinglong3-system.svc
- op: replace
path: /spec/template/spec/volumes/1/secret/secretName
value: ql3-postgres-ca
- op: replace
path: /spec/template/spec/volumes/1/secret/items/0/key
value: ca.crt
@@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- base
@@ -0,0 +1,110 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: ql3-plugin-package-executor
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-executor
app.kubernetes.io/component: plugin-package-executor
app.kubernetes.io/part-of: qinglong3
spec:
schedule: '*/2 * * * *'
concurrencyPolicy: Forbid
startingDeadlineSeconds: 60
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 2
activeDeadlineSeconds: 600
ttlSecondsAfterFinished: 600
template:
metadata:
labels:
app.kubernetes.io/name: ql3-plugin-package-executor
app.kubernetes.io/component: plugin-package-executor
app.kubernetes.io/part-of: qinglong3
spec:
serviceAccountName: ql3-plugin-package-executor
automountServiceAccountToken: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: executor
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/plugin-package/executor/pluginPackageExecutorCli.js
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_ENABLED
value: 'true'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_OWNER
value: cluster_package_executor
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_APPROVAL_BATCH_SIZE
value: '8'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_DISPATCH_BATCH_SIZE
value: '8'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_MAX_BATCHES
value: '4'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_LEASE_DURATION_MS
value: '600000'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_PAGE_SIZE
value: '16'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_MAX_PAGES
value: '16'
- name: QL3_POSTGRES_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-package-executor/ca.crt
- name: QL3_POSTGRES_APPLICATION_NAME
value: qinglong3-plugin-package-executor
- name: QL3_POSTGRES_MAX_CONNECTIONS
value: '2'
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_URL
valueFrom:
secretKeyRef:
name: ql3-cluster-plugin-package-executor
key: postgres-package-executor-url
- name: QL3_POSTGRES_TLS_SERVERNAME
valueFrom:
secretKeyRef:
name: ql3-cluster-plugin-package-executor
key: postgres-tls-servername
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: postgres-package-executor-ca
mountPath: /var/run/secrets/qinglong3/postgres-package-executor
readOnly: true
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 8Mi
- name: postgres-package-executor-ca
secret:
secretName: ql3-cluster-plugin-package-executor
defaultMode: 292
items:
- key: postgres-ca.crt
path: ca.crt
@@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- service-account.yaml
- cron-job.yaml
- network-policy.yaml
@@ -0,0 +1,31 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-plugin-package-executor
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-executor
app.kubernetes.io/component: plugin-package-executor
app.kubernetes.io/part-of: qinglong3
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: ql3-plugin-package-executor
app.kubernetes.io/component: plugin-package-executor
policyTypes:
- Ingress
- Egress
ingress: []
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: ql3-plugin-package-executor
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-executor
app.kubernetes.io/component: plugin-package-executor
app.kubernetes.io/part-of: qinglong3
automountServiceAccountToken: false
@@ -0,0 +1,51 @@
- op: replace
path: /spec/jobTemplate/spec/template/spec/containers/0/env
value:
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_ENABLED
value: 'true'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_OWNER
value: cluster_package_executor
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_APPROVAL_BATCH_SIZE
value: '8'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_DISPATCH_BATCH_SIZE
value: '8'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_MAX_BATCHES
value: '4'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_LEASE_DURATION_MS
value: '600000'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_PAGE_SIZE
value: '16'
- name: QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_MAX_PAGES
value: '16'
- name: QL3_POSTGRES_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-package-executor/ca.crt
- name: QL3_POSTGRES_APPLICATION_NAME
value: qinglong3-plugin-package-executor
- name: QL3_POSTGRES_MAX_CONNECTIONS
value: '2'
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_HOST
value: ql3-postgres-rw.qinglong3-system.svc
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_PORT
value: '5432'
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_DATABASE
value: qinglong
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_USER
valueFrom:
secretKeyRef:
name: ql3-postgres-package-executor-auth
key: username
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_PASSWORD
valueFrom:
secretKeyRef:
name: ql3-postgres-package-executor-auth
key: password
- name: QL3_POSTGRES_TLS_SERVERNAME
value: ql3-postgres-rw.qinglong3-system.svc
- op: replace
path: /spec/jobTemplate/spec/template/spec/volumes/1/secret/secretName
value: ql3-postgres-ca
- op: replace
path: /spec/jobTemplate/spec/template/spec/volumes/1/secret/items/0/key
value: ca.crt
@@ -0,0 +1,25 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../base
patches:
- target:
group: batch
version: v1
kind: CronJob
labelSelector: app.kubernetes.io/name=ql3-plugin-package-executor
path: cron-job-patch.yaml
- target:
group: networking.k8s.io
version: v1
kind: NetworkPolicy
name: ql3-plugin-package-executor
path: network-policy-patch.yaml
images:
- name: qinglong3-cluster-admin
newName: registry.example.com/qinglong/qinglong3-cluster-admin
# Fail closed until the independently verified release digest is supplied.
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,10 @@
- op: add
path: /spec/egress/-
value:
to:
- podSelector:
matchLabels:
cnpg.io/cluster: ql3-postgres
ports:
- protocol: TCP
port: 5432
@@ -0,0 +1,12 @@
# Example only. Copy into a private overlay and replace every placeholder.
# The executor must receive only the package-executor database credential.
apiVersion: v1
kind: Secret
metadata:
name: ql3-cluster-plugin-package-executor
namespace: qinglong3-system
type: Opaque
stringData:
postgres-package-executor-url: REPLACE_WITH_PACKAGE_EXECUTOR_DSN
postgres-tls-servername: REPLACE_WITH_POSTGRES_DNS_NAME
postgres-ca.crt: REPLACE_WITH_POSTGRES_CA_CERTIFICATE
@@ -0,0 +1,215 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ql3-plugin-package-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
app.kubernetes.io/part-of: qinglong3
spec:
replicas: 2
minReadySeconds: 10
revisionHistoryLimit: 3
progressDeadlineSeconds: 600
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
template:
metadata:
labels:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
app.kubernetes.io/part-of: qinglong3
spec:
serviceAccountName: ql3-plugin-package-management
automountServiceAccountToken: false
terminationGracePeriodSeconds: 30
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- topologyKey: kubernetes.io/hostname
labelSelector:
matchLabels:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
containers:
- name: management
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/plugin-package/management/pluginPackageManagementCli.js
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: QL3_PROFILE
value: cluster-admin
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_ENABLED
value: 'true'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_HOST
value: 0.0.0.0
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_PORT
value: '8443'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_TLS_CERT_FILE
value: /var/run/secrets/qinglong3/plugin-package-management-tls/tls.crt
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_TLS_KEY_FILE
value: /var/run/secrets/qinglong3/plugin-package-management-tls/tls.key
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_IDENTITY_KEYSET_FILE
value: /var/run/qinglong3/plugin-package-management-identity/keyset.json
- name: QL3_PLUGIN_PACKAGE_PUBLISHER_TRUST_FILE
value: /var/run/qinglong3/plugin-package-trust/publishers.json
- name: QL3_PLUGIN_PACKAGE_TRUST_AUTHORITY_PROJECT_ID
value: cluster-trust-authority
- name: QL3_PLUGIN_PACKAGE_TRUST_AUTHORITY_ID
value: cluster
- name: QL3_PLUGIN_PACKAGE_TRUST_OBSERVER_ID
value: cluster-package-manager
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_APPROVAL_LIFETIME_MS
value: '900000'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_QUOTA_WINDOW_MS
value: '60000'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_PROPOSE_QUOTA
value: '30'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_DECIDE_QUOTA
value: '60'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_INSPECT_QUOTA
value: '600'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_MAX_BODY_BYTES
value: '65536'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_MAX_CONNECTIONS
value: '32'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_MAX_CONCURRENT_REQUESTS
value: '16'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_REQUEST_TIMEOUT_MS
value: '10000'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_DRAIN_TIMEOUT_MS
value: '10000'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_RATE_WINDOW_MS
value: '60000'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_PEER_REQUEST_LIMIT
value: '60'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_GLOBAL_REQUEST_LIMIT
value: '600'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_MAX_RATE_LIMIT_PEERS
value: '512'
- name: QL3_POSTGRES_PACKAGE_MANAGER_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_PACKAGE_MANAGER_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-package-manager/ca.crt
- name: QL3_POSTGRES_PACKAGE_MANAGER_APPLICATION_NAME
value: qinglong3-plugin-package-manager
- name: QL3_POSTGRES_PACKAGE_MANAGER_MAX_CONNECTIONS
value: '2'
- name: QL3_POSTGRES_PACKAGE_MANAGER_URL
valueFrom:
secretKeyRef:
name: ql3-cluster-plugin-package-management-database
key: postgres-package-manager-url
- name: QL3_POSTGRES_PACKAGE_MANAGER_TLS_SERVERNAME
valueFrom:
secretKeyRef:
name: ql3-cluster-plugin-package-management-database
key: postgres-tls-servername
ports:
- name: https
containerPort: 8443
protocol: TCP
startupProbe:
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 2
timeoutSeconds: 1
failureThreshold: 30
readinessProbe:
httpGet:
path: /readyz
port: https
scheme: HTTPS
periodSeconds: 5
timeoutSeconds: 2
failureThreshold: 2
livenessProbe:
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
timeoutSeconds: 2
failureThreshold: 3
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: '1'
memory: 512Mi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: management-tls
mountPath: /var/run/secrets/qinglong3/plugin-package-management-tls
readOnly: true
- name: management-identity
mountPath: /var/run/qinglong3/plugin-package-management-identity
readOnly: true
- name: plugin-package-trust
mountPath: /var/run/qinglong3/plugin-package-trust
readOnly: true
- name: postgres-package-manager-ca
mountPath: /var/run/secrets/qinglong3/postgres-package-manager
readOnly: true
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 16Mi
- name: management-tls
secret:
secretName: ql3-plugin-package-management-tls
defaultMode: 288
items:
- key: tls.crt
path: tls.crt
- key: tls.key
path: tls.key
- name: management-identity
secret:
secretName: ql3-plugin-package-management-identity
defaultMode: 292
items:
- key: keyset.json
path: keyset.json
- name: plugin-package-trust
configMap:
name: ql3-plugin-publisher-trust
defaultMode: 292
items:
- key: publishers.json
path: publishers.json
- name: postgres-package-manager-ca
secret:
secretName: ql3-cluster-plugin-package-management-database
defaultMode: 292
items:
- key: postgres-ca.crt
path: ca.crt
@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- service-account.yaml
- service.yaml
- deployment.yaml
- pod-disruption-budget.yaml
- network-policy.yaml
@@ -0,0 +1,38 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-plugin-package-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
app.kubernetes.io/part-of: qinglong3
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
qinglong.io/plugin-package-management-client: 'true'
ports:
- protocol: TCP
port: 8443
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
@@ -0,0 +1,15 @@
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: ql3-plugin-package-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
app.kubernetes.io/part-of: qinglong3
spec:
minAvailable: 1
selector:
matchLabels:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: ql3-plugin-package-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
app.kubernetes.io/part-of: qinglong3
automountServiceAccountToken: false
@@ -0,0 +1,19 @@
apiVersion: v1
kind: Service
metadata:
name: ql3-plugin-package-management
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
app.kubernetes.io/part-of: qinglong3
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: ql3-plugin-package-management
app.kubernetes.io/component: plugin-package-management
ports:
- name: https
port: 8443
targetPort: https
protocol: TCP
@@ -0,0 +1,85 @@
- op: replace
path: /spec/template/spec/containers/0/env
value:
- name: QL3_PROFILE
value: cluster-admin
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_ENABLED
value: 'true'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_HOST
value: 0.0.0.0
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_PORT
value: '8443'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_TLS_CERT_FILE
value: /var/run/secrets/qinglong3/plugin-package-management-tls/tls.crt
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_TLS_KEY_FILE
value: /var/run/secrets/qinglong3/plugin-package-management-tls/tls.key
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_IDENTITY_KEYSET_FILE
value: /var/run/qinglong3/plugin-package-management-identity/keyset.json
- name: QL3_PLUGIN_PACKAGE_PUBLISHER_TRUST_FILE
value: /var/run/qinglong3/plugin-package-trust/publishers.json
- name: QL3_PLUGIN_PACKAGE_TRUST_AUTHORITY_PROJECT_ID
value: cluster-trust-authority
- name: QL3_PLUGIN_PACKAGE_TRUST_AUTHORITY_ID
value: cluster
- name: QL3_PLUGIN_PACKAGE_TRUST_OBSERVER_ID
value: cluster-package-manager
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_APPROVAL_LIFETIME_MS
value: '900000'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_QUOTA_WINDOW_MS
value: '60000'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_PROPOSE_QUOTA
value: '30'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_DECIDE_QUOTA
value: '60'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_INSPECT_QUOTA
value: '600'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_MAX_BODY_BYTES
value: '65536'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_MAX_CONNECTIONS
value: '32'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_MAX_CONCURRENT_REQUESTS
value: '16'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_REQUEST_TIMEOUT_MS
value: '10000'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_DRAIN_TIMEOUT_MS
value: '10000'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_RATE_WINDOW_MS
value: '60000'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_PEER_REQUEST_LIMIT
value: '60'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_GLOBAL_REQUEST_LIMIT
value: '600'
- name: QL3_PLUGIN_PACKAGE_MANAGEMENT_MAX_RATE_LIMIT_PEERS
value: '512'
- name: QL3_POSTGRES_PACKAGE_MANAGER_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_PACKAGE_MANAGER_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-package-manager/ca.crt
- name: QL3_POSTGRES_PACKAGE_MANAGER_APPLICATION_NAME
value: qinglong3-plugin-package-manager
- name: QL3_POSTGRES_PACKAGE_MANAGER_MAX_CONNECTIONS
value: '2'
- name: QL3_POSTGRES_PACKAGE_MANAGER_HOST
value: ql3-postgres-rw.qinglong3-system.svc
- name: QL3_POSTGRES_PACKAGE_MANAGER_PORT
value: '5432'
- name: QL3_POSTGRES_PACKAGE_MANAGER_DATABASE
value: qinglong
- name: QL3_POSTGRES_PACKAGE_MANAGER_USER
valueFrom:
secretKeyRef:
name: ql3-postgres-package-manager-auth
key: username
- name: QL3_POSTGRES_PACKAGE_MANAGER_PASSWORD
valueFrom:
secretKeyRef:
name: ql3-postgres-package-manager-auth
key: password
- name: QL3_POSTGRES_PACKAGE_MANAGER_TLS_SERVERNAME
value: ql3-postgres-rw.qinglong3-system.svc
- op: replace
path: /spec/template/spec/volumes/4/secret/secretName
value: ql3-postgres-ca
- op: replace
path: /spec/template/spec/volumes/4/secret/items/0/key
value: ca.crt
@@ -0,0 +1,25 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../base
patches:
- target:
group: apps
version: v1
kind: Deployment
labelSelector: app.kubernetes.io/name=ql3-plugin-package-management
path: deployment-patch.yaml
- target:
group: networking.k8s.io
version: v1
kind: NetworkPolicy
name: ql3-plugin-package-management
path: network-policy-patch.yaml
images:
- name: qinglong3-cluster-admin
newName: registry.example.com/qinglong/qinglong3-cluster-admin
# Fail closed until the independently verified release digest is supplied.
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,26 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-plugin-package-management
namespace: qinglong3-system
spec:
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
- to:
- podSelector:
matchLabels:
cnpg.io/cluster: ql3-postgres
ports:
- protocol: TCP
port: 5432
@@ -0,0 +1,84 @@
# Example only. Copy into a private overlay and replace every placeholder.
# This file is intentionally excluded from all committed Kustomizations.
apiVersion: v1
kind: Secret
metadata:
name: ql3-plugin-package-management-identity
namespace: qinglong3-system
type: Opaque
stringData:
keyset.json: |
{
"schemaVersion": 1,
"generation": 1,
"issuer": "https://identity.example.test/",
"audience": "qinglong3-package-management",
"keys": [
{
"alg": "EdDSA",
"crv": "Ed25519",
"kid": "REPLACE_WITH_KEY_ID",
"kty": "OKP",
"use": "sig",
"x": "REPLACE_WITH_ED25519_PUBLIC_JWK_X"
}
],
"revokedKids": [],
"assuranceMappings": [
{
"acr": "urn:example:mfa",
"assurance": "multi_factor",
"requiredAmr": ["pwd", "otp"]
}
],
"constraints": {
"maxAssertionBytes": 8192,
"maxLifetimeMs": 300000,
"maxAuthenticationAgeMs": 300000,
"clockSkewMs": 5000
}
}
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-plugin-package-management-tls
namespace: qinglong3-system
type: kubernetes.io/tls
stringData:
tls.crt: REPLACE_WITH_SERVER_CERTIFICATE_CHAIN
tls.key: REPLACE_WITH_SERVER_PRIVATE_KEY
---
apiVersion: v1
kind: ConfigMap
metadata:
name: ql3-plugin-publisher-trust
namespace: qinglong3-system
data:
publishers.json: |
{
"schema": "qinglong/plugin-package-publisher-trust@v1",
"keys": [
{
"publisher": "REPLACE_WITH_PUBLISHER",
"keyId": "REPLACE_WITH_KEY_ID",
"publicKeyPem": "REPLACE_WITH_ED25519_PUBLIC_KEY_PEM",
"notBeforeMs": 0,
"notAfterMs": 1
}
]
}
---
# Non-CloudNativePG deployments only. The reviewed CloudNativePG overlay uses
# the operator-managed ql3-postgres-package-manager-auth and ql3-postgres-ca
# Secrets instead of this resource.
apiVersion: v1
kind: Secret
metadata:
name: ql3-cluster-plugin-package-management-database
namespace: qinglong3-system
type: Opaque
stringData:
postgres-package-manager-url: REPLACE_WITH_PACKAGE_MANAGER_DSN
postgres-tls-servername: REPLACE_WITH_POSTGRES_DNS_NAME
postgres-ca.crt: REPLACE_WITH_POSTGRES_CA_CERTIFICATE
@@ -0,0 +1,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- rbac.yaml
- recover-job.yaml
@@ -0,0 +1,47 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: ql3-plugin-package-recovery
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-recovery
app.kubernetes.io/component: plugin-package-recovery
app.kubernetes.io/part-of: qinglong3
automountServiceAccountToken: false
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: ql3-plugin-package-recovery
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-recovery
app.kubernetes.io/component: plugin-package-recovery
app.kubernetes.io/part-of: qinglong3
rules:
- apiGroups:
- ''
resources:
- configmaps
verbs:
- get
- create
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: ql3-plugin-package-recovery
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-recovery
app.kubernetes.io/component: plugin-package-recovery
app.kubernetes.io/part-of: qinglong3
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ql3-plugin-package-recovery
subjects:
- kind: ServiceAccount
name: ql3-plugin-package-recovery
namespace: qinglong3-system
@@ -0,0 +1,120 @@
apiVersion: batch/v1
kind: Job
metadata:
name: ql3-plugin-package-recovery
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-plugin-package-recovery
app.kubernetes.io/component: plugin-package-recovery
app.kubernetes.io/part-of: qinglong3
spec:
backoffLimit: 0
activeDeadlineSeconds: 600
ttlSecondsAfterFinished: 600
template:
metadata:
labels:
app.kubernetes.io/name: ql3-plugin-package-recovery
app.kubernetes.io/component: plugin-package-recovery
app.kubernetes.io/part-of: qinglong3
spec:
serviceAccountName: ql3-plugin-package-recovery
automountServiceAccountToken: true
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: recovery
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/plugin-package/recovery/pluginPackageRecoveryCli.js
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: QL3_CLUSTER_IDENTITY
valueFrom:
configMapKeyRef:
name: ql3-plugin-package-recovery-config
key: cluster-identity
- name: QL3_KUBERNETES_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: QL3_PLUGIN_PACKAGE_OCI_REGISTRIES
valueFrom:
configMapKeyRef:
name: ql3-plugin-package-recovery-config
key: oci-registries
- name: QL3_PLUGIN_PACKAGE_PUBLISHER_TRUST_FILE
value: /var/run/qinglong3/plugin-package-trust/publishers.json
- name: QL3_PLUGIN_PACKAGE_TRUST_AUTHORITY_ID
value: cluster
- name: QL3_PLUGIN_PACKAGE_OCI_TIMEOUT_MS
value: '15000'
- name: QL3_PLUGIN_PACKAGE_RECOVERY_PAGE_SIZE
value: '16'
- name: QL3_PLUGIN_PACKAGE_RECOVERY_MAX_PAGES
value: '16'
- name: QL3_POSTGRES_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-package-executor/ca.crt
- name: QL3_POSTGRES_APPLICATION_NAME
value: qinglong3-plugin-package-recovery
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_URL
valueFrom:
secretKeyRef:
name: ql3-cluster-plugin-package-recovery
key: postgres-package-executor-url
- name: QL3_POSTGRES_TLS_SERVERNAME
valueFrom:
secretKeyRef:
name: ql3-cluster-plugin-package-recovery
key: postgres-tls-servername
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: '1'
memory: 512Mi
volumeMounts:
- name: tmp
mountPath: /tmp
- name: plugin-package-trust
mountPath: /var/run/qinglong3/plugin-package-trust
readOnly: true
- name: postgres-package-executor-ca
mountPath: /var/run/secrets/qinglong3/postgres-package-executor
readOnly: true
volumes:
- name: tmp
emptyDir:
medium: Memory
sizeLimit: 16Mi
- name: plugin-package-trust
configMap:
name: ql3-plugin-publisher-trust
defaultMode: 292
items:
- key: publishers.json
path: publishers.json
- name: postgres-package-executor-ca
secret:
secretName: ql3-cluster-plugin-package-recovery
defaultMode: 292
items:
- key: postgres-ca.crt
path: ca.crt
@@ -0,0 +1,19 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../base
patches:
- target:
group: batch
version: v1
kind: Job
labelSelector: app.kubernetes.io/name=ql3-plugin-package-recovery
path: recover-job-patch.yaml
images:
- name: qinglong3-cluster-admin
newName: registry.example.com/qinglong/qinglong3-cluster-admin
# Fail closed until the independently verified release digest is supplied.
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
@@ -0,0 +1,58 @@
- op: replace
path: /spec/template/spec/containers/0/env
value:
- name: QL3_CLUSTER_IDENTITY
valueFrom:
configMapKeyRef:
name: ql3-plugin-package-recovery-config
key: cluster-identity
- name: QL3_KUBERNETES_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: QL3_PLUGIN_PACKAGE_OCI_REGISTRIES
valueFrom:
configMapKeyRef:
name: ql3-plugin-package-recovery-config
key: oci-registries
- name: QL3_PLUGIN_PACKAGE_PUBLISHER_TRUST_FILE
value: /var/run/qinglong3/plugin-package-trust/publishers.json
- name: QL3_PLUGIN_PACKAGE_TRUST_AUTHORITY_ID
value: cluster
- name: QL3_PLUGIN_PACKAGE_OCI_TIMEOUT_MS
value: '15000'
- name: QL3_PLUGIN_PACKAGE_RECOVERY_PAGE_SIZE
value: '16'
- name: QL3_PLUGIN_PACKAGE_RECOVERY_MAX_PAGES
value: '16'
- name: QL3_POSTGRES_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-package-executor/ca.crt
- name: QL3_POSTGRES_APPLICATION_NAME
value: qinglong3-plugin-package-recovery
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_HOST
value: ql3-postgres-rw.qinglong3-system.svc
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_PORT
value: '5432'
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_DATABASE
value: qinglong
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_USER
valueFrom:
secretKeyRef:
name: ql3-postgres-package-executor-auth
key: username
- name: QL3_POSTGRES_PACKAGE_EXECUTOR_PASSWORD
valueFrom:
secretKeyRef:
name: ql3-postgres-package-executor-auth
key: password
- name: QL3_POSTGRES_TLS_SERVERNAME
value: ql3-postgres-rw.qinglong3-system.svc
- op: replace
path: /spec/template/spec/volumes/2/secret/secretName
value: ql3-postgres-ca
- op: replace
path: /spec/template/spec/volumes/2/secret/items/0/key
value: ca.crt
@@ -0,0 +1,31 @@
# Example only. Copy into a private deployment overlay and replace all values.
# Publisher public keys are not credentials, but they are release authority and
# must be reviewed independently from the package registry.
apiVersion: v1
kind: ConfigMap
metadata:
name: ql3-plugin-package-recovery-config
namespace: qinglong3-system
data:
cluster-identity: REPLACE_WITH_STABLE_CLUSTER_IDENTITY
oci-registries: ghcr.io
---
apiVersion: v1
kind: ConfigMap
metadata:
name: ql3-plugin-publisher-trust
namespace: qinglong3-system
data:
publishers.json: |
{
"schema": "qinglong/plugin-package-publisher-trust@v1",
"keys": [
{
"publisher": "REPLACE_WITH_PUBLISHER",
"keyId": "REPLACE_WITH_KEY_ID",
"publicKeyPem": "REPLACE_WITH_ED25519_PUBLIC_KEY_PEM",
"notBeforeMs": 0,
"notAfterMs": 1
}
]
}
@@ -0,0 +1,16 @@
{
"schema": "qinglong/plugin-package-registry-credentials@v1",
"credentials": [
{
"registry": "registry.example.com",
"scheme": "bearer",
"token": "REPLACE_WITH_PRIVATE_REGISTRY_TOKEN"
},
{
"registry": "registry-backup.example.com:5443",
"scheme": "basic",
"username": "REPLACE_WITH_PRIVATE_REGISTRY_USERNAME",
"password": "REPLACE_WITH_PRIVATE_REGISTRY_PASSWORD"
}
]
}
@@ -0,0 +1,13 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../cloudnative-pg
patches:
- target:
group: batch
version: v1
kind: Job
labelSelector: app.kubernetes.io/name=ql3-plugin-package-recovery
path: recover-job-patch.yaml
@@ -0,0 +1,25 @@
apiVersion: batch/v1
kind: Job
metadata:
name: ql3-plugin-package-recovery
namespace: qinglong3-system
spec:
template:
spec:
containers:
- name: recovery
env:
- name: QL3_PLUGIN_PACKAGE_REGISTRY_CREDENTIAL_FILE
value: /var/run/secrets/qinglong3/plugin-package-registry/credentials.json
volumeMounts:
- name: plugin-package-registry-credentials
mountPath: /var/run/secrets/qinglong3/plugin-package-registry
readOnly: true
volumes:
- name: plugin-package-registry-credentials
secret:
secretName: ql3-plugin-package-registry-credentials
defaultMode: 288
items:
- key: credentials.json
path: credentials.json
@@ -0,0 +1,106 @@
# PostgreSQL CA overlap rotation
This runbook defines the operator-neutral QingLong 3.0 trust transition. It
does not elect or promote PostgreSQL and does not replace the database
operator's fencing procedure.
## Preconditions
- The runtime, migration and Worker-ingress URLs target one stable read-write
FQDN. Their `verify-full` servername is an explicit DNS name covered by the
endpoint certificate SAN; IP literals and implicit servername inference are
rejected.
- The endpoint routes new connections only to a writable primary. Every new
QingLong activation independently requires `pg_is_in_recovery() = false` and
`transaction_read_only = off`.
- The database operator has a reviewed primary fencing procedure and, when
zero acknowledged-write loss is claimed, a synchronous `remote_apply` or
equivalent RPO-0 policy.
- Three non-secret files are prepared: `old-ca.pem`, `overlap-ca.pem`, and
`new-ca.pem`. The overlap file must be the exact union of old and new trust
anchors. It may retain a shared root or intermediate, but the transition
must introduce and retire at least one anchor.
Build the package and validate the transition before touching a Secret:
```bash
pnpm audit:postgres-ca-overlap:ql3 -- \
--old=/absolute/path/old-ca.pem \
--overlap=/absolute/path/overlap-ca.pem \
--new=/absolute/path/new-ca.pem
```
Record the returned v1 contract, counts and SHA-256 set digests with the
change. The auditor reuses the production bounded CA loader: every file must
be an absolute, non-group/world-writable regular file containing 116 unique
CA certificates and no trailing data.
## Phase A: expand trust
1. Update each authority's Secret independently to the exact overlap bundle:
runtime, migration and Worker ingress must not be merged into one Secret.
2. Roll the runtime Deployment. Do not rely on kubelet's projected-Secret
symlink update: QingLong reads the bundle only during a new activation.
3. Wait for every old runtime Pod to terminate and every replacement to pass
TLS, writable-primary readiness, schema/role readiness, startup recovery
and lifecycle activation.
4. If Worker ingress is enabled, roll and verify it through its independent
database Pool. Run the migration Job only when a reviewed migration is
actually required; CA rotation alone does not authorize DDL.
Do not rotate the server certificate while any QingLong Pod still uses the
old-only bundle.
## Phase B: rotate the endpoint certificate
Use the PostgreSQL operator or reviewed proxy procedure to install a
certificate chaining to the new trust anchor. Preserve the stable FQDN and
prove:
- the certificate SAN covers the configured servername;
- the endpoint still routes only to the externally fenced writable primary;
- existing QingLong Pods remain ready with the overlap bundle;
- a fresh QingLong activation establishes a new TLS connection and passes the
complete readiness/recovery gate.
Force a controlled QingLong rollout after the server certificate changes.
This removes ambiguity from long-lived connections that were established
before the rotation.
## Phase C: contract trust
1. Confirm every runtime and Worker-ingress Pod was activated after the
endpoint certificate rotation.
2. Replace each projected overlap bundle with the exact new-only bundle.
3. Roll each workload again and wait for all readiness gates.
4. Retire the old CA only after no old-only Pod, migration Job or database
endpoint certificate remains.
Keep the overlap bundle and old server key material under the database
operator's approved rollback retention policy; do not put private keys in
QingLong Secrets or evidence.
## Rollback
- Before Phase B, restore old-only bundles and roll workloads.
- During Phase B, keep the overlap bundle. Either repair the new certificate
or restore the old endpoint certificate through the operator; do not
contract trust.
- During or after Phase C, restore the overlap bundle first and roll QingLong
before reverting the endpoint certificate. Removing the new trust anchor
first can strand every new activation.
- A failed activation stays not-ready. Do not weaken `verify-full`, point
QingLong at a standby, or revive the old activation in place.
## Required evidence and remaining boundary
Capture the CA-overlap audit JSON, Secret resource versions, Deployment
revisions, replacement Pod identities, configured FQDN/servername, endpoint
certificate serial/fingerprint, readiness timestamps and operator fencing
record. Never record database passwords, private keys or bearer material.
This runbook and local auditor prove the bundle topology and rollout ordering.
They are not evidence that a specific operator/proxy performed TLS routing,
certificate rotation, node/storage STONITH or Kubernetes control-plane
failover. A production release must execute this runbook against the selected
operator and retain its independent evidence.
@@ -0,0 +1,98 @@
# Isolated Prompt output lost-key recovery verifier
This caller-driven Job verifies one externally recovered Prompt output key in a
separate `qinglong3-recovery` namespace. It is not part of the default Cluster
Kustomization and does not provision, wrap, unwrap, export or reactivate key
material.
The deployment authority must prepare a private read-only PVC named
`ql3-prompt-output-external-recovery-workspace`. Copy `command.example.json` to
`command.json` and provide exactly the referenced files:
- one provider-neutral atomic custody bundle containing the signed content-free
receipt and its digest-bound wrapped blob;
- the 32-byte material produced by the external KMS/HSM unwrap ceremony;
- the exact durable `keyId/materialProof/catalogDigest` fact from an isolated
PostgreSQL/CNPG restore;
- the immutable encrypted Artifact from that same restore generation;
- the pinned custody signing public key;
- an unexpired recovery authorization signed by two different strong Users,
plus both pinned approver public keys.
The verifier never accepts the receipt and wrapped blob as separate files, so a
partial copy or cross-generation substitution cannot create a mixed recovery
input. Every workspace file must be a non-symlink, single-link, read-only
regular file.
Private files must not be readable by `other`; the recovered material must be
exactly 32 bytes. Provisioning the PVC, copying backup evidence and invoking the
KMS/HSM are deployment-authority responsibilities outside this Job.
The repository includes a concrete Vault Transit adapter without making Vault
part of the QingLong runtime. Copy `vault-transit-wrap-command.example.json`
or `vault-transit-unwrap-command.example.json` into a private authority,
replace every placeholder, make commands/keys/token/material read-only, and run:
```sh
pnpm custody:vault-transit:ql3 -- wrap --command-file /owner-private/wrap.json
pnpm custody:vault-transit:ql3 -- unwrap --command-file /owner-private/unwrap.json
```
Production mode requires HTTPS plus an explicit CA file and reads the Vault
token only from a private file. Wrap creates one no-replace `0400`
provider-neutral bundle; replay verifies that bundle without another Vault
call. Unwrap verifies the signing authority, bundle digest,
provider/key-version authority and exact material proof before creating one
no-replace `0400` recovered material file. Copy that bundle unchanged into the
verifier workspace as `custody-bundle.json`. The Vault token, endpoint and key
name are never embedded in the bundle. Do not run either Vault command in the
tokenless, deny-all verifier Job.
The opt-in live gate uses the exact reviewed Vault image digest and removes its
random container and private workspace on both success and failure:
```sh
QL3_RUN_VAULT_TRANSIT_LIVE=true pnpm test:vault-transit-custody-live:ql3
```
That gate proves the actual Vault 1.21.4 Transit API over TLS 1.3 with an
explicit private CA. It initializes a persistent file barrier with three seal
shares and a threshold of two, wraps the key, replaces the entire non-root
read-only-rootfs container, observes the persisted server sealed, re-unseals
it, verifies the Transit key survived and then unwraps the bundle. A different
CA is rejected before API access. This remains a single-host file-storage
fixture, not HA integrated storage, HSM auto-unseal, enterprise PKI/external
IdP or CNPG restore evidence.
The separate opt-in PostgreSQL composition gate proves that the exact durable
key fact and encrypted Artifact can cross a real logical backup boundary before
the offline verifier receives them:
```sh
QL3_RUN_POSTGRES_BACKUP_RECOVERY_LIVE=true \
pnpm test:postgres-backup-prompt-output-recovery-live:ql3
```
It uses the digest-pinned PostgreSQL 18 image on a random loopback-only port,
runs the complete QL3 core and AI migration streams, and persists a canonical
Package publication plus Prompt admission/start/completion/finalization,
encrypted Artifact and key-rotation record through production repositories. It
then creates a custom-format backup, removes the whole source container and
anonymous volume, restores into a different container/volume, reopens the
production Artifact and rotation repositories, and only then exports the
restored fact and Artifact to the existing two-User verifier. The restored
lineage includes the exact 52 core and 16 AI migration histories and one row for
each required publication and Prompt chain fact. The gate removes both random
containers, their anonymous volumes and its private directory on success or
failure. This is full production-schema logical PostgreSQL backup composition
evidence; it is not CloudNativePG Barman WAL/PITR or an external IdP ceremony.
The Pod has no Role or RoleBinding, disables ServiceAccount token projection,
mounts the PVC read-only and is selected by an ingress/egress deny-all
NetworkPolicy. It receives no PostgreSQL URL, Kubernetes credential, cloud
credential, KMS endpoint or HSM session. Successful stdout is a content-free,
authorization-bound recovery proof. Failure output contains only stable error
metadata. The CLI wipes its owned recovered and wrapped buffers before exit.
This verifies recoverability only. It does not authorize plaintext export,
production Secret mutation, retirement reversal, keyring reconstruction or
bulk re-encryption. Those operations require separate policy and evidence.
@@ -0,0 +1,67 @@
apiVersion: batch/v1
kind: Job
metadata:
name: ql3-prompt-output-external-recovery-verifier
namespace: qinglong3-recovery
labels:
app.kubernetes.io/name: ql3-prompt-output-external-recovery-verifier
app.kubernetes.io/component: ai-recovery-verifier
app.kubernetes.io/part-of: qinglong3
qinglong.io/execution-model: caller-driven
spec:
backoffLimit: 0
activeDeadlineSeconds: 120
ttlSecondsAfterFinished: 600
template:
metadata:
labels:
app.kubernetes.io/name: ql3-prompt-output-external-recovery-verifier
app.kubernetes.io/component: ai-recovery-verifier
app.kubernetes.io/part-of: qinglong3
qinglong.io/execution-model: caller-driven
spec:
serviceAccountName: ql3-prompt-output-external-recovery-verifier
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: verifier
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/prompt-output/external-recovery/promptOutputExternalRecoveryCli.js
args:
- run
- --command-file
- /var/run/qinglong3/prompt-output-external-recovery/command.json
terminationMessagePolicy: File
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 200m
memory: 128Mi
volumeMounts:
- name: recovery-workspace
mountPath: /var/run/qinglong3/prompt-output-external-recovery
readOnly: true
volumes:
- name: recovery-workspace
persistentVolumeClaim:
claimName: ql3-prompt-output-external-recovery-workspace
readOnly: true
@@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- service-account.yaml
- network-policy.yaml
- job.yaml
@@ -0,0 +1,7 @@
apiVersion: v1
kind: Namespace
metadata:
name: qinglong3-recovery
labels:
app.kubernetes.io/part-of: qinglong3
qinglong.io/security-domain: isolated-recovery
@@ -0,0 +1,14 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ql3-prompt-output-external-recovery-verifier-deny-all
namespace: qinglong3-recovery
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: ql3-prompt-output-external-recovery-verifier
policyTypes:
- Ingress
- Egress
ingress: []
egress: []
@@ -0,0 +1,9 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: ql3-prompt-output-external-recovery-verifier
namespace: qinglong3-recovery
labels:
app.kubernetes.io/name: ql3-prompt-output-external-recovery-verifier
app.kubernetes.io/part-of: qinglong3
automountServiceAccountToken: false
@@ -0,0 +1,20 @@
{
"schemaVersion": 1,
"operation": "cluster.prompt-output-key.verify-recovery",
"authorizationFile": "/var/run/qinglong3/prompt-output-external-recovery/authorization.json",
"custodyBundleFile": "/var/run/qinglong3/prompt-output-external-recovery/custody-bundle.json",
"recoveredMaterialFile": "/var/run/qinglong3/prompt-output-external-recovery/recovered-material.bin",
"durableKeyFactFile": "/var/run/qinglong3/prompt-output-external-recovery/durable-key-fact.json",
"artifactFile": "/var/run/qinglong3/prompt-output-external-recovery/artifact.json",
"custodyPublicKeyFile": "/var/run/qinglong3/prompt-output-external-recovery/custody-public-key.pem",
"approverPublicKeyFiles": [
{
"userId": "replace-with-first-reviewer-user-id",
"filePath": "/var/run/qinglong3/prompt-output-external-recovery/first-reviewer-public-key.pem"
},
{
"userId": "replace-with-second-reviewer-user-id",
"filePath": "/var/run/qinglong3/prompt-output-external-recovery/second-reviewer-public-key.pem"
}
]
}
@@ -0,0 +1,15 @@
{
"schemaVersion": 1,
"operation": "external.prompt-output-key.vault-transit.unwrap",
"vault": {
"transport": "https",
"endpoint": "https://vault.example.invalid:8200",
"caFile": "/owner-private/vault/ca.pem",
"tokenFile": "/owner-private/vault/unwrap-token",
"transitMount": "prompt-output-transit",
"keyName": "prompt-output-custody"
},
"bundleFile": "/owner-private/prompt-output/custody-bundle.json",
"custodyPublicKeyFile": "/owner-private/prompt-output/custody-signing-public.pem",
"recoveredMaterialOutputFile": "/owner-private/prompt-output/recovered-material.bin"
}
@@ -0,0 +1,22 @@
{
"schemaVersion": 1,
"operation": "external.prompt-output-key.vault-transit.wrap",
"vault": {
"transport": "https",
"endpoint": "https://vault.example.invalid:8200",
"caFile": "/owner-private/vault/ca.pem",
"tokenFile": "/owner-private/vault/wrap-token",
"transitMount": "prompt-output-transit",
"keyName": "prompt-output-custody"
},
"materialFile": "/owner-private/prompt-output/material.bin",
"bundleOutputFile": "/owner-private/prompt-output/custody-bundle.json",
"custody": {
"custodyId": "replace-custody-id",
"keyId": "replace-key-id",
"sourceGeneration": 1,
"sourceCatalogDigest": "0000000000000000000000000000000000000000000000000000000000000000",
"receiptSigningPrivateKeyFile": "/owner-private/prompt-output/custody-signing-private.pem",
"receiptSigningPublicKeyFile": "/owner-private/prompt-output/custody-signing-public.pem"
}
}
@@ -0,0 +1,98 @@
apiVersion: batch/v1
kind: Job
metadata:
name: ql3-prompt-output-gc
namespace: qinglong3-system
labels:
app.kubernetes.io/name: ql3-prompt-output-gc
app.kubernetes.io/component: ai-maintenance
app.kubernetes.io/part-of: qinglong3
qinglong.io/execution-model: caller-driven
spec:
backoffLimit: 0
activeDeadlineSeconds: 300
ttlSecondsAfterFinished: 600
template:
metadata:
labels:
app.kubernetes.io/name: ql3-prompt-output-gc
app.kubernetes.io/component: ai-maintenance
app.kubernetes.io/part-of: qinglong3
qinglong.io/execution-model: caller-driven
spec:
serviceAccountName: ql3-prompt-output-gc
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: collector
image: qinglong3-cluster-admin:3.0.0-alpha.0
imagePullPolicy: IfNotPresent
command:
- node
- /opt/qinglong/node_modules/@qinglong/cluster-admin/dist/prompt-output/retention/promptOutputGcCli.js
args:
- run
- --policy-file
- /var/run/qinglong3/prompt-output-retention/retention-policies.json
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
env:
- name: QL3_PROMPT_OUTPUT_GC_LIMIT
value: '32'
- name: QL3_POSTGRES_TLS_MODE
value: verify-full
- name: QL3_POSTGRES_TLS_CA_FILE
value: /var/run/secrets/qinglong3/postgres-ai-maintenance/ca.crt
- name: QL3_POSTGRES_APPLICATION_NAME
value: qinglong3-prompt-output-gc
- name: QL3_POSTGRES_AI_MAINTENANCE_URL
valueFrom:
secretKeyRef:
name: ql3-cluster-ai-maintenance
key: postgres-ai-maintenance-url
- name: QL3_POSTGRES_TLS_SERVERNAME
valueFrom:
secretKeyRef:
name: ql3-cluster-ai-maintenance
key: postgres-tls-servername
resources:
requests:
cpu: 25m
memory: 48Mi
limits:
cpu: 250m
memory: 128Mi
volumeMounts:
- name: policies
mountPath: /var/run/qinglong3/prompt-output-retention
readOnly: true
- name: postgres-ca
mountPath: /var/run/secrets/qinglong3/postgres-ai-maintenance
readOnly: true
volumes:
- name: policies
configMap:
name: ql3-prompt-output-retention-policies
defaultMode: 292
items:
- key: retention-policies.json
path: retention-policies.json
- name: postgres-ca
secret:
secretName: ql3-cluster-ai-maintenance
defaultMode: 292
items:
- key: postgres-ca.crt
path: ca.crt
@@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- service-account.yaml
- job.yaml
- network-policy.yaml

Some files were not shown because too many files have changed in this diff Show More