diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/api-server-egress-patch.example.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/api-server-egress-patch.example.yaml new file mode 100644 index 00000000..b52bd6f7 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/api-server-egress-patch.example.yaml @@ -0,0 +1,12 @@ +# Example only. Copy into a private overlay and replace the placeholder with +# the exact Kubernetes API endpoint CIDR. The base policy intentionally has no +# broad TCP/443 escape hatch, so the controller fails closed without this rule. +- op: add + path: /spec/egress/- + value: + to: + - ipBlock: + cidr: REPLACE_WITH_API_SERVER_CIDR + ports: + - protocol: TCP + port: 443 diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/cron-job.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/cron-job.yaml index a4e69c42..926786c6 100644 --- a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/cron-job.yaml +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/cron-job.yaml @@ -26,7 +26,7 @@ spec: app.kubernetes.io/part-of: qinglong3 spec: serviceAccountName: ql3-plugin-package-executor - automountServiceAccountToken: false + automountServiceAccountToken: true restartPolicy: Never securityContext: runAsNonRoot: true @@ -65,6 +65,82 @@ spec: value: '16' - name: QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_MAX_PAGES value: '16' + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_ENABLED + value: 'true' + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_LIMIT + value: '8' + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_NAMESPACE + value: qinglong3-system + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_SERVICE_ACCOUNT + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: serviceAccountName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_SOURCE_SECRET + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: sourceSecretName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_IMAGE + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: image + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_CA_SECRET + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresCaSecretName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_CA_KEY + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresCaKey + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_SERVERNAME + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresServerName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_SECRET + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresUrlSecretName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_KEY + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresUrlSecretKey + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_AUTH_SECRET + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresAuthSecretName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_HOST + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresHost + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_PORT + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresPort + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_DATABASE + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresDatabase + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_USERNAME_KEY + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresUsernameKey + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_PASSWORD_KEY + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresPasswordKey - name: QL3_POSTGRES_TLS_MODE value: verify-full - name: QL3_POSTGRES_TLS_CA_FILE diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/kustomization.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/kustomization.yaml index e1d3706c..1add182a 100644 --- a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/kustomization.yaml +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/kustomization.yaml @@ -3,5 +3,11 @@ kind: Kustomization resources: - service-account.yaml + - secret-action-service-account.yaml + - secret-action-admission-config.yaml + - role.yaml + - role-binding.yaml + - validating-admission-policy.yaml + - validating-admission-policy-binding.yaml - cron-job.yaml - network-policy.yaml diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/role-binding.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/role-binding.yaml new file mode 100644 index 00000000..5d19561a --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/role-binding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ql3-plugin-package-executor + namespace: qinglong3-system + labels: + app.kubernetes.io/name: ql3-plugin-package-executor + app.kubernetes.io/component: plugin-package-executor + app.kubernetes.io/part-of: qinglong3 +subjects: + - kind: ServiceAccount + name: ql3-plugin-package-executor + namespace: qinglong3-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ql3-plugin-package-executor diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/role.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/role.yaml new file mode 100644 index 00000000..f8a094ee --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/role.yaml @@ -0,0 +1,25 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ql3-plugin-package-executor + namespace: qinglong3-system + labels: + app.kubernetes.io/name: ql3-plugin-package-executor + app.kubernetes.io/component: plugin-package-executor + app.kubernetes.io/part-of: qinglong3 +rules: + - apiGroups: + - batch + resources: + - jobs + verbs: + - create + - get + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ql3-plugin-package-secret-action-admission + verbs: + - get diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/secret-action-admission-config.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/secret-action-admission-config.yaml new file mode 100644 index 00000000..d899b479 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/secret-action-admission-config.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ql3-plugin-package-secret-action-admission + namespace: qinglong3-system + labels: + app.kubernetes.io/name: ql3-plugin-package-secret-action + app.kubernetes.io/component: plugin-package-executor + app.kubernetes.io/part-of: qinglong3 +data: + # Release automation must replace this unavailable digest together with the + # controller image before the opt-in operation can execute. + image: registry.example.com/qinglong/qinglong3-cluster-admin@sha256:0000000000000000000000000000000000000000000000000000000000000000 + serviceAccountName: ql3-plugin-package-secret-action + sourceSecretName: ql3-cluster-plugin-package-values + postgresConnectionMode: url + postgresCaSecretName: ql3-cluster-plugin-package-executor + postgresCaKey: postgres-ca.crt + postgresServerName: REPLACE_WITH_POSTGRES_DNS_NAME + postgresUrlSecretName: ql3-cluster-plugin-package-executor + postgresUrlSecretKey: postgres-package-executor-url + postgresAuthSecretName: '' + postgresHost: '' + postgresPort: '' + postgresDatabase: '' + postgresUsernameKey: '' + postgresPasswordKey: '' diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/secret-action-service-account.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/secret-action-service-account.yaml new file mode 100644 index 00000000..f5c1c4be --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/secret-action-service-account.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ql3-plugin-package-secret-action + namespace: qinglong3-system + labels: + app.kubernetes.io/name: ql3-plugin-package-secret-action + app.kubernetes.io/component: plugin-package-executor + app.kubernetes.io/part-of: qinglong3 +automountServiceAccountToken: false diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/validating-admission-policy-binding.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/validating-admission-policy-binding.yaml new file mode 100644 index 00000000..76a650ce --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/validating-admission-policy-binding.yaml @@ -0,0 +1,20 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: ql3-plugin-package-secret-action + labels: + app.kubernetes.io/name: ql3-plugin-package-secret-action + app.kubernetes.io/component: plugin-package-executor + app.kubernetes.io/part-of: qinglong3 +spec: + policyName: ql3-plugin-package-secret-action + paramRef: + name: ql3-plugin-package-secret-action-admission + namespace: qinglong3-system + parameterNotFoundAction: Deny + matchResources: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: qinglong3-system + validationActions: + - Deny diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/validating-admission-policy.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/validating-admission-policy.yaml new file mode 100644 index 00000000..13949e16 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/validating-admission-policy.yaml @@ -0,0 +1,163 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: ql3-plugin-package-secret-action + labels: + app.kubernetes.io/name: ql3-plugin-package-secret-action + app.kubernetes.io/component: plugin-package-executor + app.kubernetes.io/part-of: qinglong3 +spec: + failurePolicy: Fail + paramKind: + apiVersion: v1 + kind: ConfigMap + matchConstraints: + resourceRules: + - apiGroups: + - batch + apiVersions: + - v1 + operations: + - CREATE + resources: + - jobs + scope: Namespaced + matchConditions: + - name: exact-controller-identity + expression: request.userInfo.username == 'system:serviceaccount:qinglong3-system:ql3-plugin-package-executor' + variables: + - name: pod + expression: object.spec.template.spec + - name: executor + expression: variables.pod.containers[0] + - name: values + expression: variables.pod.volumes.filter(v, v.name == 'plugin-package-values')[0] + - name: ca + expression: variables.pod.volumes.filter(v, v.name == 'postgres-ca')[0] + validations: + - expression: >- + object.metadata.namespace == 'qinglong3-system' && + object.metadata.name.matches('^ql3-package-secret-[0-9a-f]{32}$') && + object.metadata.name == 'ql3-package-secret-' + object.metadata.annotations['qinglong.io/secret-action-job-digest'].substring(0, 32) && + object.metadata.labels['app.kubernetes.io/name'] == 'ql3-plugin-package-secret-action' && + object.metadata.labels['app.kubernetes.io/component'] == 'plugin-package-executor' && + object.metadata.labels['app.kubernetes.io/part-of'] == 'qinglong3' && + object.metadata.annotations['qinglong.io/secret-action-job-schema'] == 'qinglong/plugin-package-kubernetes-secret-action-job@v1' && + object.metadata.annotations['qinglong.io/secret-action-job-digest'].matches('^[0-9a-f]{64}$') && + object.metadata.annotations['qinglong.io/approved-action-digest'].matches('^[0-9a-f]{64}$') && + object.metadata.annotations['qinglong.io/approved-action-type'] in ['plugin_package.secret_binding.bind', 'plugin_package.secret_binding.transition'] + message: Secret action Job metadata is outside the reviewed contract + - expression: >- + object.spec.backoffLimit == 2 && + object.spec.activeDeadlineSeconds == 600 && + object.spec.ttlSecondsAfterFinished == 3600 && + variables.pod.serviceAccountName == params.data.serviceAccountName && + object.spec.template.metadata.labels['app.kubernetes.io/name'] == 'ql3-plugin-package-secret-action' && + object.spec.template.metadata.labels['app.kubernetes.io/component'] == 'plugin-package-executor' && + object.spec.template.metadata.labels['app.kubernetes.io/part-of'] == 'qinglong3' && + object.spec.template.metadata.labels['qinglong.io/secret-action-job'] == object.metadata.name && + variables.pod.automountServiceAccountToken == false && + variables.pod.enableServiceLinks == false && + variables.pod.restartPolicy == 'Never' && + (!has(variables.pod.hostNetwork) || variables.pod.hostNetwork == false) && + (!has(variables.pod.hostPID) || variables.pod.hostPID == false) && + (!has(variables.pod.hostIPC) || variables.pod.hostIPC == false) && + (!has(variables.pod.shareProcessNamespace) || variables.pod.shareProcessNamespace == false) && + (!has(variables.pod.initContainers) || variables.pod.initContainers.size() == 0) && + (!has(variables.pod.ephemeralContainers) || variables.pod.ephemeralContainers.size() == 0) && + variables.pod.containers.size() == 1 && + variables.pod.volumes.size() == 3 + message: Secret action Job Pod authority is outside the reviewed contract + - expression: >- + variables.pod.securityContext.runAsNonRoot == true && + variables.pod.securityContext.runAsUser == 10001 && + variables.pod.securityContext.runAsGroup == 10001 && + variables.pod.securityContext.fsGroup == 10001 && + variables.pod.securityContext.seccompProfile.type == 'RuntimeDefault' && + variables.executor.name == 'executor' && + variables.executor.image == params.data.image && + variables.executor.imagePullPolicy == 'IfNotPresent' && + variables.executor.command == ['node', '/opt/qinglong/node_modules/@qinglong/cluster-admin/dist/plugin-package/executor/pluginPackageExecutorCli.js'] && + variables.executor.securityContext.allowPrivilegeEscalation == false && + variables.executor.securityContext.readOnlyRootFilesystem == true && + variables.executor.securityContext.capabilities.drop == ['ALL'] && + variables.executor.resources.requests.cpu == '25m' && + variables.executor.resources.requests.memory == '48Mi' && + variables.executor.resources.limits.cpu == '250m' && + variables.executor.resources.limits.memory == '192Mi' + message: Secret action Job executor image, command or sandbox is outside the reviewed contract + - expression: >- + ((params.data.postgresConnectionMode == 'url' && variables.executor.env.size() == 11) || + (params.data.postgresConnectionMode == 'fields' && variables.executor.env.size() == 15)) && + variables.executor.env.all(e, e.name in [ + 'QL3_PLUGIN_PACKAGE_EXECUTOR_ENABLED', + 'QL3_PLUGIN_PACKAGE_EXECUTOR_OWNER', + 'QL3_PLUGIN_PACKAGE_EXECUTOR_DISPATCH_ID', + 'QL3_PLUGIN_PACKAGE_EXECUTOR_SECRET_ROOT', + 'QL3_PLUGIN_PACKAGE_EXECUTOR_LEASE_DURATION_MS', + 'QL3_POSTGRES_TLS_MODE', + 'QL3_POSTGRES_TLS_CA_FILE', + 'QL3_POSTGRES_TLS_SERVERNAME', + 'QL3_POSTGRES_APPLICATION_NAME', + 'QL3_POSTGRES_MAX_CONNECTIONS', + 'QL3_POSTGRES_PACKAGE_EXECUTOR_URL', + 'QL3_POSTGRES_PACKAGE_EXECUTOR_HOST', + 'QL3_POSTGRES_PACKAGE_EXECUTOR_PORT', + 'QL3_POSTGRES_PACKAGE_EXECUTOR_DATABASE', + 'QL3_POSTGRES_PACKAGE_EXECUTOR_USER', + 'QL3_POSTGRES_PACKAGE_EXECUTOR_PASSWORD' + ]) && + variables.executor.env.exists_one(e, e.name == 'QL3_PLUGIN_PACKAGE_EXECUTOR_ENABLED' && e.value == 'true') && + variables.executor.env.exists_one(e, e.name == 'QL3_PLUGIN_PACKAGE_EXECUTOR_OWNER' && e.value.matches('^package_secret_[0-9a-f]{24}$')) && + variables.executor.env.exists_one(e, e.name == 'QL3_PLUGIN_PACKAGE_EXECUTOR_DISPATCH_ID' && e.value.matches('^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$')) && + variables.executor.env.exists_one(e, e.name == 'QL3_PLUGIN_PACKAGE_EXECUTOR_SECRET_ROOT' && e.value == '/var/run/secrets/qinglong3/plugin-package-values') && + variables.executor.env.exists_one(e, e.name == 'QL3_PLUGIN_PACKAGE_EXECUTOR_LEASE_DURATION_MS' && e.value == '600000') && + variables.executor.env.exists_one(e, e.name == 'QL3_POSTGRES_TLS_MODE' && e.value == 'verify-full') && + variables.executor.env.exists_one(e, e.name == 'QL3_POSTGRES_TLS_CA_FILE' && e.value == '/var/run/secrets/qinglong3/postgres/ca.crt') && + variables.executor.env.exists_one(e, e.name == 'QL3_POSTGRES_TLS_SERVERNAME' && e.value == params.data.postgresServerName) && + variables.executor.env.exists_one(e, e.name == 'QL3_POSTGRES_APPLICATION_NAME' && e.value == 'qinglong3-package-secret-action') && + variables.executor.env.exists_one(e, e.name == 'QL3_POSTGRES_MAX_CONNECTIONS' && e.value == '1') + message: Secret action Job environment is outside the reviewed contract + - expression: >- + (params.data.postgresConnectionMode == 'url' && + variables.executor.env.exists_one(e, + e.name == 'QL3_POSTGRES_PACKAGE_EXECUTOR_URL' && + e.valueFrom.secretKeyRef.name == params.data.postgresUrlSecretName && + e.valueFrom.secretKeyRef.key == params.data.postgresUrlSecretKey) && + !variables.executor.env.exists(e, e.name in ['QL3_POSTGRES_PACKAGE_EXECUTOR_HOST', 'QL3_POSTGRES_PACKAGE_EXECUTOR_PORT', 'QL3_POSTGRES_PACKAGE_EXECUTOR_DATABASE', 'QL3_POSTGRES_PACKAGE_EXECUTOR_USER', 'QL3_POSTGRES_PACKAGE_EXECUTOR_PASSWORD'])) || + (params.data.postgresConnectionMode == 'fields' && + !variables.executor.env.exists(e, e.name == 'QL3_POSTGRES_PACKAGE_EXECUTOR_URL') && + variables.executor.env.exists_one(e, e.name == 'QL3_POSTGRES_PACKAGE_EXECUTOR_HOST' && e.value == params.data.postgresHost) && + variables.executor.env.exists_one(e, e.name == 'QL3_POSTGRES_PACKAGE_EXECUTOR_PORT' && e.value == params.data.postgresPort) && + variables.executor.env.exists_one(e, e.name == 'QL3_POSTGRES_PACKAGE_EXECUTOR_DATABASE' && e.value == params.data.postgresDatabase) && + variables.executor.env.exists_one(e, e.name == 'QL3_POSTGRES_PACKAGE_EXECUTOR_USER' && e.valueFrom.secretKeyRef.name == params.data.postgresAuthSecretName && e.valueFrom.secretKeyRef.key == params.data.postgresUsernameKey) && + variables.executor.env.exists_one(e, e.name == 'QL3_POSTGRES_PACKAGE_EXECUTOR_PASSWORD' && e.valueFrom.secretKeyRef.name == params.data.postgresAuthSecretName && e.valueFrom.secretKeyRef.key == params.data.postgresPasswordKey)) + message: Secret action Job database authority is outside the reviewed contract + - expression: >- + variables.executor.volumeMounts.size() == 3 && + variables.executor.volumeMounts[0].name == 'tmp' && + variables.executor.volumeMounts[0].mountPath == '/tmp' && + (!has(variables.executor.volumeMounts[0].readOnly) || variables.executor.volumeMounts[0].readOnly == false) && + variables.executor.volumeMounts[1].name == 'postgres-ca' && + variables.executor.volumeMounts[1].mountPath == '/var/run/secrets/qinglong3/postgres' && + variables.executor.volumeMounts[1].readOnly == true && + variables.executor.volumeMounts[2].name == 'plugin-package-values' && + variables.executor.volumeMounts[2].mountPath == '/var/run/secrets/qinglong3/plugin-package-values' && + variables.executor.volumeMounts[2].readOnly == true && + variables.pod.volumes.exists_one(v, v.name == 'tmp' && v.emptyDir.medium == 'Memory' && v.emptyDir.sizeLimit == '8Mi') && + variables.ca.secret.secretName == params.data.postgresCaSecretName && + variables.ca.secret.optional == false && + variables.ca.secret.defaultMode == 292 && + variables.ca.secret.items.size() == 1 && + variables.ca.secret.items[0].key == params.data.postgresCaKey && + variables.ca.secret.items[0].path == 'ca.crt' && + ((has(variables.values.secret) && + variables.values.secret.secretName == params.data.sourceSecretName && + variables.values.secret.optional == false && + variables.values.secret.defaultMode == 288 && + variables.values.secret.items.size() >= 1 && + variables.values.secret.items.size() <= 64 && + variables.values.secret.items.all(i, i.key.matches('^[0-9a-f]{64}$') && i.path == i.key) && + variables.values.secret.items.all(i, variables.values.secret.items.filter(j, j.key == i.key).size() == 1)) || + (has(variables.values.emptyDir) && variables.values.emptyDir.sizeLimit == '1Ki')) + message: Secret action Job volumes are outside the exact-key projection contract diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/cron-job-patch.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/cron-job-patch.yaml index a4d6bf52..9d4fbce2 100644 --- a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/cron-job-patch.yaml +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/cron-job-patch.yaml @@ -17,6 +17,82 @@ value: '16' - name: QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_MAX_PAGES value: '16' + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_ENABLED + value: 'true' + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_LIMIT + value: '8' + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_NAMESPACE + value: qinglong3-system + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_SERVICE_ACCOUNT + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: serviceAccountName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_SOURCE_SECRET + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: sourceSecretName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_IMAGE + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: image + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_CA_SECRET + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresCaSecretName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_CA_KEY + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresCaKey + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_SERVERNAME + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresServerName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_SECRET + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresUrlSecretName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_KEY + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresUrlSecretKey + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_AUTH_SECRET + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresAuthSecretName + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_HOST + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresHost + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_PORT + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresPort + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_DATABASE + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresDatabase + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_USERNAME_KEY + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresUsernameKey + - name: QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_PASSWORD_KEY + valueFrom: + configMapKeyRef: + name: ql3-plugin-package-secret-action-admission + key: postgresPasswordKey - name: QL3_POSTGRES_TLS_MODE value: verify-full - name: QL3_POSTGRES_TLS_CA_FILE diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/kustomization.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/kustomization.yaml index 3e0af84c..8ef9ecdb 100644 --- a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/kustomization.yaml +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/kustomization.yaml @@ -17,6 +17,11 @@ patches: kind: NetworkPolicy name: ql3-plugin-package-executor path: network-policy-patch.yaml + - target: + version: v1 + kind: ConfigMap + name: ql3-plugin-package-secret-action-admission + path: secret-action-admission-config-patch.yaml images: - name: qinglong3-cluster-admin diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/secret-action-admission-config-patch.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/secret-action-admission-config-patch.yaml new file mode 100644 index 00000000..7df2028f --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/cloudnative-pg/secret-action-admission-config-patch.yaml @@ -0,0 +1,18 @@ +- op: replace + path: /data + value: + image: registry.example.com/qinglong/qinglong3-cluster-admin@sha256:0000000000000000000000000000000000000000000000000000000000000000 + serviceAccountName: ql3-plugin-package-secret-action + sourceSecretName: ql3-cluster-plugin-package-values + postgresConnectionMode: fields + postgresCaSecretName: ql3-postgres-ca + postgresCaKey: ca.crt + postgresServerName: ql3-postgres-rw.qinglong3-system.svc + postgresUrlSecretName: '' + postgresUrlSecretKey: '' + postgresAuthSecretName: ql3-postgres-package-executor-auth + postgresHost: ql3-postgres-rw.qinglong3-system.svc + postgresPort: '5432' + postgresDatabase: qinglong + postgresUsernameKey: username + postgresPasswordKey: password diff --git a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/config.example.yaml b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/config.example.yaml index 90cef1df..11895eac 100644 --- a/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/config.example.yaml +++ b/deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/config.example.yaml @@ -21,3 +21,7 @@ metadata: namespace: qinglong3-system type: Opaque data: {} + +# The controller also needs one private-overlay NetworkPolicy patch containing +# only the exact Kubernetes API endpoint CIDR and TCP port. Start from +# api-server-egress-patch.example.yaml; do not permit arbitrary TCP/443 egress. diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 3538843b..dbbbc84c 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -30,6 +30,8 @@ - D-306A/ADR-0394（已接受）：Package Task source 以 `package-secret` placeholder 引用逻辑 requirement，materialization 只用当前 generation 的 D-305 binding 编译为已有、固定 version 的 Task `SecretRef`；Package source 直接携带 SecretRef、缺失 binding、未批准 `secret.use`、跨 binding 引用和 optional/required 漂移全部失败关闭。binding 快照不含明文并进入 materialized revision digest，Local/Cluster 启动发布复用既有 repository/pool，Task dispatch、Local 短时解密和 Cluster offer/lease-fenced delivery 不另造协议。SQLite/PostgreSQL INSERT trigger 同时防止直接写库绕过；Local 只读 readiness 继续不加载 DDL。Local contract v47、Cluster v59；不新增 package、表、索引、连接、daemon、watcher、timer、cache 或 workload。完整 18-package clean build/test 退出 0；backend 1,188 pass/2 条件 skip/0 fail；五项 package/dependency/edge/service-manager/local-image 审计零 finding，workspace 仍无 single-source/shallow-source package，两个有序 migration ledger 精确为 PostgreSQL 61、SQLite 95 个 source；PostgreSQL 18.4 arm64 HA 125 项 gate 全绿、timeline `1→2`，报告 SHA-256 为 `f9107e8e54892a788779758f0573ac8d6a80f6d086516a1f5f5bbacb59bbb4be`。D-306A 不冒充产品闭环：Local bind/rebind 命令、Cluster Approved Action/API 与新 generation rotation/revocation 编排属于 D-306B。 - D-306B1/ADR-0395（已接受）：当前 active、尚未绑定 Package generation 的首次 Secret binding 已形成 Local 与 Cluster 产品闭环，且不允许原地 rebind。共享 content-free plan 由服务端从 installation/proposal/lock/Manifest/generation 重建；Local 使用短生命周期 `ql3-package`、Owner human confirmation 与单 SQLite transaction，Cluster 使用既有 package-management HTTPS/CLI、package-manager separation-of-duty Approval 和短生命周期 package-executor。三节点 K3s `v1.34.3+k3s1` arm64 现场门已在真实 PostgreSQL `18.4` 上完成两个 management Pod 跨节点部署、正式 client `plan→跨副本 replay→propose→双人 decide→inspect`、真实 executor Job 与只读 Kubernetes Secret projection。management/executor 均无 Secret API 读取权和 ServiceAccount token；management 不挂载 Package value，executor 只验证投影元数据；最终恰好一条 immutable binding，Approval consumed、execution succeeded，数据库敏感值扫描为 0。16/16 gate 的 owner-private、低敏报告通过独立 exact-shape 审计，SHA-256 为 `aaabb5ebea77c50bce671f91dd3051671fd20875c11a8f787fe8933f29dbfa4d`。完整 18-package clean build/test、backend 与七项边界审计，以及 PostgreSQL 18.4 physical HA 125 gate/timeline `1→2` 证据继续有效；没有新增 workspace package、migration、表、索引、依赖或常驻 workload。B2 rebind/rotation/revocation 必须通过新 Package generation 独立推进。 - D-306B2/ADR-0396（进行中）：Secret rebind/rotation/revocation 不更新历史 binding，而是作为下一 Package generation 的 activation 前置事实。共享 transition plan v1 同时绑定上一 active target、可选的上一 binding、durable install history 的最后尝试 generation、新 target、可选下一 binding plan、逐 requirement 与 SecretRef 差异及独立 digest；上一 active Manifest 没有 Secret requirement 时 binding 可空，但 target/lock/generation lineage 不可省略。失败 install 也永久消耗 generation，重试必须使用 `lastAttemptGeneration + 1`，active lineage 继续由 `previousActiveLockDigest` 指回旧代。SQLite capability v49（0097/0098）已完成 immutable transition receipt ledger、Local Owner staged `plan→execute`、单事务 binding/audit/receipt 和 activation prerequisite。PostgreSQL capability v62（pg-0063）具有 receipt ledger，Cluster management 与 package-executor 也已完成 separation-of-duty 产品编排：executor 在一个 SERIALIZABLE transaction 中复验 current staged head、上一 active lineage、durable 最大 generation 与可选上一 binding，并原子提交可选目标 binding 和 immutable receipt；数据库 trigger 和最小角色 readiness 防止绕过，recovery/直接 activation 缺 receipt 均失败关闭。Cluster startup recovery 现把 binding/transition receipt 编译为 content-blind Kubernetes active pointer v3：只保存 source Secret 名、不可逆 SHA-256 key/path、`0440`、binding/receipt/projection digest 和逻辑 assignment，不保存 SecretRef 或明文；同一次 ConfigMap `resourceVersion` CAS 同时切换 Package generation 与投影声明。rotate 只投影下一代 exact key；revoke 生成显式空 projection，Pod volume renderer 对空项返回不挂载，避免 Kubernetes 空 `items` 被解释为投影全部 key。无 Secret generation 继续发布兼容 v2；publisher 不获得 Secret `get/list`，不新增 watcher/controller。响应丢失通过 durable pointer inspect 精确收敛，projection source 不可用或 digest 漂移时旧 active pointer 保持不变。真实三节点 K3s `v1.34.3+k3s1` 已完成两个受限 actor 同一 resourceVersion 的 v3 rotation 竞争：1 成功/1 冲突、最终恰好 1 pointer、1 个 exact projection item、Secret API read denied，projection digest `22add8965accf3f736935167963b9dbdeab8fba05f739f24d39dec941aae9680`、transition receipt digest `355b89ecd54422af33fa573780c8b70ed41da4df226ec301bdd5b6c71de609e1`；随后生产 renderer 驱动 2 副本 workload 分布到 2 个节点，源 Secret 同时保留目标 key 与 decoy key，而 Pod 只看到 exact path、文件模式 `0440`。第三个同权限 actor 以 generation 3 transition receipt `252e8cd1d0f8a2b63f8c0af6247861c08e5b7ae8886c0d65cd90537be3e9f9ac` 发布显式空 projection；Deployment 滚动产生全新 Pod UID，两个新 Pod 均无 Secret volume/mount 和投影根目录，源 Secret 保留，actor 的 Secret `get` 仍为 403；临时容器/网络已清理。实现没有新增 workspace package，Secret projection/renderer 内聚在既有 `cluster-admin/plugin-package/secret-binding`，18-package boundary 仍为 `singleSourcePackages=[]`、`shallowSourcePackages=[]`，cluster dependency 与 edge import 审计无 finding。阶段提交 `9d7431c2` 后，完整 18-package 串行 build/test 退出 0，backend 1,192 pass/2 skip/0 fail；PostgreSQL 18.4 physical HA 125 gate、timeline `1→2` 通过，报告 SHA-256 为 `2f0d1107cc6d447868bfaeb3650284acd803924c6cd3cdee978b3eb5882eb26c`；Edge 本机观测的模块加载 RSS 增量约 6.0 MiB、1 万行输出峰值增量约 3.9 MiB，但尚无固定物理低配门。executor 精确投影重构已开始落地：共享 dispatcher 增加不扫描队列的 `dispatchById`，在租约领取前拒绝未配置 handler 的动作；executor 的 exact mode 跳过全部 Approval consumer，只执行一个 durable dispatch。既有 `cluster-admin/plugin-package/executor` 内新增 immutable Kubernetes Job renderer，以 dispatch 与 approval plan 联合校验生成确定性名称，只接受 digest-pinned 镜像，并只挂载去重后的 exact SHA-256 `items`（`0440`、`optional:false`、无 API token）；零 SecretRef 的 revoke 使用 1 KiB `emptyDir`，绝不以空 `items` 误挂全量 Secret。常规 batch CronJob 已移除 Package values volume、Secret root 和 dispatch-id authority；未挂 Secret root 时 dispatcher 根本不注册 binding/transition handler，因此相关 durable execution 保持 pending，不会被错误领取后 blocked。该切片不增加 package、依赖或常驻进程，并把 action Job 内存 request/limit 固定为 48/192 MiB、数据库池固定 1，适合低配节点。生产 rollout controller 尚不能直接取得不受约束的 `jobs.create`：该权限可通过自定义 PodSpec 间接放大到任意 Secret/镜像，必须先用 admission policy 把 ServiceAccount、digest 镜像、command、source Secret、exact item/path 与数据库 SecretRef 固定，再接入 create/get-only adapter 和恢复状态机。本阶段完整 18-package clean build/test 退出 0；backend 1195 项为 1193 pass、2 条条件 skip、0 fail；package boundary 确认为 18 个 package、`singleSourcePackages=[]`、`shallowSourcePackages=[]`，cluster dependency、edge import 和 cluster deployment 审计均无 finding。Edge arm64 本机观测模块加载 RSS 增量 8,945,664 bytes，1 万行输出峰值增量 5,226,496 bytes；仍仅为观测而非固定物理低配门。PostgreSQL `18.4` arm64 physical HA 125 项 gate、timeline `1→2` 通过，报告 SHA-256 为 `45fab400eb449774d50429103dd766a2755166530ac54ddc1056f777bc16c15f`，临时 Docker 资源已清理。升级失败自动回滚、真实 controller/RBAC/admission 现场门和固定低配物理证据仍待完成。 + - 2026-08-14 收口更新（取代上一句关于 controller 尚未启用的描述）：生产 controller 已在既有 `cluster-admin/plugin-package/executor` 内接入 create/get-only Kubernetes Job adapter。它先按确定性名称 GET；仅当 durable execution 仍可创建且 approval 未过期时使用 Strict CREATE；CREATE 的 409 或响应丢失均回到 exact GET 收敛，`executing` 但 Job 缺失、过期 plan、终态 Job 与 renderer contract 漂移全部返回 `recoveryRequired`，不会盲目重建。Controller ServiceAccount 的 RBAC 仅允许 Job `create|get`，action ServiceAccount 无 API token；`admissionregistration.k8s.io/v1` ValidatingAdmissionPolicy 以固定参数 ConfigMap 和请求者身份锁死 digest 镜像、command、两个 ServiceAccount、source Secret、exact SHA-256 item/path、PostgreSQL SecretRef、安全上下文、资源额度和 volume/mount 形状，参数缺失与策略错误均失败关闭。基础 NetworkPolicy 继续只允许 DNS，集群 overlay 必须显式提供 API Server 的精确 CIDR/TCP 443 出口补丁。 + - 真实 K3s `v1.34.3+k3s1` 已完成 admission 编译与现场门：合规 Job 的 server dry-run 通过，篡改镜像被策略拒绝，删除参数 ConfigMap 后创建被拒绝；controller SA 的 `list|watch|delete jobs`、Pod 创建和 Secret 读取均被拒绝，action SA 的 Job/Pod 创建与 Secret 读取也均被拒绝。实现仍保持 18 个 package，未新增 workspace package、Edge daemon/timer/watcher 或低配设备常驻负担；短生命周期 controller 与按需 action Job 仅属于 Cluster profile。完整 18-package clean build/test 退出 0；backend 1196 项为 1194 pass、2 条条件 skip、0 fail；cluster-admin 339 pass/3 skip、cluster-postgres 328 pass/2 skip，package boundary、cluster dependency、edge import 和 cluster deployment 审计均无 finding。PostgreSQL `18.4` arm64 physical HA 125 项、timeline `1→2` 通过，报告 SHA-256 为 `a3d34e61ea2064e1cde574e533137186e09fdce9048455da64f582906037fa0d`，临时 Docker 资源已清理。ADR 仍为 Proposed：升级失败自动回滚、终态 Job 的 durable 恢复决议和固定物理低配设备证据尚未完成。 - D-302/ADR-0390（已接受） Cluster operator context 增加无网络、无 mutation 的内建 `ql3-cluster-admin context validate` 预检。它先复用 owner-private context reader，再让每个 entry 经过与真实请求相同的 production HTTPS/Kubernetes configuration preparation，验证精确 route、hostname、CA、 diff --git a/docs/adr/ADR-0396-generation-transition-plugin-package-secret-binding.md b/docs/adr/ADR-0396-generation-transition-plugin-package-secret-binding.md index 8a6d8242..a24fd000 100644 --- a/docs/adr/ADR-0396-generation-transition-plugin-package-secret-binding.md +++ b/docs/adr/ADR-0396-generation-transition-plugin-package-secret-binding.md @@ -42,3 +42,8 @@ D-306B1 只允许给当前 active 且尚未绑定的 Package generation 做首 - Cluster management/executor 的 separation-of-duty 产品编排仍未消费 transition plan；真实 Kubernetes rotation/revoke、升级失败回滚及物理低配设备证据也仍待完成。因此 ADR 继续保持 Proposed，下一切片把既有 package-manager Approval 与 package-executor 接入这条 durable authority，再进入 Kubernetes/低配现场门。 - 后续切片已经完成 Cluster transition Approval/执行、Kubernetes active pointer v3 与三节点 rotation/revoke rollout 原语现场门；该历史未完成描述由本条取代。当前 executor 重构进一步增加 exact `dispatchById` start-barrier 路径和由 durable dispatch + approval plan 联合绑定的 action-scoped Job renderer。常规 batch CronJob 不再挂载 `ql3-cluster-plugin-package-values`，也不注册需要 Secret projection 的 handler；action Job 只投影单动作所需的去重 SHA-256 key，固定 `0440`、`optional:false`、digest 镜像、tokenless ServiceAccount、单连接 PostgreSQL 与 48 MiB request。零 Secret transition 使用空目录而不是空 Secret `items`。因此“整个 optional Secret 暴露给批处理 executor”的生产缺口已经失败关闭，但自动 controller 仍未启用：Kubernetes `jobs.create` 是可间接选择镜像/ServiceAccount/Secret 的放大权限，必须由 ValidatingAdmissionPolicy（或同等级外部 admission）把 renderer 契约固定后，才允许 create/get-only RBAC、响应丢失收敛与失败 Job 恢复。升级失败自动回滚、controller/admission/RBAC 真实门及固定物理低配证据继续阻断 ADR Accepted。 - 本阶段完整性门已闭合：18-package clean build/test 退出 0；backend 1195 项为 1193 pass、2 条条件 skip、0 fail；package boundary 仍为 18 个 package 且无 single-source/shallow-source package，cluster dependency、edge import、cluster deployment 均零 finding。Edge arm64 本机观测模块加载 RSS `+8,945,664` bytes、1 万行输出峰值 `+5,226,496` bytes，不冒充固定物理低配门。PostgreSQL `18.4` arm64 physical HA 125 项、timeline `1→2` 通过，报告 SHA-256 `45fab400eb449774d50429103dd766a2755166530ac54ddc1056f777bc16c15f`，临时 Docker 资源已清理。 +- 后续 controller/admission 切片取代上一条“自动 controller 仍未启用”的历史描述：生产 executor 已接入确定性 create/get-only Kubernetes Job controller。它以 durable approval/execution 为唯一创建依据；CREATE 的 409 或响应丢失通过 exact GET 收敛，`executing` 但 Job 缺失、approval 过期、终态 Job 或 contract 漂移均进入 `recoveryRequired`，不自动重建可能已经产生外部效果的动作。常规 dispatcher 只查询已注册 action type，因此不会误领取 Secret action。 +- Controller ServiceAccount 只具有 Job `create|get`，action ServiceAccount 无 API token且没有 Job/Pod/Secret 权限。`admissionregistration.k8s.io/v1` ValidatingAdmissionPolicy 以请求者身份和固定参数 ConfigMap 锁死 digest 镜像、command、ServiceAccount、source Secret、exact SHA-256 item/path、PostgreSQL SecretRef、安全上下文、资源与 volume/mount；`failurePolicy: Fail`、`parameterNotFoundAction: Deny`。基础 NetworkPolicy 保持 DNS-only，生产集群必须用私有 overlay 显式加入 API Server 精确 CIDR/TCP 443 出口，避免为 controller 放开任意公网。 +- 真实 K3s `v1.34.3+k3s1` 现场门已证明策略可由 API Server 编译：合规 Job dry-run 被接受，镜像漂移和参数 ConfigMap 缺失被拒绝；controller SA 的 `list|watch|delete jobs`、Pod 创建和 Secret 读取均被拒绝，action SA 的 Job/Pod 创建和 Secret 读取也均被拒绝。实现没有新增 workspace package、Edge daemon/timer/watcher 或低配设备常驻负担，18-package boundary 继续为 `singleSourcePackages=[]`、`shallowSourcePackages=[]`。 +- 本切片完整性门：controller/renderer/process 定向 18/18；cluster-admin 339 pass/3 条件 skip、cluster-postgres 328 pass/2 条件 skip；完整 18-package clean build/test 退出 0；backend 1196 项为 1194 pass、2 条条件 skip、0 fail；package boundary、cluster dependency、edge import、cluster deployment 均零 finding。PostgreSQL `18.4` arm64 physical HA 125 项、timeline `1→2` 通过，报告 SHA-256 `a3d34e61ea2064e1cde574e533137186e09fdce9048455da64f582906037fa0d`，临时 Docker 资源已清理。 +- ADR 继续保持 Proposed：升级失败自动回滚、终态 Job 的 durable 恢复决议和固定物理低配设备证据尚未完成。当前 controller 明确暴露恢复要求，不以不安全的自动重试冒充闭环。 diff --git a/packages/ql3-cluster-admin/package.json b/packages/ql3-cluster-admin/package.json index 5c9e440f..3646508e 100644 --- a/packages/ql3-cluster-admin/package.json +++ b/packages/ql3-cluster-admin/package.json @@ -225,6 +225,11 @@ "require": "./dist/plugin-package/executor/pluginPackageKubernetesSecretActionJob.js", "default": "./dist/plugin-package/executor/pluginPackageKubernetesSecretActionJob.js" }, + "./plugin-package-kubernetes-secret-action-controller": { + "types": "./dist/plugin-package/executor/pluginPackageKubernetesSecretActionController.d.ts", + "require": "./dist/plugin-package/executor/pluginPackageKubernetesSecretActionController.js", + "default": "./dist/plugin-package/executor/pluginPackageKubernetesSecretActionController.js" + }, "./prompt-output-gc-process": { "types": "./dist/prompt-output/retention/promptOutputGcProcess.d.ts", "require": "./dist/prompt-output/retention/promptOutputGcProcess.js", diff --git a/packages/ql3-cluster-admin/src/plugin-package/executor/pluginPackageExecutorProcess.ts b/packages/ql3-cluster-admin/src/plugin-package/executor/pluginPackageExecutorProcess.ts index 1c6270c8..3dd98c42 100644 --- a/packages/ql3-cluster-admin/src/plugin-package/executor/pluginPackageExecutorProcess.ts +++ b/packages/ql3-cluster-admin/src/plugin-package/executor/pluginPackageExecutorProcess.ts @@ -15,6 +15,9 @@ import { isPostgresTlsDnsServername, loadPostgresCertificateAuthorityFile, loadPostgresConnectionEnvironment, + PostgresApprovedActionExecutionRepository, + PostgresPluginPackageSecretBindingApprovalPlanReader, + PostgresPluginPackageSecretBindingTransitionApprovalPlanReader, type PostgresConnectionOptions, type PostgresPoolOptions, type PostgresSchemaReadinessReport, @@ -48,6 +51,12 @@ import { ProjectedPluginPackageSecretExistenceInspector } from '../secret-bindin import { runClusterPluginPackagePublisherRevocation, } from '../publisher/pluginPackagePublisherRevocation'; +import { + createClusterPluginPackageKubernetesSecretActionController, + type ClusterPluginPackageKubernetesSecretActionControllerResource, + type PluginPackageKubernetesSecretActionControllerSummary, +} from './pluginPackageKubernetesSecretActionController'; +import type { PluginPackageKubernetesSecretActionJobOptions } from './pluginPackageKubernetesSecretActionJob'; export type ClusterPluginPackageExecutorProcessEnvironment = Readonly< Record @@ -66,6 +75,11 @@ export type ClusterPluginPackageExecutorProcessConfig = revocationMaxPages: number; dispatchId: string | null; secretProjectionRoot: string | null; + kubernetesSecretActions: Readonly<{ + enabled: true; + limit: number; + job: Readonly; + }> | null; database: Readonly<{ connection: PostgresConnectionOptions; pool: PostgresPoolOptions; @@ -78,6 +92,7 @@ export interface ClusterPluginPackageExecutorBatchResult { readonly secretBindingApprovals: Readonly; readonly secretBindingTransitionApprovals: Readonly; readonly dispatch: Readonly; + readonly secretActionJobs: Readonly; } export type ClusterPluginPackageExecutorProcessResult = @@ -116,6 +131,13 @@ export interface RunClusterPluginPackageExecutorProcessOptions { pool: PostgresPool, ) => Promise; readonly now?: () => number; + readonly createSecretActionController?: ( + options: Parameters< + typeof createClusterPluginPackageKubernetesSecretActionController + >[0], + ) => Promise< + Readonly + >; } export class ClusterPluginPackageExecutorProcessConfigError extends TypeError { @@ -131,6 +153,11 @@ export class ClusterPluginPackageExecutorProcessConfigError extends TypeError { const SAFE_OWNER = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; const SAFE_DISPATCH_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const DNS_LABEL = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/; +const DNS_NAME = /^(?=.{1,253}$)[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?$/; +const SECRET_KEY = /^[A-Za-z0-9._-]{1,253}$/; +const IMAGE_DIGEST = + /^[a-z0-9](?:[a-z0-9._:/-]{0,510}[a-z0-9])?@sha256:[0-9a-f]{64}$/; function enabledValue( environment: ClusterPluginPackageExecutorProcessEnvironment, @@ -307,6 +334,179 @@ function secretProjectionRoot( return value; } +function requiredEnvironment( + environment: ClusterPluginPackageExecutorProcessEnvironment, + name: string, + maximumLength: number, +): string { + const value = boundedValue(environment, name, maximumLength); + if (value === undefined) { + throw new ClusterPluginPackageExecutorProcessConfigError( + `${name} is required when the Kubernetes Secret action controller is enabled`, + ); + } + return value; +} + +function kubernetesSecretActions( + environment: ClusterPluginPackageExecutorProcessEnvironment, +): Readonly<{ + enabled: true; + limit: number; + job: Readonly; +}> | null { + const enabled = environment.QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_ENABLED; + if (enabled === undefined || enabled === '' || enabled === 'false') return null; + if (enabled !== 'true') { + throw new ClusterPluginPackageExecutorProcessConfigError( + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_ENABLED must be true or false', + ); + } + const namespace = + boundedValue(environment, 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_NAMESPACE', 63) ?? + 'qinglong3-system'; + const serviceAccountName = + boundedValue( + environment, + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_SERVICE_ACCOUNT', + 63, + ) ?? 'ql3-plugin-package-secret-action'; + const sourceSecretName = + boundedValue( + environment, + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_SOURCE_SECRET', + 63, + ) ?? 'ql3-cluster-plugin-package-values'; + const image = requiredEnvironment( + environment, + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_IMAGE', + 640, + ); + const postgresCaSecretName = requiredEnvironment( + environment, + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_CA_SECRET', + 63, + ); + const postgresCaKey = + boundedValue( + environment, + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_CA_KEY', + 253, + ) ?? 'ca.crt'; + const servername = requiredEnvironment( + environment, + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_SERVERNAME', + 253, + ); + const urlSecretName = boundedValue( + environment, + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_SECRET', + 63, + ); + const urlSecretKey = boundedValue( + environment, + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_KEY', + 253, + ); + const fieldNames = [ + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_AUTH_SECRET', + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_HOST', + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_PORT', + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_DATABASE', + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_USERNAME_KEY', + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_PASSWORD_KEY', + ] as const; + const fields = fieldNames.map((name) => boundedValue(environment, name, 253)); + const urlMode = urlSecretName !== undefined || urlSecretKey !== undefined; + const fieldMode = fields.some((value) => value !== undefined); + if ( + !DNS_LABEL.test(namespace) || + !DNS_LABEL.test(serviceAccountName) || + !DNS_LABEL.test(sourceSecretName) || + !IMAGE_DIGEST.test(image) || + !DNS_LABEL.test(postgresCaSecretName) || + !SECRET_KEY.test(postgresCaKey) || + !DNS_NAME.test(servername) || + urlMode === fieldMode + ) { + throw new ClusterPluginPackageExecutorProcessConfigError( + 'Kubernetes Secret action Job identity is invalid', + ); + } + let connection: PluginPackageKubernetesSecretActionJobOptions['postgres']['connection']; + if (urlMode) { + if ( + urlSecretName === undefined || + urlSecretKey === undefined || + !DNS_LABEL.test(urlSecretName) || + !SECRET_KEY.test(urlSecretKey) + ) { + throw new ClusterPluginPackageExecutorProcessConfigError( + 'Kubernetes Secret action PostgreSQL URL reference is invalid', + ); + } + connection = Object.freeze({ + mode: 'url', + secretName: urlSecretName, + urlKey: urlSecretKey, + }); + } else { + if (fields.some((value) => value === undefined)) { + throw new ClusterPluginPackageExecutorProcessConfigError( + 'Kubernetes Secret action PostgreSQL field reference is incomplete', + ); + } + const [authSecretName, host, portValue, database, usernameKey, passwordKey] = + fields as [string, string, string, string, string, string]; + const port = Number(portValue); + if ( + !DNS_LABEL.test(authSecretName) || + !DNS_NAME.test(host) || + !Number.isSafeInteger(port) || + port < 1 || + port > 65_535 || + !/^[A-Za-z_][A-Za-z0-9_$-]{0,62}$/.test(database) || + !SECRET_KEY.test(usernameKey) || + !SECRET_KEY.test(passwordKey) + ) { + throw new ClusterPluginPackageExecutorProcessConfigError( + 'Kubernetes Secret action PostgreSQL field reference is invalid', + ); + } + connection = Object.freeze({ + mode: 'fields', + authSecretName, + host, + port, + database, + usernameKey, + passwordKey, + }); + } + return Object.freeze({ + enabled: true, + limit: integerValue( + environment, + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_LIMIT', + 8, + 1, + 32, + ), + job: Object.freeze({ + namespace, + serviceAccountName, + sourceSecretName, + image, + postgres: Object.freeze({ + connection, + caSecretName: postgresCaSecretName, + caKey: postgresCaKey, + servername, + }), + }), + }); +} + export function loadClusterPluginPackageExecutorProcessConfig( environment: ClusterPluginPackageExecutorProcessEnvironment, ): ClusterPluginPackageExecutorProcessConfig { @@ -382,6 +582,7 @@ export function loadClusterPluginPackageExecutorProcessConfig( ), dispatchId, secretProjectionRoot: secretProjectionRoot(environment), + kubernetesSecretActions: kubernetesSecretActions(environment), database: databaseConfig(environment), }); } @@ -424,6 +625,18 @@ function emptyApprovalSummary(): Readonly<{ }); } +function emptySecretActionJobSummary(): Readonly { + return Object.freeze({ + scanned: 0, + created: 0, + existing: 0, + active: 0, + recoveryRequired: 0, + unavailable: 0, + truncated: false, + }); +} + export async function runClusterPluginPackageExecutorProcess( options: RunClusterPluginPackageExecutorProcessOptions, ): Promise { @@ -446,6 +659,8 @@ export async function runClusterPluginPackageExecutorProcess( typeof options.createDispatcher !== 'function') || (options.assertReady !== undefined && typeof options.assertReady !== 'function') || + (options.createSecretActionController !== undefined && + typeof options.createSecretActionController !== 'function') || (options.now !== undefined && typeof options.now !== 'function') ) { throw new TypeError('Plugin Package executor process options are invalid'); @@ -464,6 +679,9 @@ export async function runClusterPluginPackageExecutorProcess( }); const database = await openDatabase(); let failure: unknown; + let secretActionControllerResource: + | Readonly + | undefined; try { const evidence = await ( options.assertReady ?? assertPostgresPackageExecutorSchemaReady @@ -531,6 +749,7 @@ export async function runClusterPluginPackageExecutorProcess( secretBindingApprovals: emptyApprovalSummary(), secretBindingTransitionApprovals: emptyApprovalSummary(), dispatch, + secretActionJobs: emptySecretActionJobSummary(), }), ]), }); @@ -559,6 +778,32 @@ export async function runClusterPluginPackageExecutorProcess( limit: config.approvalBatchSize, ...(options.now ? { now: options.now } : {}), }); + let secretActionJobs = emptySecretActionJobSummary(); + if (config.kubernetesSecretActions !== null) { + if (!secretActionControllerResource) { + const createController = + options.createSecretActionController ?? + createClusterPluginPackageKubernetesSecretActionController; + secretActionControllerResource = await createController({ + executions: new PostgresApprovedActionExecutionRepository( + database.pool, + ), + bindingPlans: + new PostgresPluginPackageSecretBindingApprovalPlanReader( + database.pool, + ), + transitionPlans: + new PostgresPluginPackageSecretBindingTransitionApprovalPlanReader( + database.pool, + ), + job: config.kubernetesSecretActions.job, + ...(options.now ? { now: options.now } : {}), + }); + } + secretActionJobs = await secretActionControllerResource.controller.reconcile({ + limit: config.kubernetesSecretActions.limit, + }); + } const dispatch = await dispatcher.dispatchBatch({ limit: config.dispatchBatchSize, }); @@ -568,9 +813,10 @@ export async function runClusterPluginPackageExecutorProcess( secretBindingApprovals, secretBindingTransitionApprovals, dispatch, + secretActionJobs, }); batches.push(batch); - if (isIdleBatch(batch)) break; + if (secretActionJobs.scanned > 0 || isIdleBatch(batch)) break; } return Object.freeze({ status: 'completed', @@ -581,16 +827,33 @@ export async function runClusterPluginPackageExecutorProcess( failure = error; throw error; } finally { + let disposeError: unknown; + try { + secretActionControllerResource?.dispose(); + } catch (error) { + disposeError = error; + } + let closeError: unknown; try { await database.close(); - } catch (closeError) { + } catch (error) { + closeError = error; + } + const cleanupErrors = [disposeError, closeError].filter( + (error) => error !== undefined, + ); + if (cleanupErrors.length > 0) { if (failure !== undefined) { throw new AggregateError( - [failure, closeError], - 'Plugin Package executor process failed and PostgreSQL did not close', + [failure, ...cleanupErrors], + 'Plugin Package executor process and resource cleanup failed', ); } - throw closeError; + if (cleanupErrors.length === 1) throw cleanupErrors[0]; + throw new AggregateError( + cleanupErrors, + 'Plugin Package executor resource cleanup failed', + ); } } } diff --git a/packages/ql3-cluster-admin/src/plugin-package/executor/pluginPackageKubernetesSecretActionController.ts b/packages/ql3-cluster-admin/src/plugin-package/executor/pluginPackageKubernetesSecretActionController.ts new file mode 100644 index 00000000..b258dd45 --- /dev/null +++ b/packages/ql3-cluster-admin/src/plugin-package/executor/pluginPackageKubernetesSecretActionController.ts @@ -0,0 +1,403 @@ +import type { ApprovedActionExecutionSnapshot } from '@qinglong/runtime-core/approved-action-execution'; +import { + PLUGIN_PACKAGE_SECRET_BINDING_ACTION_TYPE, + type PluginPackageSecretBindingApprovalPlan, +} from '@qinglong/runtime-core/plugin-package-secret-binding-approval-plan'; +import { + PLUGIN_PACKAGE_SECRET_BINDING_TRANSITION_ACTION_TYPE, + type PluginPackageSecretBindingTransitionApprovalPlan, +} from '@qinglong/runtime-core/plugin-package-secret-binding-transition-approval-plan'; + +import { + createPluginPackageKubernetesSecretActionJob, + type PluginPackageKubernetesSecretActionJobOptions, +} from './pluginPackageKubernetesSecretActionJob'; + +const FIELD_MANAGER = 'qinglong-plugin-package-secret-action-controller'; +const MAX_PAGE_SIZE = 32; + +type SecretActionApprovalPlan = + | PluginPackageSecretBindingApprovalPlan + | PluginPackageSecretBindingTransitionApprovalPlan; + +type JsonObject = Record; + +export interface PluginPackageSecretActionApprovalPlanReader { + findByActionRef(actionRef: string): Promise | null>; +} + +export interface PluginPackageKubernetesSecretActionExecutionReader { + listReconciliableExecutions(query: Readonly<{ + nowMs: number; + limit: number; + actionTypes: readonly string[]; + }>): Promise[]; + truncated: boolean; + }>>; +} + +export interface PluginPackageKubernetesSecretActionJobResource { + readonly metadata?: Readonly<{ + name?: string; + namespace?: string; + labels?: Readonly>; + annotations?: Readonly>; + }>; + readonly spec?: Readonly>; + readonly status?: Readonly<{ + conditions?: readonly Readonly<{ + type?: string; + status?: string; + }>[]; + }>; +} + +export interface PluginPackageKubernetesSecretActionJobApi { + createNamespacedJob( + request: Readonly<{ + namespace: string; + body: Readonly>; + fieldManager: typeof FIELD_MANAGER; + fieldValidation: 'Strict'; + }>, + ): Promise; + readNamespacedJob( + request: Readonly<{ + name: string; + namespace: string; + }>, + ): Promise; +} + +export interface PluginPackageKubernetesSecretActionControllerOptions { + readonly executions: PluginPackageKubernetesSecretActionExecutionReader; + readonly bindingPlans: PluginPackageSecretActionApprovalPlanReader; + readonly transitionPlans: PluginPackageSecretActionApprovalPlanReader; + readonly jobs: PluginPackageKubernetesSecretActionJobApi; + readonly job: Readonly; + readonly now?: () => number; +} + +export interface PluginPackageKubernetesSecretActionControllerSummary { + readonly scanned: number; + readonly created: number; + readonly existing: number; + readonly active: number; + readonly recoveryRequired: number; + readonly unavailable: number; + readonly truncated: boolean; +} + +export interface ClusterPluginPackageKubernetesSecretActionControllerResource { + readonly controller: PluginPackageKubernetesSecretActionController; + dispose(): void; +} + +export class PluginPackageKubernetesSecretActionControllerConflictError extends Error { + readonly code = 'PLUGIN_PACKAGE_KUBERNETES_SECRET_ACTION_CONTROLLER_CONFLICT'; + + constructor() { + super('Kubernetes Secret action Job conflicts with the durable dispatch'); + this.name = 'PluginPackageKubernetesSecretActionControllerConflictError'; + } +} + +export class PluginPackageKubernetesSecretActionControllerUnavailableError extends Error { + readonly code = + 'PLUGIN_PACKAGE_KUBERNETES_SECRET_ACTION_CONTROLLER_UNAVAILABLE'; + + constructor(options?: ErrorOptions) { + super('Kubernetes Secret action Job authority is unavailable', options); + this.name = + 'PluginPackageKubernetesSecretActionControllerUnavailableError'; + } +} + +function apiStatus(error: unknown): number | null { + if (!error || typeof error !== 'object') return null; + if ('code' in error && typeof error.code === 'number') return error.code; + if ( + 'response' in error && + error.response && + typeof error.response === 'object' && + 'statusCode' in error.response && + typeof error.response.statusCode === 'number' + ) { + return error.response.statusCode; + } + return null; +} + +function object(value: unknown): JsonObject | null { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as JsonObject) + : null; +} + +function expectedSubset(expected: unknown, observed: unknown): boolean { + if (Array.isArray(expected)) { + return ( + Array.isArray(observed) && + expected.length === observed.length && + expected.every((value, index) => expectedSubset(value, observed[index])) + ); + } + const expectedObject = object(expected); + if (expectedObject) { + const observedObject = object(observed); + return ( + observedObject !== null && + Object.entries(expectedObject).every(([key, value]) => + expectedSubset(value, observedObject[key]), + ) + ); + } + return Object.is(expected, observed); +} + +function terminalStatus( + job: Readonly, +): 'active' | 'complete' | 'failed' { + const conditions = job.status?.conditions ?? []; + const complete = conditions.some( + (condition) => condition.type === 'Complete' && condition.status === 'True', + ); + const failed = conditions.some( + (condition) => condition.type === 'Failed' && condition.status === 'True', + ); + if (complete && failed) { + throw new PluginPackageKubernetesSecretActionControllerConflictError(); + } + if (complete) return 'complete'; + if (failed) return 'failed'; + return 'active'; +} + +function assertObservedJob( + expected: Readonly>, + observed: Readonly, +): void { + const expectedMetadata = object(expected.metadata)!; + const expectedSpec = object(expected.spec)!; + const observedMetadata = observed.metadata; + const observedPodSpec = object( + object(object(observed.spec)?.template)?.spec, + ); + if ( + observedMetadata?.name !== expectedMetadata.name || + observedMetadata?.namespace !== expectedMetadata.namespace || + !expectedSubset(expectedMetadata.labels, observedMetadata?.labels) || + !expectedSubset(expectedMetadata.annotations, observedMetadata?.annotations) || + !expectedSubset(expectedSpec, observed.spec) || + observedPodSpec === null || + observedPodSpec.hostNetwork === true || + observedPodSpec.hostPID === true || + observedPodSpec.hostIPC === true || + (Array.isArray(observedPodSpec.initContainers) && + observedPodSpec.initContainers.length > 0) || + (Array.isArray(observedPodSpec.ephemeralContainers) && + observedPodSpec.ephemeralContainers.length > 0) + ) { + throw new PluginPackageKubernetesSecretActionControllerConflictError(); + } +} + +export class PluginPackageKubernetesSecretActionController { + readonly #now: () => number; + + constructor( + private readonly options: PluginPackageKubernetesSecretActionControllerOptions, + ) { + if ( + !options || + typeof options !== 'object' || + !options.executions || + typeof options.executions.listReconciliableExecutions !== 'function' || + !options.bindingPlans || + typeof options.bindingPlans.findByActionRef !== 'function' || + !options.transitionPlans || + typeof options.transitionPlans.findByActionRef !== 'function' || + !options.jobs || + typeof options.jobs.createNamespacedJob !== 'function' || + typeof options.jobs.readNamespacedJob !== 'function' || + !options.job || + typeof options.job !== 'object' || + (options.now !== undefined && typeof options.now !== 'function') + ) { + throw new TypeError('Kubernetes Secret action controller options are invalid'); + } + this.#now = options.now ?? Date.now; + } + + async reconcile( + input: Readonly<{ limit?: number }> = {}, + ): Promise> { + if ( + !input || + typeof input !== 'object' || + Array.isArray(input) || + Object.keys(input).some((key) => key !== 'limit') + ) { + throw new TypeError('Kubernetes Secret action reconciliation is invalid'); + } + const limit = input.limit ?? 8; + if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_PAGE_SIZE) { + throw new RangeError('Kubernetes Secret action reconciliation limit is invalid'); + } + const nowMs = this.#now(); + if (!Number.isSafeInteger(nowMs) || nowMs < 0) { + throw new RangeError('Kubernetes Secret action controller clock is invalid'); + } + let page; + try { + page = await this.options.executions.listReconciliableExecutions({ + nowMs, + limit, + actionTypes: Object.freeze([ + PLUGIN_PACKAGE_SECRET_BINDING_ACTION_TYPE, + PLUGIN_PACKAGE_SECRET_BINDING_TRANSITION_ACTION_TYPE, + ]), + }); + } catch (error) { + throw new PluginPackageKubernetesSecretActionControllerUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + const summary = { + scanned: page.executions.length, + created: 0, + existing: 0, + active: 0, + recoveryRequired: 0, + unavailable: 0, + truncated: page.truncated, + }; + for (const snapshot of page.executions) { + try { + const result = await this.#reconcileOne(snapshot, nowMs); + summary[result] += 1; + } catch (error) { + if ( + error instanceof + PluginPackageKubernetesSecretActionControllerConflictError + ) { + throw error; + } + summary.unavailable += 1; + } + } + return Object.freeze({ ...summary }); + } + + async #reconcileOne( + snapshot: Readonly, + nowMs: number, + ): Promise<'created' | 'existing' | 'active' | 'recoveryRequired'> { + const plan = await this.#plan(snapshot); + if (!plan) { + throw new PluginPackageKubernetesSecretActionControllerUnavailableError(); + } + const desired = createPluginPackageKubernetesSecretActionJob({ + dispatch: snapshot.dispatch, + approvalPlan: plan, + options: this.options.job, + }); + const metadata = object(desired.metadata)!; + const name = metadata.name as string; + const namespace = metadata.namespace as string; + let observed: PluginPackageKubernetesSecretActionJobResource; + let disposition: 'created' | 'existing' | 'active' = 'active'; + try { + observed = await this.options.jobs.readNamespacedJob({ name, namespace }); + } catch (error) { + if (apiStatus(error) !== 404) throw error; + if ( + snapshot.execution.status === 'executing' || + nowMs > plan.expiresAtMs + ) { + return 'recoveryRequired'; + } + try { + observed = await this.options.jobs.createNamespacedJob({ + namespace, + body: desired, + fieldManager: FIELD_MANAGER, + fieldValidation: 'Strict', + }); + disposition = 'created'; + } catch (createError) { + // CREATE may have succeeded even when its response was lost. Converge + // every ambiguous failure through an exact-name GET before surfacing it. + try { + observed = await this.options.jobs.readNamespacedJob({ + name, + namespace, + }); + } catch (readAfterCreateError) { + if (apiStatus(readAfterCreateError) === 404) throw createError; + throw readAfterCreateError; + } + disposition = 'existing'; + } + } + assertObservedJob(desired, observed); + const terminal = terminalStatus(observed); + if (terminal !== 'active') return 'recoveryRequired'; + return disposition; + } + + #plan( + snapshot: Readonly, + ): Promise | null> { + const { actionType, actionRef } = snapshot.dispatch.action; + if (actionType === PLUGIN_PACKAGE_SECRET_BINDING_ACTION_TYPE) { + return this.options.bindingPlans.findByActionRef(actionRef); + } + if (actionType === PLUGIN_PACKAGE_SECRET_BINDING_TRANSITION_ACTION_TYPE) { + return this.options.transitionPlans.findByActionRef(actionRef); + } + return Promise.resolve(null); + } +} + + +type KubernetesModule = typeof import('@kubernetes/client-node', { + with: { 'resolution-mode': 'import' } +}); + +export async function createClusterPluginPackageKubernetesSecretActionController( + options: Omit< + PluginPackageKubernetesSecretActionControllerOptions, + 'jobs' + >, +): Promise> { + const kubernetes = (await import('@kubernetes/client-node')) as KubernetesModule; + const config = new kubernetes.KubeConfig(); + config.loadFromCluster(); + const jobs = config.makeApiClient( + kubernetes.BatchV1Api, + ) as unknown as PluginPackageKubernetesSecretActionJobApi; + let active = true; + return Object.freeze({ + controller: new PluginPackageKubernetesSecretActionController({ + ...options, + jobs, + }), + dispose() { + if (!active) return; + active = false; + for (const user of config.getUsers()) { + const mutable = user as { + token?: string; + certData?: string; + keyData?: string; + }; + mutable.token = ''; + mutable.certData = ''; + mutable.keyData = ''; + } + config.setCurrentContext('disposed'); + }, + }); +} diff --git a/packages/ql3-cluster-admin/test/pluginPackageExecutorProcess.test.cjs b/packages/ql3-cluster-admin/test/pluginPackageExecutorProcess.test.cjs index 0a360e03..0a206fc6 100644 --- a/packages/ql3-cluster-admin/test/pluginPackageExecutorProcess.test.cjs +++ b/packages/ql3-cluster-admin/test/pluginPackageExecutorProcess.test.cjs @@ -27,6 +27,27 @@ function environment(overrides = {}) { }; } +function actionControllerEnvironment(overrides = {}) { + return environment({ + QL3_PLUGIN_PACKAGE_EXECUTOR_SECRET_ROOT: undefined, + QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_ENABLED: 'true', + QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_LIMIT: '6', + QL3_PLUGIN_PACKAGE_SECRET_ACTION_IMAGE: + 'registry.example.com/qinglong/qinglong3-cluster-admin@sha256:' + + 'c'.repeat(64), + QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_CA_SECRET: + 'ql3-cluster-plugin-package-executor', + QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_CA_KEY: 'postgres-ca.crt', + QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_SERVERNAME: + 'postgres.qinglong3-system.svc', + QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_SECRET: + 'ql3-cluster-plugin-package-executor', + QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_KEY: + 'postgres-package-executor-url', + ...overrides, + }); +} + test('disabled executor opens no PostgreSQL authority', async () => { let opened = 0; const result = await runClusterPluginPackageExecutorProcess({ @@ -74,6 +95,27 @@ test('loads one bounded action-scoped dispatch without widening batch limits', ( assert.equal(config.maxBatches, 4); }); +test('loads a bounded digest-pinned Kubernetes Secret action controller', () => { + const config = loadClusterPluginPackageExecutorProcessConfig( + actionControllerEnvironment(), + ); + assert.equal(config.enabled, true); + assert.equal(config.secretProjectionRoot, null); + assert.equal(config.kubernetesSecretActions.limit, 6); + assert.equal( + config.kubernetesSecretActions.job.serviceAccountName, + 'ql3-plugin-package-secret-action', + ); + assert.equal( + config.kubernetesSecretActions.job.postgres.connection.mode, + 'url', + ); + assert.equal( + config.kubernetesSecretActions.job.postgres.connection.secretName, + 'ql3-cluster-plugin-package-executor', + ); +}); + test('action-scoped mode skips every Approval consumer and shared queue scan', async () => { const calls = []; const pool = {}; @@ -151,6 +193,129 @@ test('action-scoped mode skips every Approval consumer and shared queue scan', a assert.equal(result.batches[0].dispatch.succeeded, 1); }); +test('batch mode consumes approvals before reconciling exact Secret action Jobs', async () => { + const calls = []; + const pool = { + async query() { + throw new Error('repositories are injected behind the controller factory'); + }, + async connect() { + throw new Error('repositories are injected behind the controller factory'); + }, + }; + const approvalSummary = { + scanned: 0, + consumed: 0, + existing: 0, + expired: 0, + blocked: 0, + }; + const readiness = { + ready: true, + writablePrimary: true, + serverVersionNum: 180004, + serverMajor: 18, + currentUser: 'ql3_package_executor', + contractName: 'control-core', + contractVersion: 62, + migrationIds: ['pg-0063-plugin-package-secret-binding-transition-receipts'], + }; + const result = await runClusterPluginPackageExecutorProcess({ + environment: actionControllerEnvironment(), + async openDatabase() { + calls.push('open'); + return { + pool, + async close() { + calls.push('close'); + }, + }; + }, + async assertReady() { + calls.push('ready'); + return readiness; + }, + async consumeApprovals() { + calls.push('publisher-approvals'); + return approvalSummary; + }, + async consumeTrustTransitionApprovals() { + calls.push('trust-approvals'); + return approvalSummary; + }, + async consumeSecretBindingApprovals() { + calls.push('binding-approvals'); + return { ...approvalSummary, scanned: 1, consumed: 1 }; + }, + async consumeSecretBindingTransitionApprovals() { + calls.push('transition-approvals'); + return approvalSummary; + }, + async createSecretActionController(options) { + assert.equal(options.job.image.endsWith('c'.repeat(64)), true); + calls.push('controller-open'); + return { + controller: { + async reconcile({ limit }) { + calls.push(`reconcile:${limit}`); + return { + scanned: 1, + created: 1, + existing: 0, + active: 0, + recoveryRequired: 0, + unavailable: 0, + truncated: false, + }; + }, + }, + dispose() { + calls.push('controller-close'); + }, + }; + }, + createDispatcher() { + return { + async dispatchById() { + throw new Error('batch mode must not exact dispatch'); + }, + async dispatchBatch() { + calls.push('dispatch-batch'); + return { + scanned: 0, + claimed: 0, + started: 0, + succeeded: 0, + failed: 0, + blocked: 0, + retrying: 0, + deferred: 0, + recoveryRequired: 0, + alreadyTerminal: 0, + unavailable: 0, + truncated: false, + }; + }, + }; + }, + }); + assert.equal(result.batches.length, 1); + assert.equal(result.batches[0].secretActionJobs.created, 1); + assert.deepEqual(calls, [ + 'open', + 'ready', + 'publisher-approvals', + 'trust-approvals', + 'binding-approvals', + 'transition-approvals', + 'controller-open', + 'reconcile:6', + 'dispatch-batch', + 'controller-close', + 'close', + ]); +}); + test('rejects implicit insecure PostgreSQL and unbounded work', () => { for (const invalid of [ environment({ QL3_POSTGRES_ALLOW_INSECURE: undefined }), @@ -161,6 +326,12 @@ test('rejects implicit insecure PostgreSQL and unbounded work', () => { environment({ QL3_PLUGIN_PACKAGE_EXECUTOR_DISPATCH_ID: 'dispatch id with spaces', }), + actionControllerEnvironment({ + QL3_PLUGIN_PACKAGE_SECRET_ACTION_IMAGE: 'tag-only:latest', + }), + actionControllerEnvironment({ + QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_SECRET: undefined, + }), ]) { assert.throws( () => loadClusterPluginPackageExecutorProcessConfig(invalid), diff --git a/packages/ql3-cluster-admin/test/pluginPackageKubernetesSecretActionController.test.cjs b/packages/ql3-cluster-admin/test/pluginPackageKubernetesSecretActionController.test.cjs new file mode 100644 index 00000000..5ca63755 --- /dev/null +++ b/packages/ql3-cluster-admin/test/pluginPackageKubernetesSecretActionController.test.cjs @@ -0,0 +1,366 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + consumeApprovalRequest, + createApprovalRequest, + decideApprovalRequest, +} = require('@qinglong/runtime-core/approved-action'); +const { + createApprovedActionExecution, +} = require('@qinglong/runtime-core/approved-action-execution'); +const { + createPluginPackageResourceGeneration, +} = require('@qinglong/runtime-core/plugin-package-resource-generation'); +const { + createPluginPackageSecretBindingApprovalPlan, +} = require('@qinglong/runtime-core/plugin-package-secret-binding-approval-plan'); +const { + createPluginPackageSecretBindingPlan, +} = require('@qinglong/runtime-core/plugin-package-secret-binding-plan'); +const { + createSecretRef, +} = require('@qinglong/runtime-core/secret-reference'); +const { + PluginPackageKubernetesSecretActionController, + PluginPackageKubernetesSecretActionControllerConflictError, +} = require('@qinglong/cluster-admin/plugin-package-kubernetes-secret-action-controller'); + +const REQUESTER = Object.freeze({ type: 'user', id: 'cluster-owner' }); +const REVIEWER = Object.freeze({ type: 'user', id: 'security-reviewer' }); +const CONSUMER = Object.freeze({ + type: 'system', + id: 'cluster_package_executor', +}); +const FENCE = Object.freeze({ projectVersion: 3, bindingVersion: 4 }); + +function fixture() { + const manifest = { + apiVersion: 'qinglong.io/v1alpha1', + kind: 'Package', + metadata: { + name: 'controller-fixture', + displayName: 'Controller Fixture', + version: '1.0.0', + description: 'Secret action controller fixture', + license: 'Apache-2.0', + }, + spec: { + compatibility: { + qinglong: '>=3.0.0-0 <4.0.0', + architectures: ['arm64'], + deploymentProfiles: ['cluster-control'], + }, + runtimes: [], + resources: { + memory: { recommended: '32Mi' }, + disk: { install: '4Mi', working: '8Mi' }, + }, + permissions: { + network: { allowedHosts: [] }, + secrets: [{ name: 'TOKEN', required: true }], + tools: ['secret.use'], + }, + contents: { tasks: [], workflows: [], prompts: [], tools: [] }, + }, + }; + const generation = createPluginPackageResourceGeneration({ + installationId: 'install-controller-1', + projectId: 'project-1', + packageName: 'controller-fixture', + lockDigest: 'a'.repeat(64), + generation: 1, + previousActiveLockDigest: null, + contentDigest: 'b'.repeat(64), + contents: manifest.spec.contents, + }); + const bindingPlan = createPluginPackageSecretBindingPlan({ + generation, + manifest, + assignments: [ + { + name: 'TOKEN', + secretRef: createSecretRef({ + projectId: 'project-1', + name: 'runtime-token', + version: 2, + }), + }, + ], + plannedAtMs: 100, + }); + const plan = createPluginPackageSecretBindingApprovalPlan({ + actionRef: 'secret-binding:controller-fixture', + bindingPlan, + requestedBy: REQUESTER, + expiresAtMs: 1_000, + }); + const action = require('@qinglong/runtime-core/plugin-package-secret-binding-approval-plan') + .pluginPackageSecretBindingApprovedAction(plan); + const approved = decideApprovalRequest( + createApprovalRequest({ + id: 'approval-controller-1', + projectId: 'project-1', + action, + risk: 'high', + decisionMode: 'separation_of_duty', + requestedBy: REQUESTER, + requestedAtMs: 110, + expiresAtMs: 900, + requestFence: FENCE, + }), + { + expectedVersion: 1, + decisionId: 'decision-controller-1', + decision: 'approved', + reasonCode: 'reviewed', + principal: { + subject: REVIEWER, + authenticationId: 'auth-reviewer', + authenticatedAtMs: 100, + expiresAtMs: 800, + assurance: 'multi_factor', + }, + decidedAtMs: 120, + authorizationFence: FENCE, + }, + ); + const dispatch = consumeApprovalRequest(approved, { + expectedVersion: 2, + consumptionId: 'consume-controller-1', + dispatchId: 'dispatch-controller-1', + action, + requestedBy: REQUESTER, + consumedBy: CONSUMER, + consumedAtMs: 130, + authorizationFence: FENCE, + }).dispatch; + return { + plan, + snapshot: Object.freeze({ + dispatch, + execution: createApprovedActionExecution(dispatch), + }), + }; +} + +function jobOptions() { + return { + namespace: 'qinglong3-system', + serviceAccountName: 'ql3-plugin-package-secret-action', + sourceSecretName: 'ql3-cluster-plugin-package-values', + image: + 'registry.example.com/qinglong/qinglong3-cluster-admin@sha256:' + + 'c'.repeat(64), + postgres: { + connection: { + mode: 'url', + secretName: 'ql3-cluster-plugin-package-executor', + urlKey: 'postgres-package-executor-url', + }, + caSecretName: 'ql3-cluster-plugin-package-executor', + caKey: 'postgres-ca.crt', + servername: 'postgres.qinglong3-system.svc', + }, + }; +} + +function apiError(code) { + return Object.assign(new Error(`Kubernetes ${code}`), { code }); +} + +function controller({ read, create, now = 200, snapshot: snapshotOverride }) { + const { plan, snapshot } = fixture(); + return new PluginPackageKubernetesSecretActionController({ + executions: { + async listReconciliableExecutions(query) { + assert.deepEqual(query.actionTypes, [ + 'plugin_package.secret_binding.bind', + 'plugin_package.secret_binding.transition', + ]); + return { + executions: [snapshotOverride ?? snapshot], + truncated: false, + }; + }, + }, + bindingPlans: { + async findByActionRef(actionRef) { + assert.equal(actionRef, plan.actionRef); + return plan; + }, + }, + transitionPlans: { + async findByActionRef() { + throw new Error('transition reader must not run'); + }, + }, + jobs: { + readNamespacedJob: read, + createNamespacedJob: create, + }, + job: jobOptions(), + now: () => now, + }); +} + +test('creates one Strict deterministic Job without claiming the execution', async () => { + const calls = []; + const subject = controller({ + async read(request) { + calls.push(['read', request]); + throw apiError(404); + }, + async create(request) { + calls.push(['create', request]); + return request.body; + }, + }); + const result = await subject.reconcile({ limit: 4 }); + assert.equal(result.created, 1); + assert.equal(result.recoveryRequired, 0); + assert.equal(calls[0][0], 'read'); + assert.equal(calls[1][0], 'create'); + assert.equal(calls[1][1].fieldValidation, 'Strict'); + assert.equal( + calls[1][1].fieldManager, + 'qinglong-plugin-package-secret-action-controller', + ); + assert.match(calls[1][1].body.metadata.name, /^ql3-package-secret-[0-9a-f]{32}$/); +}); + +test('converges a concurrent create through one exact get', async () => { + let desired; + let reads = 0; + const subject = controller({ + async read() { + reads += 1; + if (reads === 1) throw apiError(404); + return { + ...desired, + metadata: { + ...desired.metadata, + uid: 'server-owned-uid', + }, + spec: { + ...desired.spec, + completionMode: 'NonIndexed', + }, + }; + }, + async create(request) { + desired = request.body; + throw apiError(409); + }, + }); + const result = await subject.reconcile(); + assert.equal(result.existing, 1); + assert.equal(reads, 2); +}); + +test('converges a lost successful CREATE response through one exact get', async () => { + let desired; + let reads = 0; + const subject = controller({ + async read() { + reads += 1; + if (reads === 1) throw apiError(404); + return desired; + }, + async create(request) { + desired = request.body; + throw apiError(503); + }, + }); + const result = await subject.reconcile(); + assert.equal(result.existing, 1); + assert.equal(result.unavailable, 0); + assert.equal(reads, 2); +}); + +test('does not recreate a missing Job for an already executing action', async () => { + const { snapshot } = fixture(); + let creates = 0; + const subject = controller({ + snapshot: { + ...snapshot, + execution: { ...snapshot.execution, status: 'executing' }, + }, + async read() { + throw apiError(404); + }, + async create() { + creates += 1; + throw new Error('must not create'); + }, + }); + const result = await subject.reconcile(); + assert.equal(result.recoveryRequired, 1); + assert.equal(creates, 0); +}); + +test('does not create a missing Job after the approval plan expires', async () => { + let creates = 0; + const subject = controller({ + now: 1_001, + async read() { + throw apiError(404); + }, + async create() { + creates += 1; + throw new Error('must not create'); + }, + }); + const result = await subject.reconcile(); + assert.equal(result.recoveryRequired, 1); + assert.equal(creates, 0); +}); + +test('marks a terminal Job with a nonterminal execution as recovery required', async () => { + let desired; + const subject = controller({ + async read() { + if (!desired) throw apiError(404); + return desired; + }, + async create(request) { + desired = { + ...request.body, + status: { + conditions: [{ type: 'Failed', status: 'True' }], + }, + }; + return desired; + }, + }); + const result = await subject.reconcile(); + assert.equal(result.recoveryRequired, 1); + assert.equal(result.created, 0); +}); + +test('fails closed when a deterministic Job name contains another contract', async () => { + const subject = controller({ + async read() { + const error = apiError(404); + throw error; + }, + async create(request) { + return { + ...request.body, + metadata: { + ...request.body.metadata, + annotations: { + ...request.body.metadata.annotations, + 'qinglong.io/secret-action-job-digest': 'f'.repeat(64), + }, + }, + }; + }, + }); + await assert.rejects( + () => subject.reconcile(), + PluginPackageKubernetesSecretActionControllerConflictError, + ); +}); diff --git a/packages/ql3-cluster-postgres/src/approved-action/approvedActionExecutionRepository.ts b/packages/ql3-cluster-postgres/src/approved-action/approvedActionExecutionRepository.ts index bef1c433..cd05e6e4 100644 --- a/packages/ql3-cluster-postgres/src/approved-action/approvedActionExecutionRepository.ts +++ b/packages/ql3-cluster-postgres/src/approved-action/approvedActionExecutionRepository.ts @@ -379,6 +379,62 @@ export class PostgresApprovedActionExecutionRepository } } + async listReconciliableExecutions( + query: Readonly<{ + nowMs: number; + limit: number; + actionTypes: readonly string[]; + }>, + ): Promise[]; + truncated: boolean; + }>> { + assertPageSize(query.limit); + const handledActionTypes = actionTypes(query.actionTypes); + if (!Number.isSafeInteger(query.nowMs) || query.nowMs < 0) { + throw new ApprovedActionExecutionStateConflictError(); + } + if (handledActionTypes.length === 0) { + return Object.freeze({ executions: Object.freeze([]), truncated: false }); + } + try { + const result = await this.pool.query( + `SELECT execution.execution_json AS "executionJson", + execution.execution_digest AS "executionDigest", + dispatch.dispatch_json AS "dispatchJson" + FROM "ql3"."approved_action_executions" AS execution + JOIN "ql3"."approved_action_dispatches" AS dispatch + ON dispatch.dispatch_id = execution.dispatch_id + WHERE dispatch.action_type = ANY($1::varchar[]) + AND ( + ( + execution.status IN ('pending','leased','retry_wait') + AND execution.eligible_at_ms <= $2 + ) + OR execution.status = 'executing' + ) + ORDER BY execution.updated_at_ms, execution.dispatch_id + LIMIT $3`, + [handledActionTypes, query.nowMs, query.limit + 1], + ); + const truncated = result.rows.length > query.limit; + const rows = truncated ? result.rows.slice(0, query.limit) : result.rows; + return Object.freeze({ + executions: Object.freeze( + rows.map((row) => + normalizeApprovedActionExecutionSnapshot({ + dispatch: parseDispatch(row), + execution: parseExecution(row), + }), + ), + ), + truncated, + }); + } catch (error) { + throw mappedError(error); + } + } + claimExecution( command: ClaimApprovedActionExecutionCommand, ): Promise { diff --git a/packages/ql3-cluster-postgres/src/entrypoints/packageExecutor.ts b/packages/ql3-cluster-postgres/src/entrypoints/packageExecutor.ts index 3d378196..458be7c3 100644 --- a/packages/ql3-cluster-postgres/src/entrypoints/packageExecutor.ts +++ b/packages/ql3-cluster-postgres/src/entrypoints/packageExecutor.ts @@ -41,6 +41,7 @@ export { export { PostgresPluginPackageMaterializedRevisionRepository } from '../plugin-package/installation/pluginPackageMaterializedRevisionRepository'; export { PostgresApprovalRequestRepository } from '../approved-action/approvalRequestRepository'; +export { PostgresApprovedActionExecutionRepository } from '../approved-action/approvedActionExecutionRepository'; export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository'; export { PostgresPluginPackageSecretBindingRepository } from '../plugin-package/installation/pluginPackageSecretBindingRepository'; export { PostgresPluginPackageSecretBindingActivationPrerequisite } from '../plugin-package/secret-binding/pluginPackageSecretBindingActivationPrerequisite'; diff --git a/packages/ql3-cluster-postgres/test/entrypoints.test.cjs b/packages/ql3-cluster-postgres/test/entrypoints.test.cjs index 8e717d78..7994ca11 100644 --- a/packages/ql3-cluster-postgres/test/entrypoints.test.cjs +++ b/packages/ql3-cluster-postgres/test/entrypoints.test.cjs @@ -367,11 +367,16 @@ test('Approved Action execution authority is isolated behind its explicit subpat const runtime = require('@qinglong/cluster-postgres/runtime'); const admin = require('@qinglong/cluster-postgres/admin'); const ingress = require('@qinglong/cluster-postgres/worker-ingress'); + const packageExecutor = require('@qinglong/cluster-postgres/package-executor'); const authority = require('@qinglong/cluster-postgres/approved-action-execution'); assert.equal(root.PostgresApprovedActionExecutionRepository, undefined); assert.equal(runtime.PostgresApprovedActionExecutionRepository, undefined); assert.equal(admin.PostgresApprovedActionExecutionRepository, undefined); assert.equal(ingress.PostgresApprovedActionExecutionRepository, undefined); + assert.equal( + packageExecutor.PostgresApprovedActionExecutionRepository, + authority.PostgresApprovedActionExecutionRepository, + ); assert.equal( typeof authority.PostgresApprovedActionExecutionRepository, 'function', diff --git a/packages/ql3-cluster-postgres/test/postgres.integration.test.cjs b/packages/ql3-cluster-postgres/test/postgres.integration.test.cjs index 3bbef27f..88811a45 100644 --- a/packages/ql3-cluster-postgres/test/postgres.integration.test.cjs +++ b/packages/ql3-cluster-postgres/test/postgres.integration.test.cjs @@ -3581,6 +3581,22 @@ if (!migrationConnectionString) { const executions = new PostgresApprovedActionExecutionRepository( executorDatabase.pool, ); + const pendingSecretActions = + await executions.listReconciliableExecutions({ + nowMs: claimedAtMs, + limit: 1, + actionTypes: [consumed.dispatch.action.actionType], + }); + assert.equal(pendingSecretActions.truncated, false); + assert.equal(pendingSecretActions.executions.length, 1); + assert.equal( + pendingSecretActions.executions[0].dispatch.id, + consumed.dispatch.id, + ); + assert.equal( + pendingSecretActions.executions[0].execution.status, + 'pending', + ); const claimed = await executions.claimExecution({ dispatchId: consumed.dispatch.id, owner: 'package_admission_dispatcher', @@ -3598,6 +3614,22 @@ if (!migrationConnectionString) { expectedVersion: claimed.snapshot.execution.version, startedAtMs: admittedAtMs, }); + const executingSecretActions = + await executions.listReconciliableExecutions({ + nowMs: admittedAtMs, + limit: 1, + actionTypes: [consumed.dispatch.action.actionType], + }); + assert.equal(executingSecretActions.truncated, false); + assert.equal(executingSecretActions.executions.length, 1); + assert.equal( + executingSecretActions.executions[0].dispatch.id, + consumed.dispatch.id, + ); + assert.equal( + executingSecretActions.executions[0].execution.status, + 'executing', + ); const lock = resolvePluginPackageInstallProposal( proposal, consumed.dispatch, diff --git a/scripts/ql3-cluster-deployment-audit.cjs b/scripts/ql3-cluster-deployment-audit.cjs index af82bd86..534d1465 100644 --- a/scripts/ql3-cluster-deployment-audit.cjs +++ b/scripts/ql3-cluster-deployment-audit.cjs @@ -3185,6 +3185,12 @@ function assertPluginPackageExecutorDeployment(readFile, root, findings) { const base = path.join(operationRoot, 'base'); const expectedResources = [ 'service-account.yaml', + 'secret-action-service-account.yaml', + 'secret-action-admission-config.yaml', + 'role.yaml', + 'role-binding.yaml', + 'validating-admission-policy.yaml', + 'validating-admission-policy-binding.yaml', 'cron-job.yaml', 'network-policy.yaml', ]; @@ -3196,6 +3202,25 @@ function assertPluginPackageExecutorDeployment(readFile, root, findings) { ); const name = 'ql3-plugin-package-executor'; const serviceAccount = namedResource(resources, 'ServiceAccount', name); + const actionName = 'ql3-plugin-package-secret-action'; + const actionServiceAccount = namedResource( + resources, + 'ServiceAccount', + actionName, + ); + const admissionConfig = namedResource(resources, 'ConfigMap', actionName + '-admission'); + const role = namedResource(resources, 'Role', name); + const roleBinding = namedResource(resources, 'RoleBinding', name); + const admissionPolicy = namedResource( + resources, + 'ValidatingAdmissionPolicy', + actionName, + ); + const admissionBinding = namedResource( + resources, + 'ValidatingAdmissionPolicyBinding', + actionName, + ); const cronJob = namedResource(resources, 'CronJob', name); const networkPolicy = namedResource(resources, 'NetworkPolicy', name); const pod = objectAt(cronJob, [ @@ -3211,10 +3236,11 @@ function assertPluginPackageExecutorDeployment(readFile, root, findings) { JSON.stringify(kustomization?.resources) !== JSON.stringify(expectedResources) || serviceAccount?.automountServiceAccountToken !== false || + actionServiceAccount?.automountServiceAccountToken !== false || cronJob?.spec?.schedule !== '*/2 * * * *' || cronJob?.spec?.concurrencyPolicy !== 'Forbid' || pod?.serviceAccountName !== name || - pod?.automountServiceAccountToken !== false || + pod?.automountServiceAccountToken !== true || pod?.restartPolicy !== 'Never' || pod?.securityContext?.runAsNonRoot !== true || pod?.securityContext?.seccompProfile?.type !== 'RuntimeDefault' || @@ -3227,7 +3253,71 @@ function assertPluginPackageExecutorDeployment(readFile, root, findings) { findings.push( finding( 'QL3_CLUSTER_PLUGIN_EXECUTOR_LIFECYCLE', - 'Plugin Package executor must remain a bounded, tokenless, non-overlapping CronJob', + 'Plugin Package executor must remain bounded and non-overlapping, with Kubernetes authority isolated from tokenless Secret action Jobs', + ), + ); + } + const expectedRoleRules = [ + { + apiGroups: ['batch'], + resources: ['jobs'], + verbs: ['create', 'get'], + }, + { + apiGroups: [''], + resources: ['configmaps'], + resourceNames: ['ql3-plugin-package-secret-action-admission'], + verbs: ['get'], + }, + ]; + const admissionExpressions = (admissionPolicy?.spec?.validations ?? []) + .map((validation) => validation?.expression) + .join('\n'); + if ( + JSON.stringify(role?.rules) !== JSON.stringify(expectedRoleRules) || + JSON.stringify(roleBinding?.subjects) !== + JSON.stringify([ + { + kind: 'ServiceAccount', + name, + namespace: 'qinglong3-system', + }, + ]) || + JSON.stringify(roleBinding?.roleRef) !== + JSON.stringify({ + apiGroup: 'rbac.authorization.k8s.io', + kind: 'Role', + name, + }) || + admissionPolicy?.spec?.failurePolicy !== 'Fail' || + admissionPolicy?.spec?.paramKind?.apiVersion !== 'v1' || + admissionPolicy?.spec?.paramKind?.kind !== 'ConfigMap' || + admissionPolicy?.spec?.matchConstraints?.resourceRules?.[0]?.operations?.[0] !== + 'CREATE' || + admissionPolicy?.spec?.matchConstraints?.resourceRules?.[0]?.resources?.[0] !== + 'jobs' || + admissionPolicy?.spec?.matchConditions?.[0]?.expression !== + "request.userInfo.username == 'system:serviceaccount:qinglong3-system:ql3-plugin-package-executor'" || + !admissionExpressions.includes("variables.executor.image == params.data.image") || + !admissionExpressions.includes('variables.pod.automountServiceAccountToken == false') || + !admissionExpressions.includes('variables.values.secret.items.all') || + admissionBinding?.spec?.policyName !== actionName || + admissionBinding?.spec?.paramRef?.name !== actionName + '-admission' || + admissionBinding?.spec?.paramRef?.namespace !== 'qinglong3-system' || + admissionBinding?.spec?.paramRef?.parameterNotFoundAction !== 'Deny' || + JSON.stringify(admissionBinding?.spec?.validationActions) !== + JSON.stringify(['Deny']) || + admissionConfig?.data?.serviceAccountName !== actionName || + admissionConfig?.data?.sourceSecretName !== + 'ql3-cluster-plugin-package-values' || + !String(admissionConfig?.data?.image).endsWith( + '@sha256:0000000000000000000000000000000000000000000000000000000000000000', + ) + ) { + findings.push( + finding( + 'QL3_CLUSTER_PLUGIN_EXECUTOR_KUBERNETES_AUTHORITY', + 'Plugin Package controller must have create/get-only Job RBAC behind deny-on-error exact-contract admission and a tokenless action identity', ), ); } @@ -3254,6 +3344,9 @@ function assertPluginPackageExecutorDeployment(readFile, root, findings) { ['QL3_PLUGIN_PACKAGE_EXECUTOR_LEASE_DURATION_MS', '600000'], ['QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_PAGE_SIZE', '16'], ['QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_MAX_PAGES', '16'], + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_ENABLED', 'true'], + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_LIMIT', '8'], + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_NAMESPACE', 'qinglong3-system'], ['QL3_POSTGRES_TLS_MODE', 'verify-full'], [ 'QL3_POSTGRES_TLS_CA_FILE', @@ -3270,6 +3363,50 @@ function assertPluginPackageExecutorDeployment(readFile, root, findings) { ); } } + for (const [environmentName, key] of [ + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_SERVICE_ACCOUNT', 'serviceAccountName'], + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_SOURCE_SECRET', 'sourceSecretName'], + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_IMAGE', 'image'], + [ + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_CA_SECRET', + 'postgresCaSecretName', + ], + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_CA_KEY', 'postgresCaKey'], + [ + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_SERVERNAME', + 'postgresServerName', + ], + [ + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_SECRET', + 'postgresUrlSecretName', + ], + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_URL_KEY', 'postgresUrlSecretKey'], + [ + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_AUTH_SECRET', + 'postgresAuthSecretName', + ], + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_HOST', 'postgresHost'], + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_PORT', 'postgresPort'], + ['QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_DATABASE', 'postgresDatabase'], + [ + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_USERNAME_KEY', + 'postgresUsernameKey', + ], + [ + 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_POSTGRES_PASSWORD_KEY', + 'postgresPasswordKey', + ], + ]) { + const ref = env.get(environmentName)?.valueFrom?.configMapKeyRef; + if (ref?.name !== actionName + '-admission' || ref?.key !== key) { + findings.push( + finding( + 'QL3_CLUSTER_PLUGIN_EXECUTOR_SECRET_ACTION_CONFIG', + `${environmentName} must come from the admission-bound ConfigMap`, + ), + ); + } + } const executorUrl = env.get('QL3_POSTGRES_PACKAGE_EXECUTOR_URL')?.valueFrom ?.secretKeyRef; if ( @@ -3348,6 +3485,38 @@ function assertPluginPackageExecutorDeployment(readFile, root, findings) { ), ); } + const apiServerEgressExample = yaml.load( + readFile( + path.join(operationRoot, 'api-server-egress-patch.example.yaml'), + 'utf8', + ), + ); + if ( + JSON.stringify(apiServerEgressExample) !== + JSON.stringify([ + { + op: 'add', + path: '/spec/egress/-', + value: { + to: [ + { + ipBlock: { + cidr: 'REPLACE_WITH_API_SERVER_CIDR', + }, + }, + ], + ports: [{ protocol: 'TCP', port: 443 }], + }, + }, + ]) + ) { + findings.push( + finding( + 'QL3_CLUSTER_PLUGIN_EXECUTOR_API_EGRESS', + 'Plugin Package controller API egress example must require one exact private-overlay CIDR and TCP 443', + ), + ); + } const cloudNativeRoot = path.join(operationRoot, 'cloudnative-pg'); const cloudNative = yaml.load( readFile(path.join(cloudNativeRoot, 'kustomization.yaml'), 'utf8'), @@ -3364,12 +3533,21 @@ function assertPluginPackageExecutorDeployment(readFile, root, findings) { ) : undefined; const cloudNativeEnv = environmentByName({ env: environmentPatch?.value }); + const secretActionConfigPatch = yaml.load( + readFile( + path.join(cloudNativeRoot, 'secret-action-admission-config-patch.yaml'), + 'utf8', + ), + ); + const secretActionConfig = secretActionConfigPatch?.[0]?.value; const unreleasedDigest = 'sha256:0000000000000000000000000000000000000000000000000000000000000000'; if ( JSON.stringify(cloudNative?.resources) !== JSON.stringify(['../base']) || cloudNative?.patches?.[0]?.path !== 'cron-job-patch.yaml' || cloudNative?.patches?.[1]?.path !== 'network-policy-patch.yaml' || + cloudNative?.patches?.[2]?.path !== + 'secret-action-admission-config-patch.yaml' || JSON.stringify(cloudNative?.images) !== JSON.stringify([ { @@ -3387,6 +3565,19 @@ function assertPluginPackageExecutorDeployment(readFile, root, findings) { ?.secretKeyRef?.name !== 'ql3-postgres-package-executor-auth' || cloudNativeEnv.get('QL3_POSTGRES_PACKAGE_EXECUTOR_PASSWORD')?.valueFrom ?.secretKeyRef?.name !== 'ql3-postgres-package-executor-auth' || + secretActionConfigPatch?.[0]?.op !== 'replace' || + secretActionConfigPatch?.[0]?.path !== '/data' || + secretActionConfig?.postgresConnectionMode !== 'fields' || + secretActionConfig?.postgresCaSecretName !== 'ql3-postgres-ca' || + secretActionConfig?.postgresServerName !== + 'ql3-postgres-rw.qinglong3-system.svc' || + secretActionConfig?.postgresAuthSecretName !== + 'ql3-postgres-package-executor-auth' || + secretActionConfig?.postgresHost !== + 'ql3-postgres-rw.qinglong3-system.svc' || + secretActionConfig?.postgresPort !== '5432' || + secretActionConfig?.postgresDatabase !== 'qinglong' || + !String(secretActionConfig?.image).endsWith(`@${unreleasedDigest}`) || !patch.some( (operation) => operation?.path === diff --git a/test/back/ql3ClusterDeploymentAudit.test.cjs b/test/back/ql3ClusterDeploymentAudit.test.cjs index 0cf0ad8c..a1555cfc 100644 --- a/test/back/ql3ClusterDeploymentAudit.test.cjs +++ b/test/back/ql3ClusterDeploymentAudit.test.cjs @@ -64,6 +64,56 @@ test('requires every Cluster Admin Kubernetes workload to override the image com ); }); +test('rejects widened Plugin Package Secret action Kubernetes authority', () => { + const widenedRole = auditClusterDeployment({ + root: ROOT, + readFile: intercept( + 'deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/role.yaml', + (source) => source.replace(' - get\n', ' - get\n - list\n'), + ), + }); + assert.equal(widenedRole.compatible, false); + assert.equal( + widenedRole.findings.some( + ({ code }) => code === 'QL3_CLUSTER_PLUGIN_EXECUTOR_KUBERNETES_AUTHORITY', + ), + true, + ); + + const ignoredAdmissionFailure = auditClusterDeployment({ + root: ROOT, + readFile: intercept( + 'deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/validating-admission-policy.yaml', + (source) => source.replace('failurePolicy: Fail', 'failurePolicy: Ignore'), + ), + }); + assert.equal(ignoredAdmissionFailure.compatible, false); + assert.equal( + ignoredAdmissionFailure.findings.some( + ({ code }) => code === 'QL3_CLUSTER_PLUGIN_EXECUTOR_KUBERNETES_AUTHORITY', + ), + true, + ); + + const actionToken = auditClusterDeployment({ + root: ROOT, + readFile: intercept( + 'deploy/kubernetes/ql3-cluster/operations/plugin-package-executor/base/secret-action-service-account.yaml', + (source) => source.replace( + 'automountServiceAccountToken: false', + 'automountServiceAccountToken: true', + ), + ), + }); + assert.equal(actionToken.compatible, false); + assert.equal( + actionToken.findings.some( + ({ code }) => code === 'QL3_CLUSTER_PLUGIN_EXECUTOR_LIFECYCLE', + ), + true, + ); +}); + test('requires the bounded Cluster product facade and image entrypoint', () => { const missingBinary = auditClusterDeployment({ root: ROOT, diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index bdd04a72..690c460c 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -340,10 +340,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: clusterAdmin.rootSourceFileRoles, }, { - sourceFiles: 108, + sourceFiles: 109, rootSourceFiles: 1, rootSourceLines: 61, - nestedSourceFiles: 107, + nestedSourceFiles: 108, rootSourceFileRoles: { 'modelInvocationMigrationCli.ts': 'binary_entry', },