feat(ql3): project plugin secrets into kubernetes

This commit is contained in:
whyour
2026-08-14 00:34:49 +08:00
parent d41c821ce0
commit 9d7431c2cd
13 changed files with 1437 additions and 73 deletions
@@ -0,0 +1,314 @@
#!/usr/bin/env node
'use strict';
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const {
K3sDockerLiveFixture,
waitFor,
} = require('./lib/ql3-k3s-docker-live.cjs');
const ROOT = path.resolve(__dirname, '..');
const ACTOR_FILE = path.join(
__dirname,
'ql3-plugin-package-kubernetes-live-actor.cjs',
);
const RESULT_SCHEMA = 'qinglong/plugin-package-kubernetes-live-actor-result@v1';
const NAMESPACE = 'ql3-plugin-package-live';
const SERVICE_ACCOUNT = 'ql3-plugin-package-recovery-live';
const IMAGE = 'qinglong3-cluster-admin:ql3-k3s-kubernetes-live';
function run(binary, args, options = {}) {
const result = spawnSync(binary, args, {
cwd: ROOT,
env: process.env,
encoding: 'utf8',
maxBuffer: 64 * 1024 * 1024,
stdio: options.capture ? ['ignore', 'pipe', 'pipe'] : 'inherit',
});
if (result.error) throw result.error;
if (result.status !== 0) {
throw new Error(
`${path.basename(binary)} failed with ${String(result.status)}: ` +
`${result.stderr || result.stdout || ''}`,
);
}
return result;
}
function roleDocuments() {
return [
{
apiVersion: 'v1',
kind: 'Namespace',
metadata: { name: NAMESPACE },
},
{
apiVersion: 'v1',
kind: 'ServiceAccount',
metadata: { name: SERVICE_ACCOUNT, namespace: NAMESPACE },
automountServiceAccountToken: false,
},
{
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'Role',
metadata: { name: SERVICE_ACCOUNT, namespace: NAMESPACE },
rules: [
{
apiGroups: [''],
resources: ['configmaps'],
verbs: ['get', 'create', 'update'],
},
],
},
{
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'RoleBinding',
metadata: { name: SERVICE_ACCOUNT, namespace: NAMESPACE },
roleRef: {
apiGroup: 'rbac.authorization.k8s.io',
kind: 'Role',
name: SERVICE_ACCOUNT,
},
subjects: [
{
kind: 'ServiceAccount',
name: SERVICE_ACCOUNT,
namespace: NAMESPACE,
},
],
},
];
}
function actorPod(actor) {
return {
apiVersion: 'v1',
kind: 'Pod',
metadata: {
name: `ql3-plugin-package-live-${actor}`,
namespace: NAMESPACE,
labels: {
'app.kubernetes.io/name': 'ql3-plugin-package-live',
'qinglong.io/live-actor': actor,
},
},
spec: {
serviceAccountName: SERVICE_ACCOUNT,
automountServiceAccountToken: true,
restartPolicy: 'Never',
securityContext: {
runAsNonRoot: true,
runAsUser: 10001,
runAsGroup: 10001,
fsGroup: 10001,
seccompProfile: { type: 'RuntimeDefault' },
},
containers: [
{
name: 'recovery',
image: IMAGE,
imagePullPolicy: 'Never',
command: ['node', '/opt/ql3-live/actor.cjs'],
env: [
{ name: 'NODE_PATH', value: '/opt/qinglong/node_modules' },
{ name: 'QL3_LIVE_NAMESPACE', value: NAMESPACE },
{ name: 'QL3_LIVE_ACTOR', value: actor },
],
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: { cpu: '25m', memory: '64Mi' },
limits: { cpu: '500m', memory: '256Mi' },
},
volumeMounts: [
{ name: 'actor', mountPath: '/opt/ql3-live', readOnly: true },
],
},
],
volumes: [
{
name: 'actor',
configMap: {
name: 'ql3-plugin-package-live-actor',
defaultMode: 0o444,
items: [{ key: 'actor.cjs', path: 'actor.cjs' }],
},
},
],
},
};
}
async function actorResult(fixture, actor) {
const pod = `ql3-plugin-package-live-${actor}`;
const observed = await waitFor(`${pod} termination result`, 60_000, () => {
const value = fixture.kubectlJson(['-n', NAMESPACE, 'get', 'pod', pod]);
const terminated = value.status?.containerStatuses?.find(
(container) => container.name === 'recovery',
)?.state?.terminated;
if (!terminated) {
return { ready: false, fact: value.status?.phase ?? 'unknown' };
}
if (terminated.exitCode !== 0) {
throw new Error(
`${pod} exited ${terminated.exitCode}: ${terminated.message ?? ''}`,
);
}
return { ready: true, value: terminated.message };
});
const value = JSON.parse(observed.value);
assert.equal(value.schema, RESULT_SCHEMA);
assert.equal(value.actor, actor);
assert.equal(value.error, undefined);
return value;
}
async function main() {
if (process.env.QL3_PLUGIN_PACKAGE_K3S_LIVE !== '1') {
throw new Error('refusing to run without QL3_PLUGIN_PACKAGE_K3S_LIVE=1');
}
const fixture = new K3sDockerLiveFixture({
prefix: 'ql3-plugin-v3-live',
kubectl:
process.env.QL3_KUBECTL_BIN ??
'/Applications/Docker.app/Contents/Resources/bin/kubectl',
});
const startedAt = Date.now();
try {
run('docker', [
'build',
'--file',
'deploy/containers/ql3-cluster-admin/Dockerfile',
'--tag',
IMAGE,
'--build-arg',
`SOURCE_REVISION=${process.env.GITHUB_SHA ?? 'local-k3s-live-contract'}`,
'.',
]);
const nodes = await fixture.start();
fixture.loadImage(IMAGE, 'plugin-package-v3-admin.tar');
for (const document of roleDocuments()) fixture.apply(document);
fixture.apply({
apiVersion: 'v1',
kind: 'ConfigMap',
metadata: {
name: 'ql3-plugin-package-live-actor',
namespace: NAMESPACE,
},
data: { 'actor.cjs': fs.readFileSync(ACTOR_FILE, 'utf8') },
});
fixture.apply(actorPod('a'));
fixture.apply(actorPod('b'));
await waitFor('two v3 projection CAS actors', 180_000, () => {
const pods = fixture.kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
'app.kubernetes.io/name=ql3-plugin-package-live',
]).items;
const failed = pods.find((pod) => pod.status.phase === 'Failed');
if (failed) {
const logs = fixture.kubectl(
['-n', NAMESPACE, 'logs', failed.metadata.name],
{ capture: true, quiet: true, allowFailure: true },
);
throw new Error(logs.stderr || logs.stdout);
}
return pods.length === 2 &&
pods.every((pod) => pod.status.phase === 'Succeeded')
? { ready: true, value: pods }
: {
ready: false,
fact: pods.map((pod) => `${pod.metadata.name}:${pod.status.phase}`),
};
});
const actors = await Promise.all([
actorResult(fixture, 'a'),
actorResult(fixture, 'b'),
]);
const winner = actors.find((actor) => actor.cas.status === 'fulfilled');
const loser = actors.find((actor) => actor.cas.status === 'conflict');
assert.ok(winner);
assert.ok(loser);
assert.equal(winner.final.pointerSchema.endsWith('@v3'), true);
assert.equal(winner.final.projectionItemCount, 1);
assert.equal(winner.final.projectedWorkloadVolume, true);
assert.equal(winner.final.projectionDigest, loser.final.projectionDigest);
assert.equal(
winner.final.transitionReceiptDigest,
loser.final.transitionReceiptDigest,
);
assert.deepEqual(winner.rbac, {
listConfigMaps: 403,
deleteConfigMap: 403,
readSecret: 403,
crossNamespaceRead: 403,
});
assert.deepEqual(loser.rbac, winner.rbac);
const pointers = fixture.kubectlJson([
'-n',
NAMESPACE,
'get',
'configmaps',
'-l',
'qinglong.io/plugin-package-active=v3',
]).items;
assert.equal(pointers.length, 1);
process.stdout.write(
`${JSON.stringify(
{
schemaVersion: 1,
fixture: 'qinglong/plugin-package-kubernetes-k3s-live@v1',
platform: {
kubernetesVersion: fixture.kubectlJson(['version']).serverVersion
.gitVersion,
nodeCount: nodes.length,
},
result: {
fulfilled: 1,
conflicts: 1,
pointerSchema: winner.final.pointerSchema,
projectionDigest: winner.final.projectionDigest,
transitionReceiptDigest: winner.final.transitionReceiptDigest,
exactProjectionItems: winner.final.projectionItemCount,
secretApiReadDenied: winner.rbac.readSecret === 403,
activePointers: pointers.length,
},
gates: {
realThreeNodeKubernetes: true,
resourceVersionSingleWinner: true,
v3TransitionReceiptBound: true,
exactSecretProjectionRendered: true,
secretApiReadDenied: true,
passed: true,
},
elapsedMs: Date.now() - startedAt,
},
null,
2,
)}\n`,
);
} finally {
await fixture.cleanup();
}
}
main().catch((error) => {
process.stderr.write(
`QL3 Plugin Package K3s v3 live contract failed: ${
error instanceof Error ? error.stack || error.message : String(error)
}\n`,
);
process.exitCode = 1;
});
@@ -14,8 +14,23 @@ const {
const {
createPluginPackageResourceGeneration,
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
const {
createPluginPackageSecretBindingTarget,
} = require('@qinglong/runtime-core/plugin-package-secret-binding');
const {
createPluginPackageSecretBindingTransitionPlan,
} = require('@qinglong/runtime-core/plugin-package-secret-binding-transition-plan');
const {
createPluginPackageSecretBindingFromTransitionPlan,
createPluginPackageSecretBindingTransitionReceipt,
} = require('@qinglong/runtime-core/plugin-package-secret-binding-transition-receipt');
const {
secretProjectionFileName,
} = require('@qinglong/runtime-core/secret-projection');
const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
const {
PluginPackageKubernetesActivationPublisher,
pluginPackageKubernetesProjectedSecretWorkloadVolume,
} = require('@qinglong/cluster-admin/plugin-package-kubernetes-activation');
const TOKEN_FILE = '/var/run/secrets/kubernetes.io/serviceaccount/token';
@@ -119,6 +134,83 @@ function activationIntent(overrides = {}) {
});
}
function manifest(version, secrets) {
return Object.freeze({
apiVersion: 'qinglong.io/v1alpha1',
kind: 'Package',
metadata: Object.freeze({
name: 'live-cas-package',
displayName: 'Live CAS Package',
version,
description: 'Real Kubernetes Secret projection CAS gate',
license: 'Apache-2.0',
}),
spec: Object.freeze({
compatibility: Object.freeze({
qinglong: '>=3.0.0-0 <4.0.0',
architectures: Object.freeze(['arm64']),
deploymentProfiles: Object.freeze(['cluster-control']),
}),
runtimes: Object.freeze([]),
resources: Object.freeze({
memory: Object.freeze({ recommended: '16Mi' }),
disk: Object.freeze({ install: '4Mi', working: '16Mi' }),
}),
permissions: Object.freeze({
network: Object.freeze({ allowedHosts: Object.freeze([]) }),
secrets: Object.freeze(secrets),
tools: Object.freeze(secrets.length === 0 ? [] : ['secret.use']),
}),
contents: Object.freeze({
tasks: Object.freeze(['tasks/live.yaml']),
workflows: Object.freeze([]),
prompts: Object.freeze([]),
tools: Object.freeze([]),
}),
}),
});
}
function transitionEvidence(initial, candidate) {
const secretRef = createSecretRef({
projectId: 'default',
name: `live-token-${ACTOR}`,
version: 2,
});
const previousManifest = manifest('1.0.0', []);
const nextManifest = manifest('2.0.0', [
Object.freeze({ name: 'TOKEN', required: true }),
]);
const plan = createPluginPackageSecretBindingTransitionPlan({
previousTarget: createPluginPackageSecretBindingTarget(
initial.resourceGeneration,
previousManifest,
),
previousBinding: null,
previousAttemptGeneration: 1,
nextGeneration: candidate.resourceGeneration,
nextManifest,
assignments: [Object.freeze({ name: 'TOKEN', secretRef })],
plannedAtMs: 100,
});
const binding = createPluginPackageSecretBindingFromTransitionPlan(
plan,
'approved-action-execution',
candidate.stageEvidenceDigest,
200,
);
const receipt = createPluginPackageSecretBindingTransitionReceipt({
transitionPlan: plan,
authority: Object.freeze({
kind: 'approved-action-execution',
evidenceDigest: candidate.stageEvidenceDigest,
}),
binding,
committedAtMs: 200,
});
return Object.freeze({ secretRef, binding, receipt });
}
function exactEvidence(intent) {
return Object.freeze({
lockDigest: intent.lockDigest,
@@ -222,29 +314,32 @@ async function main() {
};
let nowCalls = 0;
const publisher = new PluginPackageKubernetesActivationPublisher(
activationApi,
{ verify: async (intent) => exactEvidence(intent) },
{
clusterIdentity: 'ql3-plugin-package-live-cluster',
namespace: NAMESPACE,
now() {
nowCalls += 1;
return (ACTOR === 'a' ? 1_000 : 2_000) + nowCalls;
const createPublisher = (secretProjection) =>
new PluginPackageKubernetesActivationPublisher(
activationApi,
{ verify: async (intent) => exactEvidence(intent) },
{
clusterIdentity: 'ql3-plugin-package-live-cluster',
namespace: NAMESPACE,
now() {
nowCalls += 1;
return (ACTOR === 'a' ? 1_000 : 2_000) + nowCalls;
},
...(secretProjection === undefined ? {} : { secretProjection }),
},
},
);
);
const initial = activationIntent();
const initialPublisher = createPublisher();
let responseLoss = null;
if (ACTOR === 'a') {
await assert.rejects(
publisher.publish(initial),
initialPublisher.publish(initial),
PluginPackageActivationUnavailableError,
);
const observation = await publisher.inspect(initial);
const observation = await initialPublisher.inspect(initial);
assert.equal(observation.status, 'published');
const replay = await publisher.publish(initial);
const replay = await initialPublisher.publish(initial);
assert.deepEqual(replay, observation.receipt);
assert.equal(createCalls, 1);
assert.equal(nowCalls, 1);
@@ -258,7 +353,7 @@ async function main() {
});
} else {
await waitFor('initial active pointer', async () => {
const observation = await publisher.inspect(initial);
const observation = await initialPublisher.inspect(initial);
return {
ready: observation.status === 'published',
fact: observation.status,
@@ -272,6 +367,12 @@ async function main() {
targetGeneration: 2,
previousActiveLockDigest: INITIAL_LOCK_DIGEST,
});
const transition = transitionEvidence(initial, candidate);
const publisher = createPublisher({
sourceSecretName: 'ql3-cluster-plugin-package-values',
bindings: { find: async () => transition.binding },
transitions: { find: async () => transition.receipt },
});
let outcome;
try {
const receipt = await publisher.publish(candidate);
@@ -310,6 +411,25 @@ async function main() {
name: targetName,
});
const finalPointer = JSON.parse(finalConfigMap.data['active.json']);
assert.equal(finalPointer.schema.endsWith('@v3'), true);
assert.equal(finalPointer.secretProjection.defaultMode, 0o440);
assert.match(
finalPointer.secretProjection.transitionReceiptDigest,
/^[0-9a-f]{64}$/,
);
const projectionPath = secretProjectionFileName(transition.secretRef);
const winnerUsesThisActor =
finalPointer.intent.lockDigest === candidate.lockDigest;
if (winnerUsesThisActor) {
assert.deepEqual(finalPointer.secretProjection.items, [
{ key: projectionPath, path: projectionPath },
]);
}
const workloadVolume = pluginPackageKubernetesProjectedSecretWorkloadVolume(
finalPointer.secretProjection,
);
assert.equal(workloadVolume.volume.secret.optional, false);
assert.equal(workloadVolume.volume.secret.defaultMode, 0o440);
const rbac = Object.freeze({
listConfigMaps: await expectForbidden(() =>
@@ -335,25 +455,34 @@ async function main() {
),
});
process.stdout.write(
`${JSON.stringify({
schema: RESULT_SCHEMA,
actor: ACTOR,
serviceAccountTokenMounted: true,
responseLoss,
cas: {
...outcome,
attemptedResourceVersion: replaceResourceVersion,
replaceCalls,
},
final: {
resourceVersion: finalConfigMap.metadata.resourceVersion,
lockDigest: finalPointer.intent.lockDigest,
generation: finalPointer.receipt.generation,
},
rbac,
})}\n`,
);
const result = JSON.stringify({
schema: RESULT_SCHEMA,
actor: ACTOR,
serviceAccountTokenMounted: true,
responseLoss,
cas: {
...outcome,
attemptedResourceVersion: replaceResourceVersion,
replaceCalls,
},
final: {
resourceVersion: finalConfigMap.metadata.resourceVersion,
lockDigest: finalPointer.intent.lockDigest,
generation: finalPointer.receipt.generation,
pointerSchema: finalPointer.schema,
projectionDigest: finalPointer.secretProjection.projectionDigest,
transitionReceiptDigest:
finalPointer.secretProjection.transitionReceiptDigest,
projectionItemCount: finalPointer.secretProjection.items.length,
projectedWorkloadVolume: workloadVolume !== null,
},
rbac,
});
fs.writeFileSync('/dev/termination-log', result, {
encoding: 'utf8',
flag: 'w',
});
process.stdout.write(`${result}\n`);
}
main().catch((error) => {
@@ -505,6 +505,17 @@ async function main() {
);
assert.equal(winner.final.generation, 2);
assert.equal(winner.final.resourceVersion, loser.final.resourceVersion);
assert.equal(winner.final.pointerSchema, loser.final.pointerSchema);
assert.equal(winner.final.projectionDigest, loser.final.projectionDigest);
assert.equal(
winner.final.transitionReceiptDigest,
loser.final.transitionReceiptDigest,
);
assert.match(winner.final.projectionDigest, /^[0-9a-f]{64}$/);
assert.match(winner.final.transitionReceiptDigest, /^[0-9a-f]{64}$/);
assert.equal(winner.final.pointerSchema.endsWith('@v3'), true);
assert.equal(winner.final.projectionItemCount, 1);
assert.equal(winner.final.projectedWorkloadVolume, true);
const activePointers = kubectlJson([
'-n',
@@ -512,7 +523,7 @@ async function main() {
'get',
'configmaps',
'-l',
'qinglong.io/plugin-package-active=v2',
'qinglong.io/plugin-package-active=v3',
]).items;
assert.equal(activePointers.length, 1);
const barriers = kubectlJson([
@@ -582,6 +593,15 @@ async function main() {
winner: winner.actor,
activePointers: activePointers.length,
},
secretProjection: {
schema: winner.final.pointerSchema,
projectionDigest: winner.final.projectionDigest,
transitionReceiptDigest: winner.final.transitionReceiptDigest,
itemCount: winner.final.projectionItemCount,
defaultMode: 0o440,
workloadVolumeRendered: winner.final.projectedWorkloadVolume,
secretApiReadRequired: false,
},
rbac,
sideEffects: {
activePointers: activePointers.length,
@@ -599,6 +619,9 @@ async function main() {
concurrentReplacementSingleWinner: true,
loserObservedConflict: true,
finalPointerExactlyOne: true,
transitionReceiptBoundToV3Pointer: true,
exactSecretProjectionItemPublished: true,
projectedWorkloadVolumeUses0440: true,
configMapGetCreateUpdateAllowed: true,
configMapListDeleteDenied: true,
secretReadCreateDenied: true,
@@ -609,7 +632,9 @@ async function main() {
limitations: [
'response loss is injected after an API-confirmed create at the Kubernetes client boundary, not by dropping raw network packets',
'single-control-plane Kind proves API-server resourceVersion and RBAC semantics, not Kubernetes control-plane HA',
'the gate isolates ConfigMap publication authority and does not exercise PostgreSQL or OCI registry recovery',
'the gate uses in-memory content-blind binding/transition sources; durable PostgreSQL transition authority is proven independently',
'the gate renders the exact Secret volume source but deliberately does not create or read Secret material',
'the gate isolates ConfigMap publication authority and does not exercise OCI registry recovery',
],
},
null,