test(ql3): prove kubernetes secret revoke rollout

This commit is contained in:
whyour
2026-08-14 01:08:18 +08:00
parent 9d7431c2cd
commit 53c150c4ec
4 changed files with 539 additions and 57 deletions
@@ -21,6 +21,11 @@ const RESULT_SCHEMA = 'qinglong/plugin-package-kubernetes-live-actor-result@v1';
const NAMESPACE = 'ql3-plugin-package-live';
const SERVICE_ACCOUNT = 'ql3-plugin-package-recovery-live';
const IMAGE = 'qinglong3-cluster-admin:ql3-k3s-kubernetes-live';
const WORKLOAD = 'ql3-plugin-package-workload-live';
const SECRET_ROOT = '/var/run/secrets/qinglong3/plugin-package-values';
const WORKLOAD_REPLICAS = 2;
const SECRET_MARKER = 'ql3-live-exact-projection';
let renderWorkloadVolume = null;
function run(binary, args, options = {}) {
const result = spawnSync(binary, args, {
@@ -171,10 +176,233 @@ async function actorResult(fixture, actor) {
return value;
}
function activePointer(fixture) {
const pointers = fixture.kubectlJson([
'-n',
NAMESPACE,
'get',
'configmaps',
'-l',
'qinglong.io/plugin-package-active=v3',
]).items;
assert.equal(pointers.length, 1);
const pointer = JSON.parse(pointers[0].data['active.json']);
assert.equal(
pointer.schema,
'qinglong/plugin-package-kubernetes-active-pointer@v3',
);
return Object.freeze({ configMap: pointers[0], pointer });
}
function sourceSecret(projection) {
assert.equal(projection.items.length, 1);
const projected = projection.items[0].key;
const decoy = projected === 'f'.repeat(64) ? 'e'.repeat(64) : 'f'.repeat(64);
return Object.freeze({
decoy,
document: {
apiVersion: 'v1',
kind: 'Secret',
type: 'Opaque',
immutable: true,
metadata: {
name: projection.sourceSecretName,
namespace: NAMESPACE,
labels: {
'app.kubernetes.io/managed-by': 'qinglong3-live-gate',
'qinglong.io/live-gate-role': 'projection-source',
},
},
data: {
[projected]: Buffer.from(SECRET_MARKER, 'utf8').toString('base64'),
[decoy]: Buffer.from('ql3-live-decoy', 'utf8').toString('base64'),
},
},
});
}
function workloadDeployment(active) {
const projection = active.pointer.secretProjection;
assert.equal(typeof renderWorkloadVolume, 'function');
const rendered = renderWorkloadVolume(projection);
const labels = {
'app.kubernetes.io/name': WORKLOAD,
'app.kubernetes.io/component': 'plugin-package-workload',
};
return {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: {
name: WORKLOAD,
namespace: NAMESPACE,
labels,
},
spec: {
replicas: WORKLOAD_REPLICAS,
revisionHistoryLimit: 2,
progressDeadlineSeconds: 60,
strategy: {
type: 'RollingUpdate',
rollingUpdate: { maxUnavailable: 0, maxSurge: 1 },
},
selector: { matchLabels: labels },
template: {
metadata: {
labels,
annotations: {
'qinglong.io/plugin-package-generation-digest':
active.pointer.intent.resourceGeneration.generationDigest,
'qinglong.io/plugin-package-lock-digest':
active.pointer.intent.lockDigest,
'qinglong.io/plugin-package-secret-projection-digest':
projection?.projectionDigest ?? 'none',
},
},
spec: {
automountServiceAccountToken: false,
securityContext: {
runAsNonRoot: true,
runAsUser: 10001,
runAsGroup: 10001,
fsGroup: 10001,
seccompProfile: { type: 'RuntimeDefault' },
},
affinity: {
podAntiAffinity: {
requiredDuringSchedulingIgnoredDuringExecution: [
{
labelSelector: { matchLabels: labels },
topologyKey: 'kubernetes.io/hostname',
},
],
},
},
containers: [
{
name: 'workload',
image: IMAGE,
imagePullPolicy: 'Never',
command: [
'node',
'-e',
'setInterval(() => {}, 2147483647)',
],
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: { cpu: '10m', memory: '32Mi' },
limits: { cpu: '250m', memory: '128Mi' },
},
...(rendered
? { volumeMounts: [rendered.volumeMount] }
: {}),
},
],
...(rendered ? { volumes: [rendered.volume] } : {}),
},
},
},
};
}
async function workloadReady(fixture, active, minimumGeneration = 1) {
const expectedLockDigest = active.pointer.intent.lockDigest;
const expectedProjectionDigest =
active.pointer.secretProjection?.projectionDigest ?? 'none';
const observed = await waitFor(
`Deployment/${WORKLOAD} rollout`,
120_000,
() => {
const deployment = fixture.kubectlJson([
'-n',
NAMESPACE,
'get',
'deployment',
WORKLOAD,
]);
const pods = fixture.kubectlJson([
'-n',
NAMESPACE,
'get',
'pods',
'-l',
`app.kubernetes.io/name=${WORKLOAD}`,
]).items;
const currentPods = pods.filter(
(pod) =>
pod.metadata?.annotations?.[
'qinglong.io/plugin-package-lock-digest'
] === expectedLockDigest &&
pod.metadata?.annotations?.[
'qinglong.io/plugin-package-secret-projection-digest'
] === expectedProjectionDigest &&
pod.status?.phase === 'Running' &&
pod.status?.conditions?.some(
(condition) =>
condition.type === 'Ready' && condition.status === 'True',
),
);
const generation = deployment.metadata?.generation ?? 0;
const ready =
generation >= minimumGeneration &&
deployment.status?.observedGeneration === generation &&
deployment.status?.updatedReplicas === WORKLOAD_REPLICAS &&
deployment.status?.readyReplicas === WORKLOAD_REPLICAS &&
deployment.status?.availableReplicas === WORKLOAD_REPLICAS &&
currentPods.length === WORKLOAD_REPLICAS &&
new Set(currentPods.map((pod) => pod.spec?.nodeName)).size ===
WORKLOAD_REPLICAS;
return ready
? { ready: true, value: { deployment, pods: currentPods } }
: {
ready: false,
fact: JSON.stringify({
generation,
observedGeneration: deployment.status?.observedGeneration,
updatedReplicas: deployment.status?.updatedReplicas,
readyReplicas: deployment.status?.readyReplicas,
availableReplicas: deployment.status?.availableReplicas,
currentPods: currentPods.length,
}),
};
},
);
return observed.value;
}
function inspectWorkloadPod(fixture, pod, expectedPath) {
const source = expectedPath
? `
const fs = require('node:fs');
const root = ${JSON.stringify(SECRET_ROOT)};
const expected = ${JSON.stringify(expectedPath)};
const files = fs.readdirSync(root).filter((name) => !name.startsWith('..')).sort();
const value = fs.readFileSync(root + '/' + expected, 'utf8');
const mode = fs.statSync(root + '/' + expected).mode & 0o777;
process.stdout.write(JSON.stringify({ files, valueMatches: value === ${JSON.stringify(SECRET_MARKER)}, mode }));
`
: `
const fs = require('node:fs');
process.stdout.write(JSON.stringify({ rootAbsent: !fs.existsSync(${JSON.stringify(SECRET_ROOT)}) }));
`;
return JSON.parse(
fixture.kubectl(
['-n', NAMESPACE, 'exec', pod.metadata.name, '--', 'node', '-e', source],
{ capture: true, quiet: true },
).stdout,
);
}
async function main() {
if (process.env.QL3_PLUGIN_PACKAGE_K3S_LIVE !== '1') {
throw new Error('refusing to run without QL3_PLUGIN_PACKAGE_K3S_LIVE=1');
}
({
pluginPackageKubernetesProjectedSecretWorkloadVolume: renderWorkloadVolume,
} = require('../packages/ql3-cluster-admin/dist/plugin-package/recovery/pluginPackageKubernetesActivation.js'));
const fixture = new K3sDockerLiveFixture({
prefix: 'ql3-plugin-v3-live',
kubectl:
@@ -256,15 +484,63 @@ async function main() {
crossNamespaceRead: 403,
});
assert.deepEqual(loser.rbac, winner.rbac);
const pointers = fixture.kubectlJson([
const rotated = activePointer(fixture);
const projectedPath = rotated.pointer.secretProjection.items[0].path;
const secret = sourceSecret(rotated.pointer.secretProjection);
fixture.apply(secret.document);
fixture.apply(workloadDeployment(rotated));
const mounted = await workloadReady(fixture, rotated);
const mountedInspection = mounted.pods.map((pod) =>
inspectWorkloadPod(fixture, pod, projectedPath),
);
for (const inspection of mountedInspection) {
assert.deepEqual(inspection.files, [projectedPath]);
assert.equal(inspection.valueMatches, true);
assert.equal(inspection.mode, 0o440);
assert.equal(inspection.files.includes(secret.decoy), false);
}
fixture.apply(actorPod('c'));
const revoker = await actorResult(fixture, 'c');
assert.equal(revoker.mode, 'revoke');
assert.equal(revoker.cas.status, 'fulfilled');
assert.equal(revoker.final.pointerSchema.endsWith('@v3'), true);
assert.equal(revoker.final.projectionItemCount, 0);
assert.equal(revoker.final.projectedWorkloadVolume, false);
assert.deepEqual(revoker.rbac, winner.rbac);
const revoked = activePointer(fixture);
assert.equal(revoked.pointer.secretProjection.items.length, 0);
assert.equal(
revoked.pointer.secretProjection.transitionReceiptDigest,
revoker.final.transitionReceiptDigest,
);
fixture.apply(workloadDeployment(revoked));
const unmounted = await workloadReady(
fixture,
revoked,
(mounted.deployment.metadata?.generation ?? 1) + 1,
);
assert.equal(
unmounted.pods.some((pod) =>
mounted.pods.some((old) => old.metadata.uid === pod.metadata.uid),
),
false,
);
const revokedInspection = unmounted.pods.map((pod) =>
inspectWorkloadPod(fixture, pod, null),
);
assert.equal(
revokedInspection.every((inspection) => inspection.rootAbsent === true),
true,
);
const retainedSource = fixture.kubectlJson([
'-n',
NAMESPACE,
'get',
'configmaps',
'-l',
'qinglong.io/plugin-package-active=v3',
]).items;
assert.equal(pointers.length, 1);
'secret',
rotated.pointer.secretProjection.sourceSecretName,
]);
assert.equal(Object.keys(retainedSource.data).length, 2);
process.stdout.write(
`${JSON.stringify(
{
@@ -283,7 +559,15 @@ async function main() {
transitionReceiptDigest: winner.final.transitionReceiptDigest,
exactProjectionItems: winner.final.projectionItemCount,
secretApiReadDenied: winner.rbac.readSecret === 403,
activePointers: pointers.length,
activePointers: 1,
workloadReplicas: WORKLOAD_REPLICAS,
workloadNodes: new Set(
unmounted.pods.map((pod) => pod.spec.nodeName),
).size,
revokeProjectionItems:
revoked.pointer.secretProjection.items.length,
revokeTransitionReceiptDigest:
revoker.final.transitionReceiptDigest,
},
gates: {
realThreeNodeKubernetes: true,
@@ -291,6 +575,13 @@ async function main() {
v3TransitionReceiptBound: true,
exactSecretProjectionRendered: true,
secretApiReadDenied: true,
exactItemMountedByRealWorkload: true,
unprojectedSecretKeyAbsent: true,
workloadReplicasOnDistinctNodes: true,
revokeReceiptBound: true,
revokeRolledNewPods: true,
revokedWorkloadHasNoSecretMount: true,
sourceSecretRetainedButInaccessible: true,
passed: true,
},
elapsedMs: Date.now() - startedAt,
@@ -37,7 +37,7 @@ const TOKEN_FILE = '/var/run/secrets/kubernetes.io/serviceaccount/token';
const RESULT_SCHEMA = 'qinglong/plugin-package-kubernetes-live-actor-result@v1';
const NAMESPACE = process.env.QL3_LIVE_NAMESPACE;
const ACTOR = process.env.QL3_LIVE_ACTOR;
const PEER = ACTOR === 'a' ? 'b' : 'a';
const PEER = ACTOR === 'a' ? 'b' : ACTOR === 'b' ? 'a' : null;
const INITIAL_LOCK_DIGEST = 'a'.repeat(64);
const CANDIDATES = Object.freeze({
a: Object.freeze({
@@ -56,6 +56,14 @@ const CANDIDATES = Object.freeze({
contentDigest: '9'.repeat(64),
intentDigest: '0'.repeat(64),
}),
c: Object.freeze({
installationId: 'install-live-revoke',
lockDigest: 'f'.repeat(64),
stageReceiptDigest: 'a'.repeat(64),
stageEvidenceDigest: 'b'.repeat(64),
contentDigest: 'c'.repeat(64),
intentDigest: 'd'.repeat(64),
}),
});
function fail(message) {
@@ -171,10 +179,10 @@ function manifest(version, secrets) {
});
}
function transitionEvidence(initial, candidate) {
function transitionEvidence(initial, candidate, actor = ACTOR) {
const secretRef = createSecretRef({
projectId: 'default',
name: `live-token-${ACTOR}`,
name: `live-token-${actor}`,
version: 2,
});
const previousManifest = manifest('1.0.0', []);
@@ -211,6 +219,57 @@ function transitionEvidence(initial, candidate) {
return Object.freeze({ secretRef, binding, receipt });
}
function revokeEvidence(activeCandidate, activeTransition, candidate) {
const plan = createPluginPackageSecretBindingTransitionPlan({
previousTarget: createPluginPackageSecretBindingTarget(
activeCandidate.resourceGeneration,
manifest('2.0.0', [
Object.freeze({ name: 'TOKEN', required: true }),
]),
),
previousBinding: activeTransition.binding,
previousAttemptGeneration: 2,
nextGeneration: candidate.resourceGeneration,
nextManifest: manifest('3.0.0', []),
assignments: [],
plannedAtMs: 300,
});
const receipt = createPluginPackageSecretBindingTransitionReceipt({
transitionPlan: plan,
authority: Object.freeze({
kind: 'approved-action-execution',
evidenceDigest: candidate.stageEvidenceDigest,
}),
binding: null,
committedAtMs: 400,
});
return Object.freeze({ binding: null, receipt });
}
function activeTargetName() {
return (
'ql3p-' +
require('node:crypto')
.createHash('sha256')
.update(
Buffer.from('qinglong/plugin-package-kubernetes-target@v1\0', 'utf8'),
)
.update(
require('node:crypto')
.createHash('sha256')
.update('qinglong/plugin-package-kubernetes-cluster@v1\0', 'utf8')
.update('ql3-plugin-package-live-cluster', 'utf8')
.digest('hex'),
'utf8',
)
.update('\0', 'utf8')
.update(NAMESPACE, 'utf8')
.update('\0default\0live-cas-package', 'utf8')
.digest('hex')
.slice(0, 52)
);
}
function exactEvidence(intent) {
return Object.freeze({
lockDigest: intent.lockDigest,
@@ -252,8 +311,8 @@ async function main() {
if (!/^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/.test(NAMESPACE ?? '')) {
fail('QL3_LIVE_NAMESPACE is invalid');
}
if (ACTOR !== 'a' && ACTOR !== 'b') {
fail('QL3_LIVE_ACTOR must be a or b');
if (ACTOR !== 'a' && ACTOR !== 'b' && ACTOR !== 'c') {
fail('QL3_LIVE_ACTOR must be a, b or c');
}
const token = fs.readFileSync(TOKEN_FILE);
assert.ok(token.length >= 32 && token.length <= 16 * 1024);
@@ -286,29 +345,34 @@ async function main() {
replaceCalls += 1;
replaceResourceVersion = request.body.metadata.resourceVersion;
assert.match(replaceResourceVersion ?? '', /^[1-9][0-9]*$/);
const ownReadyName = `ql3-live-cas-ready-${ACTOR}`;
const peerReadyName = `ql3-live-cas-ready-${PEER}`;
await rawApi.createNamespacedConfigMap({
namespace: NAMESPACE,
body: readyConfigMap(ownReadyName),
fieldManager: 'qinglong-plugin-package-live-gate',
fieldValidation: 'Strict',
});
await waitFor(`peer CAS barrier ConfigMap/${peerReadyName}`, async () => {
try {
const peer = await rawApi.readNamespacedConfigMap({
namespace: NAMESPACE,
name: peerReadyName,
});
return {
ready: peer.data?.actor === PEER,
fact: `peer actor=${String(peer.data?.actor)}`,
};
} catch (error) {
if (apiStatus(error) === 404) return { ready: false };
throw error;
}
});
if (PEER !== null) {
const ownReadyName = `ql3-live-cas-ready-${ACTOR}`;
const peerReadyName = `ql3-live-cas-ready-${PEER}`;
await rawApi.createNamespacedConfigMap({
namespace: NAMESPACE,
body: readyConfigMap(ownReadyName),
fieldManager: 'qinglong-plugin-package-live-gate',
fieldValidation: 'Strict',
});
await waitFor(
`peer CAS barrier ConfigMap/${peerReadyName}`,
async () => {
try {
const peer = await rawApi.readNamespacedConfigMap({
namespace: NAMESPACE,
name: peerReadyName,
});
return {
ready: peer.data?.actor === PEER,
fact: `peer actor=${String(peer.data?.actor)}`,
};
} catch (error) {
if (apiStatus(error) === 404) return { ready: false };
throw error;
}
},
);
}
return rawApi.replaceNamespacedConfigMap(request);
},
};
@@ -329,6 +393,117 @@ async function main() {
},
);
if (ACTOR === 'c') {
const initial = activationIntent();
const activeConfigMap = await rawApi.readNamespacedConfigMap({
namespace: NAMESPACE,
name: activeTargetName(),
});
const activePointer = JSON.parse(activeConfigMap.data?.['active.json']);
const winnerActor = ['a', 'b'].find(
(actor) => CANDIDATES[actor].lockDigest === activePointer.intent.lockDigest,
);
assert.ok(winnerActor);
const activeCandidate = activationIntent({
...CANDIDATES[winnerActor],
targetGeneration: 2,
previousActiveLockDigest: INITIAL_LOCK_DIGEST,
});
const activeTransition = transitionEvidence(
initial,
activeCandidate,
winnerActor,
);
assert.equal(
activePointer.secretProjection.bindingDigest,
activeTransition.binding.bindingDigest,
);
const candidate = activationIntent({
...CANDIDATES.c,
targetGeneration: 3,
previousActiveLockDigest: activeCandidate.lockDigest,
});
const transition = revokeEvidence(
activeCandidate,
activeTransition,
candidate,
);
const publisher = createPublisher({
sourceSecretName: 'ql3-cluster-plugin-package-values',
bindings: { find: async () => null },
transitions: { find: async () => transition.receipt },
});
const receipt = await publisher.publish(candidate);
assert.equal(replaceCalls, 1);
const active = await publisher.findActiveDeployment(
'default',
'live-cas-package',
);
assert.ok(active);
assert.equal(active.resourceGeneration.lockDigest, candidate.lockDigest);
assert.equal(active.secretProjection.items.length, 0);
assert.equal(
pluginPackageKubernetesProjectedSecretWorkloadVolume(
active.secretProjection,
),
null,
);
const rbac = Object.freeze({
listConfigMaps: await expectForbidden(() =>
rawApi.listNamespacedConfigMap({ namespace: NAMESPACE }),
),
deleteConfigMap: await expectForbidden(() =>
rawApi.deleteNamespacedConfigMap({
namespace: NAMESPACE,
name: activeTargetName(),
}),
),
readSecret: await expectForbidden(() =>
rawApi.readNamespacedSecret({
namespace: NAMESPACE,
name: 'forbidden-secret',
}),
),
crossNamespaceRead: await expectForbidden(() =>
rawApi.readNamespacedConfigMap({
namespace: 'default',
name: 'kube-root-ca.crt',
}),
),
});
const result = JSON.stringify({
schema: RESULT_SCHEMA,
actor: ACTOR,
mode: 'revoke',
serviceAccountTokenMounted: true,
responseLoss: null,
cas: {
status: 'fulfilled',
receipt,
attemptedResourceVersion: replaceResourceVersion,
replaceCalls,
},
final: {
resourceVersion: activeConfigMap.metadata.resourceVersion,
lockDigest: candidate.lockDigest,
generation: receipt.generation,
pointerSchema: 'qinglong/plugin-package-kubernetes-active-pointer@v3',
projectionDigest: active.secretProjection.projectionDigest,
transitionReceiptDigest:
active.secretProjection.transitionReceiptDigest,
projectionItemCount: active.secretProjection.items.length,
projectedWorkloadVolume: false,
},
rbac,
});
fs.writeFileSync('/dev/termination-log', result, {
encoding: 'utf8',
flag: 'w',
});
process.stdout.write(`${result}\n`);
return;
}
const initial = activationIntent();
const initialPublisher = createPublisher();
let responseLoss = null;
@@ -386,26 +561,7 @@ async function main() {
}
assert.equal(replaceCalls, 1);
const targetName =
'ql3p-' +
require('node:crypto')
.createHash('sha256')
.update(
Buffer.from('qinglong/plugin-package-kubernetes-target@v1\0', 'utf8'),
)
.update(
require('node:crypto')
.createHash('sha256')
.update('qinglong/plugin-package-kubernetes-cluster@v1\0', 'utf8')
.update('ql3-plugin-package-live-cluster', 'utf8')
.digest('hex'),
'utf8',
)
.update('\0', 'utf8')
.update(NAMESPACE, 'utf8')
.update('\0default\0live-cas-package', 'utf8')
.digest('hex')
.slice(0, 52);
const targetName = activeTargetName();
const finalConfigMap = await rawApi.readNamespacedConfigMap({
namespace: NAMESPACE,
name: targetName,
@@ -458,6 +614,7 @@ async function main() {
const result = JSON.stringify({
schema: RESULT_SCHEMA,
actor: ACTOR,
mode: 'rotate',
serviceAccountTokenMounted: true,
responseLoss,
cas: {