mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 20:15:19 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,491 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const {
|
||||
canonicalPluginPackagePromptOutputKeyringManifest,
|
||||
} = require('../packages/ql3-ai/dist/prompt-output/key-management/pluginPackagePromptOutputKeyringManifest.js');
|
||||
const {
|
||||
K3sDockerLiveFixture,
|
||||
run,
|
||||
waitFor,
|
||||
} = require('./lib/ql3-k3s-docker-live.cjs');
|
||||
const {
|
||||
FIXTURE,
|
||||
LIMITATIONS,
|
||||
validatePromptOutputProjectionKubernetesLiveReport,
|
||||
} = require('./ql3-prompt-output-projection-kubernetes-live-audit.cjs');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const NAMESPACE = 'qinglong3-system';
|
||||
const SERVICE_ACCOUNT = 'ql3-prompt-output-projection';
|
||||
const POD = 'ql3-prompt-output-projection';
|
||||
const SECRET_NAME = 'ql3-prompt-output-keyring';
|
||||
const IMAGE_BASE = 'ql3-prompt-output-projection-live';
|
||||
const PRESERVE_FAILURE_ENV =
|
||||
'QL3_PROMPT_OUTPUT_PROJECTION_KUBERNETES_LIVE_PRESERVE_FAILURE';
|
||||
|
||||
function imageId(image) {
|
||||
assert.match(image.Id, /^sha256:[a-f0-9]{64}$/);
|
||||
return image.Id;
|
||||
}
|
||||
|
||||
function keyring(generation, activeKeyId, keys) {
|
||||
return Object.freeze({
|
||||
schema: 'qinglong/plugin-package-prompt-output-file-keyring@v1',
|
||||
generation,
|
||||
activeKeyId,
|
||||
keys: Object.freeze(keys),
|
||||
retirements: Object.freeze({}),
|
||||
});
|
||||
}
|
||||
|
||||
function secretManifest(manifest, revision) {
|
||||
const bytes = canonicalPluginPackagePromptOutputKeyringManifest(manifest);
|
||||
try {
|
||||
return {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: {
|
||||
name: SECRET_NAME,
|
||||
namespace: NAMESPACE,
|
||||
...(revision === undefined ? {} : { resourceVersion: revision }),
|
||||
},
|
||||
type: 'Opaque',
|
||||
immutable: false,
|
||||
data: { 'keyring.json': bytes.toString('base64') },
|
||||
};
|
||||
} finally {
|
||||
bytes.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
function actorSource() {
|
||||
return String.raw`
|
||||
'use strict';
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const {
|
||||
createPluginPackagePromptOutputArtifact,
|
||||
openPluginPackagePromptOutputArtifact,
|
||||
} = require('@qinglong/ai/plugin-package-prompt-output-artifact');
|
||||
const {
|
||||
createPluginPackagePromptOutputProjectedKeyring,
|
||||
} = require('@qinglong/ai/plugin-package-prompt-output-projected-keyring');
|
||||
|
||||
const root = '/var/run/secrets/qinglong3/ai/prompt-output-keyring';
|
||||
const sleep = (milliseconds) =>
|
||||
new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
let stage = 'create-provider';
|
||||
|
||||
async function main() {
|
||||
const provider = await createPluginPackagePromptOutputProjectedKeyring({
|
||||
rootDirectory: root,
|
||||
});
|
||||
const first = await provider.active();
|
||||
stage = 'create-artifact';
|
||||
const firstId = first.keyId;
|
||||
const expected = Object.freeze({
|
||||
provider: 'openai-compatible',
|
||||
model: 'projection-live-model',
|
||||
text: 'projection-live-private-result',
|
||||
finishReason: 'stop',
|
||||
usage: Object.freeze({
|
||||
inputTokens: 1,
|
||||
outputTokens: 1,
|
||||
totalTokens: 2,
|
||||
costMicros: 0,
|
||||
}),
|
||||
});
|
||||
const artifact = createPluginPackagePromptOutputArtifact({
|
||||
projectId: 'projection-live-project',
|
||||
runId: 'projection-live-run',
|
||||
stepRunId: 'projection-live-step',
|
||||
invocationId: 'projection-live-invocation',
|
||||
requestedBy: { type: 'system', id: 'projection-live' },
|
||||
result: expected,
|
||||
retentionPolicy: { revision: 'projection-live-v1', retentionMs: 3600000 },
|
||||
keyId: first.keyId,
|
||||
key: first.key,
|
||||
sealedAtMs: Date.now(),
|
||||
});
|
||||
first.key.fill(0);
|
||||
fs.writeFileSync('/tmp/projection-ready', 'ready\n', { flag: 'wx' });
|
||||
|
||||
stage = 'wait-for-rotation';
|
||||
const deadline = Date.now() + 180000;
|
||||
let second;
|
||||
let transientUnavailableObserved = false;
|
||||
while (Date.now() < deadline) {
|
||||
let candidate;
|
||||
try {
|
||||
candidate = await provider.active();
|
||||
} catch (error) {
|
||||
if (
|
||||
error &&
|
||||
typeof error === 'object' &&
|
||||
error.code === 'PLUGIN_PACKAGE_PROMPT_OUTPUT_PROJECTED_KEYRING_UNAVAILABLE'
|
||||
) {
|
||||
transientUnavailableObserved = true;
|
||||
await sleep(250);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (candidate.keyId !== firstId) {
|
||||
second = candidate;
|
||||
break;
|
||||
}
|
||||
candidate.key.fill(0);
|
||||
await sleep(250);
|
||||
}
|
||||
assert.ok(second, 'projected active key did not rotate');
|
||||
second.key.fill(0);
|
||||
stage = 'open-historical-artifact';
|
||||
const historical = await provider.resolve(firstId);
|
||||
assert.ok(historical, 'historical key is unavailable');
|
||||
const opened = openPluginPackagePromptOutputArtifact(
|
||||
artifact,
|
||||
historical.key,
|
||||
);
|
||||
historical.key.fill(0);
|
||||
assert.deepEqual(opened, expected);
|
||||
stage = 'verify-runtime-boundary';
|
||||
const projectedFile = fs.lstatSync(root + '/keyring.json');
|
||||
assert.equal(projectedFile.isSymbolicLink(), true);
|
||||
assert.equal(
|
||||
fs.existsSync('/var/run/secrets/kubernetes.io/serviceaccount/token'),
|
||||
false,
|
||||
);
|
||||
fs.writeFileSync(
|
||||
'/dev/termination-log',
|
||||
JSON.stringify({
|
||||
activeChanged: true,
|
||||
atomicWriterSymlink: true,
|
||||
historicalArtifactOpened: true,
|
||||
runtimeCredentialAbsent: true,
|
||||
transientUnavailableObserved,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
fs.writeFileSync(
|
||||
'/dev/termination-log',
|
||||
JSON.stringify({
|
||||
failed: true,
|
||||
stage,
|
||||
name: error instanceof Error ? error.name : 'UnknownError',
|
||||
code:
|
||||
error && typeof error === 'object' && typeof error.code === 'string'
|
||||
? error.code
|
||||
: null,
|
||||
}),
|
||||
);
|
||||
process.stderr.write((error instanceof Error ? error.stack : String(error)) + '\n');
|
||||
process.exitCode = 1;
|
||||
});
|
||||
`;
|
||||
}
|
||||
|
||||
function projectionResources(image) {
|
||||
return [
|
||||
{
|
||||
apiVersion: 'v1',
|
||||
kind: 'ServiceAccount',
|
||||
metadata: { name: SERVICE_ACCOUNT, namespace: NAMESPACE },
|
||||
automountServiceAccountToken: false,
|
||||
},
|
||||
{
|
||||
apiVersion: 'v1',
|
||||
kind: 'Pod',
|
||||
metadata: {
|
||||
name: POD,
|
||||
namespace: NAMESPACE,
|
||||
labels: { 'app.kubernetes.io/name': POD },
|
||||
},
|
||||
spec: {
|
||||
serviceAccountName: SERVICE_ACCOUNT,
|
||||
automountServiceAccountToken: false,
|
||||
restartPolicy: 'Never',
|
||||
securityContext: {
|
||||
runAsNonRoot: true,
|
||||
runAsUser: 10001,
|
||||
runAsGroup: 10001,
|
||||
fsGroup: 10001,
|
||||
seccompProfile: { type: 'RuntimeDefault' },
|
||||
},
|
||||
containers: [
|
||||
{
|
||||
name: 'projection-probe',
|
||||
image,
|
||||
imagePullPolicy: 'Never',
|
||||
command: ['node', '-e', actorSource()],
|
||||
readinessProbe: {
|
||||
exec: { command: ['test', '-f', '/tmp/projection-ready'] },
|
||||
periodSeconds: 1,
|
||||
timeoutSeconds: 1,
|
||||
failureThreshold: 180,
|
||||
},
|
||||
resources: {
|
||||
requests: { cpu: '10m', memory: '32Mi' },
|
||||
limits: { cpu: '250m', memory: '128Mi' },
|
||||
},
|
||||
securityContext: {
|
||||
allowPrivilegeEscalation: false,
|
||||
readOnlyRootFilesystem: true,
|
||||
capabilities: { drop: ['ALL'] },
|
||||
},
|
||||
volumeMounts: [
|
||||
{ name: 'tmp', mountPath: '/tmp' },
|
||||
{
|
||||
name: 'prompt-output-keyring',
|
||||
mountPath:
|
||||
'/var/run/secrets/qinglong3/ai/prompt-output-keyring',
|
||||
readOnly: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
volumes: [
|
||||
{ name: 'tmp', emptyDir: { medium: 'Memory', sizeLimit: '1Mi' } },
|
||||
{
|
||||
name: 'prompt-output-keyring',
|
||||
secret: {
|
||||
secretName: SECRET_NAME,
|
||||
defaultMode: 0o440,
|
||||
items: [{ key: 'keyring.json', path: 'keyring.json' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (process.env.QL3_PROMPT_OUTPUT_PROJECTION_KUBERNETES_LIVE !== '1') {
|
||||
throw new Error(
|
||||
'Refusing to mutate Docker/Kubernetes without ' +
|
||||
'QL3_PROMPT_OUTPUT_PROJECTION_KUBERNETES_LIVE=1',
|
||||
);
|
||||
}
|
||||
const fixture = new K3sDockerLiveFixture({
|
||||
prefix: 'ql3-key-projection-live',
|
||||
});
|
||||
const image = `${IMAGE_BASE}:${process.pid.toString(36)}-${crypto
|
||||
.randomBytes(3)
|
||||
.toString('hex')}`;
|
||||
let imageBuilt = false;
|
||||
let completed = false;
|
||||
try {
|
||||
const nodes = await fixture.start();
|
||||
const architecture = fixture.inspectImage(fixture.k3sImage).Architecture;
|
||||
run(fixture.docker, [
|
||||
'build',
|
||||
'--file',
|
||||
'deploy/containers/ql3-cluster-control/Dockerfile',
|
||||
'--target',
|
||||
'runtime-ai',
|
||||
'--tag',
|
||||
image,
|
||||
'--build-arg',
|
||||
`SOURCE_REVISION=${process.env.GITHUB_SHA || 'live-contract'}`,
|
||||
'.',
|
||||
]);
|
||||
imageBuilt = true;
|
||||
fixture.loadImage(image, 'prompt-output-projection.tar');
|
||||
fixture.kubectl([
|
||||
'apply',
|
||||
'-f',
|
||||
'deploy/kubernetes/ql3-cluster/base/namespace.yaml',
|
||||
]);
|
||||
|
||||
const firstId = 'projection-key-one';
|
||||
const secondId = 'projection-key-two';
|
||||
const firstValue = crypto.randomBytes(32).toString('base64url');
|
||||
const secondValue = crypto.randomBytes(32).toString('base64url');
|
||||
fixture.create(
|
||||
secretManifest(keyring(1, firstId, { [firstId]: firstValue })),
|
||||
);
|
||||
for (const resource of projectionResources(image)) fixture.create(resource);
|
||||
const readiness = await waitFor(
|
||||
'projection probe readiness',
|
||||
240_000,
|
||||
() => {
|
||||
const pod = fixture.kubectlJson(['-n', NAMESPACE, 'get', 'pod', POD]);
|
||||
const state = pod.status.containerStatuses?.[0]?.state;
|
||||
if (state?.terminated) {
|
||||
return { ready: true, value: { pod, terminated: state.terminated } };
|
||||
}
|
||||
return pod.status.conditions?.some(
|
||||
(condition) =>
|
||||
condition.type === 'Ready' && condition.status === 'True',
|
||||
)
|
||||
? { ready: true, value: pod }
|
||||
: { ready: false, fact: pod.status.phase ?? 'Pending' };
|
||||
},
|
||||
);
|
||||
if (readiness.value.terminated) {
|
||||
throw new Error(
|
||||
`projection probe terminated before readiness: ${
|
||||
readiness.value.terminated.message ||
|
||||
readiness.value.terminated.reason
|
||||
}`,
|
||||
);
|
||||
}
|
||||
const podBefore = fixture.kubectlJson(['-n', NAMESPACE, 'get', 'pod', POD]);
|
||||
const secretBefore = fixture.kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'get',
|
||||
'secret',
|
||||
SECRET_NAME,
|
||||
]);
|
||||
fixture.kubectl(['replace', '-f', '-'], {
|
||||
input: `${JSON.stringify(
|
||||
secretManifest(
|
||||
keyring(2, secondId, {
|
||||
[firstId]: firstValue,
|
||||
[secondId]: secondValue,
|
||||
}),
|
||||
secretBefore.metadata.resourceVersion,
|
||||
),
|
||||
)}\n`,
|
||||
capture: true,
|
||||
quiet: true,
|
||||
});
|
||||
const secretAfter = fixture.kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'get',
|
||||
'secret',
|
||||
SECRET_NAME,
|
||||
]);
|
||||
assert.notEqual(
|
||||
secretAfter.metadata.resourceVersion,
|
||||
secretBefore.metadata.resourceVersion,
|
||||
);
|
||||
|
||||
const terminated = (
|
||||
await waitFor('projection probe completion', 240_000, () => {
|
||||
const pod = fixture.kubectlJson(['-n', NAMESPACE, 'get', 'pod', POD]);
|
||||
const state = pod.status.containerStatuses?.[0]?.state?.terminated;
|
||||
return state
|
||||
? { ready: true, value: { pod, state } }
|
||||
: { ready: false, fact: pod.status.phase ?? 'Pending' };
|
||||
})
|
||||
).value;
|
||||
assert.equal(terminated.state.exitCode, 0, terminated.state.message);
|
||||
const actor = JSON.parse(terminated.state.message);
|
||||
assert.equal(typeof actor.transientUnavailableObserved, 'boolean');
|
||||
assert.deepEqual(actor, {
|
||||
activeChanged: true,
|
||||
atomicWriterSymlink: true,
|
||||
historicalArtifactOpened: true,
|
||||
runtimeCredentialAbsent: true,
|
||||
transientUnavailableObserved: actor.transientUnavailableObserved,
|
||||
});
|
||||
const podSpec = terminated.pod.spec;
|
||||
const container = podSpec.containers[0];
|
||||
const projectionVolume = podSpec.volumes.find(
|
||||
(entry) => entry.name === 'prompt-output-keyring',
|
||||
);
|
||||
const projectionMount = container.volumeMounts.find(
|
||||
(entry) => entry.name === 'prompt-output-keyring',
|
||||
);
|
||||
assert.equal(podSpec.automountServiceAccountToken, false);
|
||||
assert.equal(projectionMount.readOnly, true);
|
||||
assert.equal(projectionVolume.secret.defaultMode, 0o440);
|
||||
assert.deepEqual(projectionVolume.secret.items, [
|
||||
{ key: 'keyring.json', path: 'keyring.json' },
|
||||
]);
|
||||
|
||||
const version = fixture.kubectlJson(['version']).serverVersion.gitVersion;
|
||||
const report = Object.freeze({
|
||||
fixture: FIXTURE,
|
||||
observedAt: new Date().toISOString(),
|
||||
platform: Object.freeze({
|
||||
distribution: 'k3s',
|
||||
kubernetesVersion: version,
|
||||
architecture,
|
||||
kubernetesImageId: imageId(fixture.inspectImage(fixture.k3sImage)),
|
||||
cniName: 'flannel',
|
||||
controlPlaneNodes: nodes.filter(
|
||||
(node) =>
|
||||
node.metadata.labels?.['node-role.kubernetes.io/control-plane'] !==
|
||||
undefined,
|
||||
).length,
|
||||
workerNodes: nodes.filter(
|
||||
(node) =>
|
||||
node.metadata.labels?.['node-role.kubernetes.io/control-plane'] ===
|
||||
undefined,
|
||||
).length,
|
||||
}),
|
||||
projection: Object.freeze({
|
||||
generationBefore: 1,
|
||||
generationAfter: 2,
|
||||
defaultMode: 0o440,
|
||||
activeChanged: actor.activeChanged,
|
||||
atomicWriterSymlink: actor.atomicWriterSymlink,
|
||||
dataFileOnly: projectionVolume.secret.items.length === 1,
|
||||
historicalArtifactOpened: actor.historicalArtifactOpened,
|
||||
podIdentityStable:
|
||||
terminated.pod.metadata.uid === podBefore.metadata.uid,
|
||||
readOnlyMount: projectionMount.readOnly,
|
||||
revisionChanged:
|
||||
secretAfter.metadata.resourceVersion !==
|
||||
secretBefore.metadata.resourceVersion,
|
||||
runtimeCredentialAbsent: actor.runtimeCredentialAbsent,
|
||||
transientUnavailableObserved: actor.transientUnavailableObserved,
|
||||
}),
|
||||
gates: Object.freeze({
|
||||
contentFreeEvidence: true,
|
||||
historicalDecrypt: actor.historicalArtifactOpened,
|
||||
passed: true,
|
||||
readOnlyRuntime:
|
||||
podSpec.automountServiceAccountToken === false &&
|
||||
projectionMount.readOnly === true,
|
||||
realAtomicProjection: actor.atomicWriterSymlink,
|
||||
realKubernetesApi: true,
|
||||
rotationRecovered: actor.activeChanged,
|
||||
sameProcessRotation:
|
||||
actor.activeChanged &&
|
||||
terminated.pod.metadata.uid === podBefore.metadata.uid,
|
||||
}),
|
||||
limitations: LIMITATIONS,
|
||||
});
|
||||
const validation =
|
||||
validatePromptOutputProjectionKubernetesLiveReport(report);
|
||||
assert.equal(validation.compatible, true, validation.findings.join(', '));
|
||||
process.stdout.write(`${JSON.stringify(report)}\n`);
|
||||
completed = true;
|
||||
} finally {
|
||||
if (completed || process.env[PRESERVE_FAILURE_ENV] !== '1') {
|
||||
await fixture.cleanup().catch(() => undefined);
|
||||
if (imageBuilt) {
|
||||
run(fixture.docker, ['image', 'rm', '-f', image], {
|
||||
allowFailure: true,
|
||||
capture: true,
|
||||
quiet: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(
|
||||
`${error instanceof Error ? error.stack : String(error)}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { actorSource, projectionResources, secretManifest };
|
||||
Reference in New Issue
Block a user