feat(ql3): establish 3.0 incubation baseline

This commit is contained in:
whyour
2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
@@ -0,0 +1,187 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ApiCredentialAdministrationMutationConflictError,
} = require('@qinglong/runtime-core/api-credential-administration');
const {
apiCredentialSecretDigest,
} = require('@qinglong/runtime-core/api-credential-token');
const {
ClusterAdministrationAuthenticationError,
createClusterAdministrationService,
} = require('@qinglong/cluster-admin/administration');
const NOW = 1_000;
const PEPPER = 'A'.repeat(43);
const SUBJECT = { type: 'api_app', id: 'app_primary' };
const PRINCIPAL = {
subject: { type: 'user', id: 'usr_admin' },
authenticationId: 'session:admin:1',
authenticatedAtMs: 900,
expiresAtMs: 2_000,
assurance: 'multi_factor',
};
function request(overrides = {}) {
return {
mutationId: '123e4567-e89b-42d3-a456-426614174301',
requestId: 'request-credential-issue-1',
expectedCurrentVersion: 0,
credentialId: 'credential_primary',
subject: SUBJECT,
principal: PRINCIPAL,
notBeforeAtMs: NOW,
expiresAtMs: 2_000,
...overrides,
};
}
function repositories() {
let credentialMutation = null;
const credentialCommands = [];
const identityCommands = [];
const identity = {
subject: SUBJECT,
status: 'active',
version: 1,
createdAtMs: 100,
updatedAtMs: 100,
};
return {
identityCommands,
credentialCommands,
identities: {
async resolve() {
return identity;
},
async resolveMutation() {
return null;
},
async append(command) {
identityCommands.push(command);
return {
status: 'inserted',
identity: {
subject: command.mutation.subject,
status: command.mutation.status,
version: command.mutation.subjectVersion,
createdAtMs: command.mutation.createdAtMs,
updatedAtMs: command.mutation.createdAtMs,
},
mutation: command.mutation,
};
},
},
credentials: {
async resolveMutation() {
return credentialMutation;
},
async append(command) {
credentialCommands.push(command);
credentialMutation = {
credential: command.credential,
mutation: command.mutation,
audit: command.audit,
};
return {
status: 'inserted',
credential: command.credential,
mutation: command.mutation,
};
},
conflictWith(stored) {
credentialMutation = stored;
},
},
};
}
test('issues one token, stores only its digest and clears mutable secret bytes', async () => {
const repos = repositories();
const generated = Buffer.alloc(32, 7);
const expectedSecret = generated.toString('base64url');
const service = createClusterAdministrationService(
repos.identities,
repos.credentials,
PEPPER,
{ now: () => NOW, randomBytes: () => generated },
);
const result = await service.issueCredential(request());
assert.equal(result.status, 'inserted');
assert.equal(result.token, `ql3c_credential_primary_${expectedSecret}`);
assert.equal(repos.credentialCommands.length, 1);
assert.equal(
repos.credentialCommands[0].credential.secretDigest,
apiCredentialSecretDigest(PEPPER, 'credential_primary', expectedSecret),
);
assert.equal(repos.credentialCommands[0].audit.eventId, request().mutationId);
assert.equal(
generated.every((value) => value === 0),
true,
);
});
test('semantic mutation replay returns no token and does not generate a new secret', async () => {
const repos = repositories();
let randomCalls = 0;
const service = createClusterAdministrationService(
repos.identities,
repos.credentials,
PEPPER,
{
now: () => NOW,
randomBytes() {
randomCalls += 1;
return Buffer.alloc(32, randomCalls);
},
},
);
assert.ok((await service.issueCredential(request())).token);
const replay = await service.issueCredential(request());
assert.equal(replay.status, 'existing');
assert.equal(replay.token, null);
assert.equal(randomCalls, 1);
await assert.rejects(
service.issueCredential(request({ requestId: 'different-request' })),
ApiCredentialAdministrationMutationConflictError,
);
});
test('rejects weak principals and widened requests before repository mutation', async () => {
const repos = repositories();
const service = createClusterAdministrationService(
repos.identities,
repos.credentials,
PEPPER,
{ now: () => NOW, randomBytes: () => Buffer.alloc(32, 1) },
);
await assert.rejects(
service.issueCredential(
request({ principal: { ...PRINCIPAL, assurance: 'single_factor' } }),
),
ClusterAdministrationAuthenticationError,
);
await assert.rejects(
service.issueCredential(request({ debug: true })),
/request shape is invalid/,
);
assert.equal(repos.credentialCommands.length, 0);
});
test('maps malformed entropy output to a stable configuration error', async () => {
const repos = repositories();
const service = createClusterAdministrationService(
repos.identities,
repos.credentials,
PEPPER,
{ now: () => NOW, randomBytes: () => 'not-a-buffer' },
);
await assert.rejects(
service.issueCredential(request()),
/randomBytes returned invalid secret material/,
);
assert.equal(repos.credentialCommands.length, 0);
});
@@ -0,0 +1,34 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ClusterApprovalDecisionManagementConfigurationError,
createClusterApprovalDecisionManagementService,
} = require('@qinglong/cluster-admin/approval-decision-management');
test('composes the shared Approval decision authority over a caller-owned pool', () => {
const pool = {
async query() {
throw new Error('not used');
},
async connect() {
throw new Error('not used');
},
};
const service = createClusterApprovalDecisionManagementService({ pool });
assert.equal(typeof service.decide, 'function');
assert.equal(Object.isFrozen(service), true);
});
test('rejects missing pool authority and widened options', () => {
for (const options of [
{},
{ pool: { query() {} } },
{ pool: { query() {}, connect() {} }, transport: {} },
]) {
assert.throws(
() => createClusterApprovalDecisionManagementService(options),
ClusterApprovalDecisionManagementConfigurationError,
);
}
});
@@ -0,0 +1,223 @@
const assert = require('node:assert/strict');
const {
chmodSync,
mkdtempSync,
readFileSync,
realpathSync,
rmSync,
writeFileSync,
} = require('node:fs');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { afterEach, test } = require('node:test');
const {
ClusterPluginPackageManagementClientRequestError,
} = require('@qinglong/cluster-admin/plugin-package-management-client');
const {
executeClusterApprovalManagementClient,
validateClusterApprovalManagementClientResult,
} = require('@qinglong/cluster-admin/approval-management-client');
const FIXTURES = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls',
);
const temporaryDirectories = [];
const ACTION = Object.freeze({
permission: 'run.start',
actionType: 'tool.invoke',
actionRef: 'tool:task-1',
actionDigest: 'a'.repeat(64),
previewDigest: 'b'.repeat(64),
});
const BASE_REQUEST = Object.freeze({
projectId: 'default',
approvalRequestId: 'approval-1',
requestId: 'approval-command-1',
auditEventId: '60000000-0000-4000-8000-000000000001',
failureAuditEventId: '60000000-0000-4000-8000-000000000002',
});
function privateWrite(filePath, value) {
writeFileSync(filePath, value, { mode: 0o600 });
chmodSync(filePath, 0o600);
}
function clientFiles() {
const directory = realpathSync(
mkdtempSync(join(tmpdir(), 'ql3-approval-client-')),
);
temporaryDirectories.push(directory);
const paths = {
configFile: join(directory, 'client.json'),
commandFile: join(directory, 'command.json'),
assertionFile: join(directory, 'assertion.jwt'),
};
const caFile = join(directory, 'ca.crt');
const clientCertificateFile = join(directory, 'client.crt');
const clientPrivateKeyFile = join(directory, 'client.key');
privateWrite(caFile, readFileSync(join(FIXTURES, 'ca-cert.pem')));
privateWrite(
clientCertificateFile,
readFileSync(join(FIXTURES, 'client-cert.pem')),
);
privateWrite(
clientPrivateKeyFile,
readFileSync(join(FIXTURES, 'client-key.pem')),
);
privateWrite(
paths.configFile,
`${JSON.stringify({
schemaVersion: 1,
endpoint:
'https://approval.example.test:8447/api/v3/approvals/management',
servername: 'approval.example.test',
caFile,
clientCertificateFile,
clientPrivateKeyFile,
requestTimeoutMs: 1_000,
})}\n`,
);
privateWrite(
paths.commandFile,
`${JSON.stringify({
schemaVersion: 1,
operation: 'approval.inspect',
request: BASE_REQUEST,
})}\n`,
);
privateWrite(
paths.assertionFile,
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl',
);
return paths;
}
afterEach(() => {
for (const directory of temporaryDirectories.splice(0)) {
rmSync(directory, { recursive: true, force: true });
}
});
test('validates exact inspect results and binds them to the requested Approval', () => {
const command = {
schemaVersion: 1,
operation: 'approval.inspect',
request: BASE_REQUEST,
};
const found = {
schemaVersion: 1,
operation: 'approval.inspect',
status: 'found',
approval: {
projectId: 'default',
approvalRequestId: 'approval-1',
version: 1,
state: 'pending',
risk: 'high',
decisionMode: 'human_confirmation',
expectedAction: ACTION,
requestedBy: { type: 'agent', id: 'agent-1' },
requestedAtMs: 1_000,
expiresAtMs: 10_000,
preview: {
title: 'Run task',
summary: 'Runs one reviewed task.',
fields: [{ kind: 'identifier', label: 'Task', value: 'task-1' }],
warnings: ['external_effect'],
},
},
};
assert.deepEqual(
validateClusterApprovalManagementClientResult(found, command),
found,
);
assert.deepEqual(
validateClusterApprovalManagementClientResult(
{ ...found, status: 'absent', approval: null },
command,
),
{ ...found, status: 'absent', approval: null },
);
assert.throws(
() =>
validateClusterApprovalManagementClientResult(
{
...found,
approval: { ...found.approval, projectId: 'other' },
},
command,
),
ClusterPluginPackageManagementClientRequestError,
);
});
test('validates the durable decision tuple and rejects server-side drift', () => {
const command = {
schemaVersion: 1,
operation: 'approval.decide',
request: {
...BASE_REQUEST,
expectedVersion: 1,
expectedAction: ACTION,
decisionId: 'decision-1',
decision: 'approved',
reasonCode: 'reviewed',
},
};
const result = {
schemaVersion: 1,
operation: 'approval.decide',
status: 'decided',
approval: {
projectId: 'default',
approvalRequestId: 'approval-1',
version: 2,
state: 'approved',
expectedAction: ACTION,
decisionId: 'decision-1',
decision: 'approved',
reasonCode: 'reviewed',
decidedBy: { type: 'user', id: 'owner-1' },
decidedAtMs: 2_000,
},
};
assert.deepEqual(
validateClusterApprovalManagementClientResult(result, command),
result,
);
for (const approval of [
{ ...result.approval, version: 1 },
{ ...result.approval, state: 'rejected' },
{ ...result.approval, decisionId: 'decision-2' },
]) {
assert.throws(
() =>
validateClusterApprovalManagementClientResult(
{ ...result, approval },
command,
),
ClusterPluginPackageManagementClientRequestError,
);
}
});
test('accepts only the exact Approval route before opening one mTLS connection', async () => {
let connects = 0;
await assert.rejects(
executeClusterApprovalManagementClient(clientFiles(), {
async connect(target) {
connects += 1;
assert.deepEqual(target, {
hostname: 'approval.example.test',
port: 8447,
});
throw new Error('expected-connect-stop');
},
}),
{ code: 'QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED' },
);
assert.equal(connects, 1);
});
@@ -0,0 +1,134 @@
const assert = require('node:assert/strict');
const { readFileSync } = require('node:fs');
const { request: httpsRequest } = require('node:https');
const { resolve } = require('node:path');
const { test } = require('node:test');
const {
startClusterApprovalManagementHttp,
} = require('@qinglong/cluster-admin/approval-management-http');
const {
ClusterApprovalManagementTransportConflictError,
} = require('@qinglong/cluster-admin/approval-management-transport');
const SERVER_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-key.pem',
);
const SERVER_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem',
);
const PATH = '/api/v3/approvals/management';
function post(port, path = PATH) {
const body = Buffer.from(
JSON.stringify({
schemaVersion: 1,
operation: 'approval.inspect',
request: {
projectId: 'default',
approvalRequestId: 'approval-1',
requestId: 'approval-command-1',
auditEventId: '50000000-0000-4000-8000-000000000001',
failureAuditEventId: '50000000-0000-4000-8000-000000000002',
},
}),
);
return new Promise((resolvePromise, reject) => {
const outgoing = httpsRequest(
{
host: '127.0.0.1',
port,
path,
method: 'POST',
rejectUnauthorized: false,
agent: false,
headers: {
authorization: 'Bearer assertion',
'content-type': 'application/json',
'content-length': String(body.length),
},
},
(incoming) => {
const chunks = [];
incoming.on('data', (chunk) => chunks.push(chunk));
incoming.once('end', () => {
const bytes = Buffer.concat(chunks);
resolvePromise({
statusCode: incoming.statusCode,
body: bytes.length ? JSON.parse(bytes.toString('utf8')) : null,
});
});
},
);
outgoing.once('error', reject);
outgoing.end(body);
});
}
test('serves only the human Approval path and maps conflicts to low-sensitive facts', async () => {
const privateKey = Buffer.from(readFileSync(SERVER_KEY));
const application = await startClusterApprovalManagementHttp({
host: '127.0.0.1',
port: 0,
tls: {
privateKey,
certificate: Buffer.from(readFileSync(SERVER_CERT)),
},
identities: {
async reload() {
return {
schemaVersion: 1,
generation: 1,
digest: 'digest',
issuer: 'https://identity.example.test/',
audience: 'qinglong3-approval-management',
activeKeyIds: ['key-1'],
revokedKeyIds: [],
};
},
bind() {
return {
async authenticate() {
return {
subject: { type: 'user', id: 'owner-1' },
authenticationId: 'session-owner-1',
authenticatedAtMs: 900,
expiresAtMs: 10_000,
assurance: 'hardware',
};
},
};
},
},
transport: {
async execute(_command, authentication) {
await authentication.authenticate();
throw new ClusterApprovalManagementTransportConflictError();
},
},
now: () => 1_000,
});
try {
assert.equal(privateKey.every((value) => value === 0), true);
const conflict = await post(application.address.port);
assert.deepEqual(conflict, {
statusCode: 409,
body: {
schemaVersion: 1,
requestId: conflict.body.requestId,
error: { code: 'conflict' },
},
});
assert.match(conflict.body.requestId, /^[0-9a-f-]{36}$/);
const absent = await post(
application.address.port,
'/api/v3/automations/management',
);
assert.equal(absent.statusCode, 404);
assert.equal(absent.body.error.code, 'not_found');
} finally {
await application.close();
}
});
@@ -0,0 +1,209 @@
const assert = require('node:assert/strict');
const { readFile, writeFile, mkdtemp, rm } = require('node:fs/promises');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterApprovalManagementProcessConfigError,
loadClusterApprovalManagementProcessConfig,
startClusterApprovalManagementProcess,
} = require('@qinglong/cluster-admin/approval-management-process');
const FIXTURES = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls',
);
const NOW_MS = Date.UTC(2030, 0, 1);
function enabledEnvironment(paths, overrides = {}) {
return {
QL3_APPROVAL_MANAGEMENT_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
QL3_APPROVAL_MANAGEMENT_HOST: '127.0.0.1',
QL3_APPROVAL_MANAGEMENT_PORT: '8447',
QL3_APPROVAL_MANAGEMENT_TLS_CERT_FILE: paths.certificateFile,
QL3_APPROVAL_MANAGEMENT_TLS_KEY_FILE: paths.privateKeyFile,
QL3_APPROVAL_MANAGEMENT_CLIENT_CA_FILE:
paths.clientCertificateAuthorityFile,
QL3_APPROVAL_MANAGEMENT_CLIENT_CRL_FILE:
paths.clientCertificateRevocationListFile,
QL3_APPROVAL_MANAGEMENT_IDENTITY_KEYSET_FILE: paths.identityKeysetFile,
QL3_POSTGRES_APPROVAL_MANAGER_URL:
'postgresql://ql3_approval_manager:secret@postgres.example.test/ql3',
QL3_POSTGRES_APPROVAL_MANAGER_TLS_MODE: 'disable',
QL3_POSTGRES_APPROVAL_MANAGER_ALLOW_INSECURE: 'true',
...overrides,
};
}
async function tlsFixture(run) {
const directory = await mkdtemp(join(tmpdir(), 'ql3-approval-manager-'));
const paths = {
certificateFile: join(directory, 'tls.crt'),
privateKeyFile: join(directory, 'tls.key'),
clientCertificateAuthorityFile: join(directory, 'client-ca.crt'),
clientCertificateRevocationListFile: join(directory, 'client.crl'),
identityKeysetFile: join(directory, 'keyset.json'),
};
try {
await writeFile(
paths.certificateFile,
await readFile(join(FIXTURES, 'server-cert.pem')),
{ mode: 0o644 },
);
await writeFile(
paths.privateKeyFile,
await readFile(join(FIXTURES, 'server-key.pem')),
{ mode: 0o640 },
);
await writeFile(
paths.clientCertificateAuthorityFile,
await readFile(join(FIXTURES, 'ca-cert.pem')),
{ mode: 0o644 },
);
await writeFile(
paths.clientCertificateRevocationListFile,
await readFile(join(FIXTURES, 'empty-crl.pem')),
{ mode: 0o644 },
);
await writeFile(paths.identityKeysetFile, '{}\n', { mode: 0o644 });
return await run(paths);
} finally {
await rm(directory, { recursive: true, force: true });
}
}
test('disabled Approval manager acquires no PostgreSQL or file authority', async () => {
let opened = 0;
const runtime = await startClusterApprovalManagementProcess({
environment: { QL3_APPROVAL_MANAGEMENT_ENABLED: 'false' },
async openDatabase() {
opened += 1;
throw new Error('must not open');
},
});
assert.equal(runtime.status, 'disabled');
assert.equal(opened, 0);
await runtime.close();
});
test('loads bounded low-footprint Approval-only HTTPS and PostgreSQL configuration', async () => {
await tlsFixture(async (paths) => {
const config = loadClusterApprovalManagementProcessConfig(
enabledEnvironment(paths),
);
assert.equal(config.enabled, true);
assert.equal(config.port, 8447);
assert.equal(config.database.pool.maxConnections, 2);
assert.equal(
config.database.pool.applicationName,
'qinglong3-approval-manager',
);
assert.equal(config.http.maxConnections, 32);
assert.equal(config.http.maxConcurrentRequests, 16);
assert.match(
config.database.connection.connectionString,
/^postgresql:\/\/ql3_approval_manager:/,
);
assert.equal(config.database.connection.tls.mode, 'disable');
});
assert.throws(
() =>
loadClusterApprovalManagementProcessConfig({
QL3_APPROVAL_MANAGEMENT_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
QL3_POSTGRES_APPROVAL_MANAGER_TLS_MODE: 'disable',
}),
ClusterApprovalManagementProcessConfigError,
);
});
test('starts after dedicated readiness and identity validation then closes in order', async () => {
await tlsFixture(async (paths) => {
const order = [];
let privateKey;
let transport;
let httpClosed = 0;
let databaseClosed = 0;
const pool = {
async query() {
throw new Error('repositories must remain lazy during composition');
},
async connect() {
throw new Error('repositories must remain lazy during composition');
},
};
const runtime = await startClusterApprovalManagementProcess({
environment: enabledEnvironment(paths),
now: () => NOW_MS,
async openDatabase() {
order.push('open');
return {
pool,
async close() {
order.push('database-close');
databaseClosed += 1;
},
};
},
async assertReady(candidate) {
order.push('ready');
assert.equal(candidate, pool);
return {
ready: true,
writablePrimary: true,
serverVersionNum: 180004,
serverMajor: 18,
currentUser: 'ql3_approval_manager',
contractName: 'control-core',
contractVersion: 53,
migrationIds: ['pg-0054-approval-management-boundary'],
};
},
identities: {
async reload() {
order.push('identity');
return {
schemaVersion: 1,
generation: 3,
digest: 'identity-digest',
issuer: 'https://identity.example.test/',
audience: 'qinglong3-approval-management',
activeKeyIds: ['key-3'],
revokedKeyIds: ['key-2'],
};
},
bind() {
throw new Error('HTTP stub does not authenticate');
},
},
async startHttp(options) {
order.push('http');
privateKey = options.tls.privateKey;
transport = options.transport;
assert.ok(options.tls.clientCertificateAuthority);
assert.ok(options.tls.clientCertificateRevocationList);
return {
status: 'active',
address: { host: options.host, port: options.port },
availabilityStatus: () => 'ready',
withdraw() {},
async close() {
order.push('http-close');
httpClosed += 1;
},
};
},
});
assert.equal(runtime.status, 'active');
assert.deepEqual(order, ['open', 'ready', 'identity', 'http']);
assert.equal(typeof transport.execute, 'function');
assert.equal(privateKey.every((byte) => byte === 0), true);
assert.equal(runtime.database.contractVersion, 53);
await Promise.all([runtime.close(), runtime.close()]);
assert.equal(httpClosed, 1);
assert.equal(databaseClosed, 1);
assert.deepEqual(order.slice(-2), ['http-close', 'database-close']);
});
});
@@ -0,0 +1,231 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createApprovalRequest,
decideApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
ClusterApprovalManagementTransportAuthenticationError,
ClusterApprovalManagementTransportRequestError,
createClusterApprovalManagementTransport,
} = require('@qinglong/cluster-admin/approval-management-transport');
const ACTION = Object.freeze({
permission: 'run.start',
actionType: 'tool.invoke',
actionRef: 'tool:task-1',
actionDigest: 'a'.repeat(64),
previewDigest: 'b'.repeat(64),
});
const PRINCIPAL = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'owner-1' }),
authenticationId: 'oidc:session-1',
authenticatedAtMs: 1_000,
expiresAtMs: 20_000,
assurance: 'hardware',
});
const BASE_REQUEST = Object.freeze({
projectId: 'default',
approvalRequestId: 'approval-1',
requestId: 'approval-command-1',
auditEventId: '40000000-0000-4000-8000-000000000001',
failureAuditEventId: '40000000-0000-4000-8000-000000000002',
});
function pending() {
return createApprovalRequest({
id: 'approval-1',
projectId: 'default',
action: ACTION,
risk: 'high',
decisionMode: 'human_confirmation',
requestedBy: { type: 'agent', id: 'agent-1' },
requestedAtMs: 900,
expiresAtMs: 10_000,
requestFence: { projectVersion: 1, bindingVersion: 2 },
});
}
function inspectCommand() {
return {
schemaVersion: 1,
operation: 'approval.inspect',
request: BASE_REQUEST,
};
}
function decideCommand() {
return {
schemaVersion: 1,
operation: 'approval.decide',
request: {
...BASE_REQUEST,
expectedVersion: 1,
expectedAction: ACTION,
decisionId: 'decision-1',
decision: 'approved',
reasonCode: 'reviewed',
},
};
}
test('inspects and decides through fresh strong authentication without leaking principal facts', async () => {
const calls = [];
const failures = [];
const transport = createClusterApprovalManagementTransport({
service: {
async inspect(request, confirmAuthorization) {
calls.push(['inspect', request]);
await confirmAuthorization();
return {
request: pending(),
preview: {
title: 'Run task',
summary: 'Runs one reviewed task.',
fields: [{ kind: 'identifier', label: 'Task', value: 'task-1' }],
warnings: ['external_effect'],
},
};
},
async decide(request, confirmAuthorization) {
calls.push(['decide', request]);
await confirmAuthorization();
return {
status: 'decided',
request: decideApprovalRequest(pending(), {
expectedVersion: 1,
decisionId: request.decisionId,
decision: request.decision,
reasonCode: request.reasonCode,
principal: request.principal,
decidedAtMs: 2_000,
authorizationFence: { projectVersion: 1, bindingVersion: 2 },
}),
};
},
async recordFailure(record) {
failures.push(record);
},
},
now: () => 2_000,
});
let authenticationCalls = 0;
const authentication = {
async authenticate() {
authenticationCalls += 1;
return PRINCIPAL;
},
};
const inspected = await transport.execute(inspectCommand(), authentication);
assert.equal(authenticationCalls, 2);
assert.equal(inspected.status, 'found');
assert.equal(inspected.approval.preview.title, 'Run task');
assert.equal(inspected.approval.expectedAction.actionDigest, 'a'.repeat(64));
const decided = await transport.execute(decideCommand(), authentication);
assert.equal(authenticationCalls, 4);
assert.equal(decided.status, 'decided');
assert.equal(decided.approval.state, 'approved');
assert.equal(decided.approval.version, 2);
assert.deepEqual(calls.map(([operation]) => operation), ['inspect', 'decide']);
assert.equal(failures.length, 0);
assert.doesNotMatch(
JSON.stringify([inspected, decided]),
/authenticationId|authenticatedAtMs|assurance/,
);
});
test('records unauthenticated and reauthentication failures with schema-valid identities', async () => {
const failures = [];
const service = {
async inspect(_request, confirmAuthorization) {
await confirmAuthorization();
return null;
},
async decide() {
throw new Error('not used');
},
async recordFailure(record) {
failures.push(record);
},
};
const transport = createClusterApprovalManagementTransport({
service,
now: () => 2_000,
});
await assert.rejects(
transport.execute(inspectCommand(), {
async authenticate() {
return { ...PRINCIPAL, assurance: 'single_factor' };
},
}),
ClusterApprovalManagementTransportAuthenticationError,
);
assert.equal(failures[0].outcome, 'authentication_rejected');
assert.equal(failures[0].subject, null);
assert.equal(failures[0].authenticationId, null);
let calls = 0;
await assert.rejects(
transport.execute(inspectCommand(), {
async authenticate() {
calls += 1;
return calls === 1
? PRINCIPAL
: { ...PRINCIPAL, authenticationId: 'oidc:session-2' };
},
}),
ClusterApprovalManagementTransportAuthenticationError,
);
assert.equal(failures[1].outcome, 'denied');
assert.deepEqual(failures[1].subject, PRINCIPAL.subject);
assert.equal(failures[1].authenticationId, PRINCIPAL.authenticationId);
});
test('rejects widened or ambiguously audited commands before authentication', async () => {
let authenticationCalls = 0;
const transport = createClusterApprovalManagementTransport({
service: {
async inspect() {},
async decide() {},
async recordFailure() {},
},
});
await assert.rejects(
transport.execute(
{
...inspectCommand(),
extra: true,
},
{
async authenticate() {
authenticationCalls += 1;
return PRINCIPAL;
},
},
),
ClusterApprovalManagementTransportRequestError,
);
await assert.rejects(
transport.execute(
{
...inspectCommand(),
request: {
...BASE_REQUEST,
failureAuditEventId: BASE_REQUEST.auditEventId,
},
},
{
async authenticate() {
authenticationCalls += 1;
return PRINCIPAL;
},
},
),
ClusterApprovalManagementTransportRequestError,
);
assert.equal(authenticationCalls, 0);
});
@@ -0,0 +1,286 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createTaskDefinitionRecord,
} = require('@qinglong/runtime-core/task-definition');
const { createTriggerRecord } = require('@qinglong/runtime-core/trigger');
const {
ClusterAutomationManagementAuthorizationError,
ClusterAutomationManagementConflictError,
createClusterAutomationManagementService,
} = require('@qinglong/cluster-admin/automation-management');
const {
TaskDefinitionAdministrationMutationConflictError,
} = require('@qinglong/runtime-core/task-definition-administration');
const principal = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'operator-a' }),
authenticationId: 'session-operator-a',
authenticatedAtMs: 900,
expiresAtMs: 10_000,
assurance: 'multi_factor',
});
function taskCommand(expectedRevision = null) {
return {
projectId: 'project-a',
taskId: 'task-a',
expectedRevision,
mutationId:
expectedRevision === null
? '123e4567-e89b-42d3-a456-426614174000'
: '123e4567-e89b-42d3-a456-426614174001',
name: 'Sensitive command name',
kind: 'script',
spec: {
schema: 'qinglong/script@v1',
config: { source: 'sensitive source text' },
},
labels: { environment: 'test' },
enabled: true,
occurredAtMs: 1_000,
};
}
function triggerCommand(expectedRevision = null) {
return {
projectId: 'project-a',
triggerId: 'trigger-a',
expectedRevision,
mutationId:
expectedRevision === null
? '123e4567-e89b-42d3-a456-426614174002'
: '123e4567-e89b-42d3-a456-426614174003',
taskId: 'task-a',
taskRevision: 1,
taskContentDigest: 'a'.repeat(64),
spec: {
schema: 'qinglong/cron@v1',
config: {
expression: '*/5 * * * *',
timezone: 'UTC',
misfirePolicy: 'fire_once',
},
},
enabled: true,
occurredAtMs: 1_000,
};
}
function fixture(effect = 'allow') {
const calls = [];
const service = createClusterAutomationManagementService({
policy: {
async authorize(candidate, projectId, permission) {
calls.push(['authorize', candidate, projectId, permission]);
return {
effect,
reasons: [effect === 'allow' ? 'role_grant' : 'permission_missing'],
fence: { projectVersion: 3, bindingVersion: 5 },
};
},
},
taskDefinitions: {
async appendAuthorizedTaskDefinitionRevision(mutation) {
calls.push(['task', mutation]);
return {
status:
mutation.command.expectedRevision === null ? 'created' : 'updated',
definition: createTaskDefinitionRecord(mutation.command, 900),
};
},
async findAuthorizedCurrentTaskDefinition(read) {
calls.push(['task-inspect', read]);
return createTaskDefinitionRecord(taskCommand(), 900);
},
async listAuthorizedTaskDefinitions(read) {
calls.push(['task-list', read]);
return {
definitions: [createTaskDefinitionRecord(taskCommand(), 900)],
truncated: true,
next: { taskId: 'task-a' },
};
},
},
triggers: {
async appendAuthorizedTriggerRevision(mutation) {
calls.push(['trigger', mutation]);
return {
status:
mutation.command.expectedRevision === null ? 'created' : 'updated',
trigger: createTriggerRecord(mutation.command, 900),
};
},
async findAuthorizedCurrentTrigger(read) {
calls.push(['trigger-inspect', read]);
return createTriggerRecord(triggerCommand(), 900);
},
async listAuthorizedTriggers(read) {
calls.push(['trigger-list', read]);
return {
triggers: [createTriggerRecord(triggerCommand(), 900)],
truncated: false,
};
},
},
now: () => 1_100,
});
return { service, calls };
}
test('authorizes Task and Trigger create/update and binds allowed audit to the repository fence', async () => {
const { service, calls } = fixture();
await service.publishTask({
requestId: 'request-task-create',
command: taskCommand(),
principal,
});
await service.publishTask({
requestId: 'request-task-update',
command: taskCommand(1),
principal,
});
await service.publishTrigger({
requestId: 'request-trigger-create',
command: triggerCommand(),
principal,
});
await service.publishTrigger({
requestId: 'request-trigger-update',
command: triggerCommand(1),
principal,
});
assert.deepEqual(
calls.filter(([kind]) => kind === 'authorize').map((call) => call[3]),
['task.create', 'task.update', 'trigger.create', 'trigger.update'],
);
for (const [kind, mutation] of calls.filter(([kind]) =>
['task', 'trigger'].includes(kind),
)) {
assert.deepEqual(mutation.actor, principal.subject);
assert.deepEqual(mutation.fence, {
projectVersion: 3,
bindingVersion: 5,
});
assert.equal(mutation.audit.outcome, 'allowed');
assert.equal(mutation.audit.authenticationId, principal.authenticationId);
assert.equal(mutation.audit.eventId, mutation.command.mutationId);
assert.equal(
mutation.audit.operationId,
`${kind}.${mutation.command.expectedRevision === null ? 'create' : 'update'}`,
);
}
});
test('fails closed before mutation on weak identity or denied policy', async () => {
const denied = fixture('deny');
await assert.rejects(
denied.service.publishTask({
requestId: 'request-denied',
command: taskCommand(),
principal,
}),
ClusterAutomationManagementAuthorizationError,
);
assert.equal(denied.calls.some(([kind]) => kind === 'task'), false);
const allowed = fixture();
await assert.rejects(
allowed.service.publishTrigger({
requestId: 'request-weak',
command: triggerCommand(),
principal: { ...principal, assurance: 'single_factor' },
}),
ClusterAutomationManagementAuthorizationError,
);
assert.equal(allowed.calls.length, 0);
});
test('authorizes bounded Task and Trigger reads and binds each durable audit fence', async () => {
const { service, calls } = fixture();
const task = await service.inspectTask({
requestId: 'request-task-inspect',
auditEventId: '123e4567-e89b-42d3-a456-426614174010',
projectId: 'project-a',
taskId: 'task-a',
principal,
});
const tasks = await service.listTasks({
requestId: 'request-task-list',
auditEventId: '123e4567-e89b-42d3-a456-426614174011',
projectId: 'project-a',
limit: 1,
principal,
});
const trigger = await service.inspectTrigger({
requestId: 'request-trigger-inspect',
auditEventId: '123e4567-e89b-42d3-a456-426614174012',
projectId: 'project-a',
triggerId: 'trigger-a',
principal,
});
const triggers = await service.listTriggers({
requestId: 'request-trigger-list',
auditEventId: '123e4567-e89b-42d3-a456-426614174013',
projectId: 'project-a',
limit: 1,
after: { triggerId: 'trigger-0' },
principal,
});
assert.equal(task.taskId, 'task-a');
assert.equal(tasks.truncated, true);
assert.equal(trigger.triggerId, 'trigger-a');
assert.equal(triggers.truncated, false);
assert.deepEqual(
calls.filter(([kind]) => kind === 'authorize').map((call) => call[3]),
['task.read', 'task.read', 'trigger.read', 'trigger.read'],
);
for (const [kind, read] of calls.filter(([kind]) => kind.includes('-'))) {
assert.equal(read.audit.operationId, kind.startsWith('task') ? 'task.read' : 'trigger.read');
assert.equal(read.audit.outcome, 'allowed');
assert.deepEqual(read.actor, principal.subject);
assert.deepEqual(read.fence, { projectVersion: 3, bindingVersion: 5 });
}
});
test('maps durable mutation conflicts without leaking repository details', async () => {
const conflicting = createClusterAutomationManagementService({
policy: {
async authorize() {
return {
effect: 'allow',
reasons: ['role_grant'],
fence: { projectVersion: 1, bindingVersion: 1 },
};
},
},
taskDefinitions: {
async appendAuthorizedTaskDefinitionRevision() {
throw new TaskDefinitionAdministrationMutationConflictError();
},
async findAuthorizedCurrentTaskDefinition() { return null; },
async listAuthorizedTaskDefinitions() {
return { definitions: [], truncated: false };
},
},
triggers: {
async appendAuthorizedTriggerRevision() {
throw new Error('unused');
},
async findAuthorizedCurrentTrigger() { return null; },
async listAuthorizedTriggers() {
return { triggers: [], truncated: false };
},
},
now: () => 1_100,
});
await assert.rejects(
conflicting.publishTask({
requestId: 'request-conflict',
command: taskCommand(),
principal,
}),
ClusterAutomationManagementConflictError,
);
});
@@ -0,0 +1,374 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const {
chmodSync,
mkdtempSync,
readFileSync,
realpathSync,
rmSync,
writeFileSync,
} = require('node:fs');
const { createServer } = require('node:https');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { afterEach, test } = require('node:test');
const {
executeClusterAutomationManagementClient,
validateClusterAutomationManagementClientResult,
} = require('@qinglong/cluster-admin/automation-management-client');
const FIXTURES = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls',
);
const CLI = resolve(
__dirname,
'../dist/automation-management/automationManagementClientCli.js',
);
const temporaryDirectories = [];
const taskCommand = Object.freeze({
projectId: 'project-a',
taskId: 'task-a',
expectedRevision: null,
mutationId: '123e4567-e89b-42d3-a456-426614174000',
name: 'Sensitive task name',
kind: 'script',
spec: {
schema: 'qinglong/script@v1',
config: { source: 'sensitive script body' },
},
labels: {},
enabled: true,
occurredAtMs: 1_000,
});
const triggerCommand = Object.freeze({
projectId: 'project-a',
triggerId: 'trigger-a',
expectedRevision: null,
mutationId: '123e4567-e89b-42d3-a456-426614174002',
taskId: 'task-a',
taskRevision: 1,
taskContentDigest: 'a'.repeat(64),
spec: {
schema: 'qinglong/cron@v1',
config: {
expression: '*/5 * * * *',
timezone: 'UTC',
misfirePolicy: 'fire_once',
},
},
enabled: true,
occurredAtMs: 1_000,
});
function envelope(operation, command) {
return Object.freeze({
schemaVersion: 1,
operation,
request: Object.freeze({
requestId: `request-${operation}`,
command,
}),
});
}
function taskResult(overrides = {}) {
return {
schemaVersion: 1,
operation: 'task.publish',
status: 'created',
task: {
projectId: 'project-a',
taskId: 'task-a',
revision: 1,
kind: 'script',
enabled: true,
contentDigest: 'b'.repeat(64),
updatedAtMs: 1_001,
...overrides,
},
};
}
function triggerResult(overrides = {}) {
return {
schemaVersion: 1,
operation: 'trigger.publish',
status: 'created',
trigger: {
projectId: 'project-a',
triggerId: 'trigger-a',
revision: 1,
taskId: 'task-a',
taskRevision: 1,
taskContentDigest: 'a'.repeat(64),
enabled: true,
contentDigest: 'c'.repeat(64),
updatedAtMs: 1_001,
...overrides,
},
};
}
function privateWrite(filePath, value) {
writeFileSync(filePath, value, { mode: 0o600 });
chmodSync(filePath, 0o600);
}
function clientFiles(port, command) {
const directory = realpathSync(
mkdtempSync(join(tmpdir(), 'ql3-automation-client-')),
);
temporaryDirectories.push(directory);
const paths = {
configFile: join(directory, 'client.json'),
commandFile: join(directory, 'command.json'),
assertionFile: join(directory, 'assertion.jwt'),
};
const caFile = join(directory, 'ca.crt');
const clientCertificateFile = join(directory, 'client.crt');
const clientPrivateKeyFile = join(directory, 'client.key');
privateWrite(caFile, readFileSync(join(FIXTURES, 'ca-cert.pem')));
privateWrite(
clientCertificateFile,
readFileSync(join(FIXTURES, 'client-cert.pem')),
);
privateWrite(
clientPrivateKeyFile,
readFileSync(join(FIXTURES, 'client-key.pem')),
);
privateWrite(
paths.configFile,
`${JSON.stringify({
schemaVersion: 1,
endpoint: `https://localhost:${port}/api/v3/automations/management`,
servername: 'localhost',
caFile,
clientCertificateFile,
clientPrivateKeyFile,
requestTimeoutMs: 2_000,
})}\n`,
);
privateWrite(paths.commandFile, `${JSON.stringify(command)}\n`);
privateWrite(
paths.assertionFile,
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl',
);
return paths;
}
function listen(server) {
return new Promise((resolvePromise, reject) => {
server.once('error', reject);
server.listen(0, 'localhost', () => resolvePromise(server.address()));
});
}
function close(server) {
return new Promise((resolvePromise, reject) => {
server.close((error) => (error ? reject(error) : resolvePromise()));
});
}
afterEach(() => {
for (const directory of temporaryDirectories.splice(0)) {
rmSync(directory, { recursive: true, force: true });
}
});
test('validates Task and Trigger low-sensitive results against the request fence', () => {
const task = validateClusterAutomationManagementClientResult(
taskResult(),
envelope('task.publish', taskCommand),
);
const trigger = validateClusterAutomationManagementClientResult(
triggerResult(),
envelope('trigger.publish', triggerCommand),
);
assert.equal(task.task.taskId, 'task-a');
assert.equal(trigger.trigger.triggerId, 'trigger-a');
assert.doesNotMatch(
JSON.stringify([task, trigger]),
/Sensitive|script body|expression|mutationId|authenticationId/,
);
assert.throws(() =>
validateClusterAutomationManagementClientResult(
taskResult({ projectId: 'project-b' }),
envelope('task.publish', taskCommand),
),
);
assert.throws(() =>
validateClusterAutomationManagementClientResult(
{ ...triggerResult(), credential: 'must-not-leak' },
envelope('trigger.publish', triggerCommand),
),
);
});
test('validates bounded inspect/list results, absent state and stable cursors', () => {
const taskInspectCommand = {
schemaVersion: 1,
operation: 'task.inspect',
request: {
requestId: 'request-task-inspect',
auditEventId: '123e4567-e89b-42d3-a456-426614174010',
projectId: 'project-a',
taskId: 'task-a',
},
};
const taskInspect = validateClusterAutomationManagementClientResult(
{
schemaVersion: 1,
operation: 'task.inspect',
status: 'found',
task: taskResult().task,
},
taskInspectCommand,
);
assert.equal(taskInspect.task.taskId, 'task-a');
const triggerInspectCommand = {
schemaVersion: 1,
operation: 'trigger.inspect',
request: {
requestId: 'request-trigger-inspect',
auditEventId: '123e4567-e89b-42d3-a456-426614174011',
projectId: 'project-a',
triggerId: 'trigger-a',
},
};
const absent = validateClusterAutomationManagementClientResult(
{
schemaVersion: 1,
operation: 'trigger.inspect',
status: 'absent',
trigger: null,
},
triggerInspectCommand,
);
assert.equal(absent.trigger, null);
const taskListCommand = {
schemaVersion: 1,
operation: 'task.list',
request: {
requestId: 'request-task-list',
auditEventId: '123e4567-e89b-42d3-a456-426614174012',
projectId: 'project-a',
limit: 1,
after: { taskId: 'task-0' },
},
};
const taskList = validateClusterAutomationManagementClientResult(
{
schemaVersion: 1,
operation: 'task.list',
tasks: [taskResult().task],
truncated: true,
next: { taskId: 'task-a' },
},
taskListCommand,
);
assert.equal(taskList.next.taskId, 'task-a');
assert.throws(() =>
validateClusterAutomationManagementClientResult(
{
schemaVersion: 1,
operation: 'task.list',
tasks: [{ ...taskResult().task, spec: { secret: true } }],
truncated: false,
next: null,
},
taskListCommand,
),
);
assert.throws(() =>
validateClusterAutomationManagementClientResult(
{
schemaVersion: 1,
operation: 'task.list',
tasks: [taskResult().task],
truncated: true,
next: { taskId: 'task-b' },
},
taskListCommand,
),
);
});
test('sends one mTLS 1.3 automation command and validates its response', async () => {
let observed;
const server = createServer(
{
key: readFileSync(join(FIXTURES, 'server-key.pem')),
cert: readFileSync(join(FIXTURES, 'server-cert.pem')),
ca: readFileSync(join(FIXTURES, 'ca-cert.pem')),
requestCert: true,
rejectUnauthorized: true,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
},
(request, response) => {
const chunks = [];
request.on('data', (chunk) => chunks.push(chunk));
request.once('end', () => {
observed = {
method: request.method,
path: request.url,
authorization: request.headers.authorization,
authorized: request.socket.authorized,
protocol: request.socket.getProtocol(),
command: JSON.parse(Buffer.concat(chunks).toString('utf8')),
};
const body = Buffer.from(
JSON.stringify({
schemaVersion: 1,
requestId: 'http-request-1',
result: taskResult(),
}),
);
response.writeHead(200, {
'content-type': 'application/json; charset=utf-8',
'content-length': String(body.length),
});
response.end(body);
});
},
);
const address = await listen(server);
try {
const command = envelope('task.publish', taskCommand);
const result = await executeClusterAutomationManagementClient(
clientFiles(address.port, command),
);
assert.equal(result.requestId, 'http-request-1');
assert.equal(result.result.task.taskId, 'task-a');
assert.deepEqual(observed, {
method: 'POST',
path: '/api/v3/automations/management',
authorization:
'Bearer eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl',
authorized: true,
protocol: 'TLSv1.3',
command,
});
} finally {
await close(server);
}
});
test('CLI exposes only private file paths and stable low-sensitive errors', () => {
const help = spawnSync(process.execPath, [CLI, '--help'], {
encoding: 'utf8',
});
assert.equal(help.status, 0);
assert.match(help.stdout, /^Usage: ql3-automation-client /);
assert.doesNotMatch(help.stdout, /token|secret|command body/i);
const invalid = spawnSync(process.execPath, [CLI, '--assertion=value'], {
encoding: 'utf8',
});
assert.equal(invalid.status, 64);
assert.match(invalid.stderr, /USAGE_INVALID/);
assert.doesNotMatch(invalid.stderr, /assertion=value/);
});
@@ -0,0 +1,128 @@
const assert = require('node:assert/strict');
const { readFileSync } = require('node:fs');
const { request: httpsRequest } = require('node:https');
const { resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterAutomationManagementConflictError,
} = require('@qinglong/cluster-admin/automation-management');
const {
startClusterAutomationManagementHttp,
} = require('@qinglong/cluster-admin/automation-management-http');
const SERVER_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-key.pem',
);
const SERVER_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem',
);
const PATH = '/api/v3/automations/management';
function post(port, path = PATH) {
const body = Buffer.from(
JSON.stringify({
schemaVersion: 1,
operation: 'task.publish',
request: { requestId: 'request-task', command: {} },
}),
);
return new Promise((resolvePromise, reject) => {
const outgoing = httpsRequest(
{
host: '127.0.0.1',
port,
path,
method: 'POST',
rejectUnauthorized: false,
agent: false,
headers: {
authorization: 'Bearer assertion',
'content-type': 'application/json',
'content-length': String(body.length),
},
},
(incoming) => {
const chunks = [];
incoming.on('data', (chunk) => chunks.push(chunk));
incoming.once('end', () => {
const bytes = Buffer.concat(chunks);
resolvePromise({
statusCode: incoming.statusCode,
body: bytes.length ? JSON.parse(bytes.toString('utf8')) : null,
});
});
},
);
outgoing.once('error', reject);
outgoing.end(body);
});
}
test('serves only the automation path and maps automation conflicts to low-sensitive HTTP facts', async () => {
const privateKey = Buffer.from(readFileSync(SERVER_KEY));
const application = await startClusterAutomationManagementHttp({
host: '127.0.0.1',
port: 0,
tls: {
privateKey,
certificate: Buffer.from(readFileSync(SERVER_CERT)),
},
identities: {
async reload() {
return {
schemaVersion: 1,
generation: 1,
digest: 'digest',
issuer: 'https://identity.example.test/',
audience: 'qinglong3-automation-management',
activeKeyIds: ['key-1'],
revokedKeyIds: [],
};
},
bind() {
return {
async authenticate() {
return {
subject: { type: 'user', id: 'operator-a' },
authenticationId: 'session-operator-a',
authenticatedAtMs: 900,
expiresAtMs: 10_000,
assurance: 'multi_factor',
};
},
};
},
},
transport: {
async execute(_command, authentication) {
await authentication.authenticate();
throw new ClusterAutomationManagementConflictError();
},
},
now: () => 1_000,
});
try {
assert.equal(privateKey.every((value) => value === 0), true);
const conflict = await post(application.address.port);
assert.deepEqual(conflict, {
statusCode: 409,
body: {
schemaVersion: 1,
requestId: conflict.body.requestId,
error: { code: 'conflict' },
},
});
assert.match(conflict.body.requestId, /^[0-9a-f-]{36}$/);
const absent = await post(
application.address.port,
'/api/v3/plugin-packages/management',
);
assert.equal(absent.statusCode, 404);
assert.equal(absent.body.error.code, 'not_found');
} finally {
await application.close();
}
});
@@ -0,0 +1,506 @@
const assert = require('node:assert/strict');
const {
chmodSync,
mkdtempSync,
readFileSync,
realpathSync,
rmSync,
writeFileSync,
} = require('node:fs');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { test } = require('node:test');
const { ProjectPolicyEngine } = require('@qinglong/runtime-core/project-policy');
const {
PostgresProjectPolicyRepository,
PostgresTaskDefinitionAdministrationRepository,
PostgresTriggerAdministrationRepository,
assertPostgresAutomationManagerSchemaReady,
createPostgresDatabaseOpener,
} = require('@qinglong/cluster-postgres/automation-manager');
const {
runPostgresMigrations,
} = require('@qinglong/cluster-postgres/migration');
const {
createClusterAutomationManagementService,
} = require('@qinglong/cluster-admin/automation-management');
const {
executeClusterAutomationManagementClient,
} = require('@qinglong/cluster-admin/automation-management-client');
const {
startClusterAutomationManagementHttp,
} = require('@qinglong/cluster-admin/automation-management-http');
const {
createClusterAutomationManagementTransport,
} = require('@qinglong/cluster-admin/automation-management-transport');
const MIGRATION_URL = process.env.QL3_TEST_POSTGRES_URL;
const AUTOMATION_URL =
process.env.QL3_TEST_POSTGRES_AUTOMATION_MANAGER_URL;
const FIXTURES = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls',
);
function opener(role, connectionString, applicationName) {
return createPostgresDatabaseOpener({
role,
connection: {
connectionString,
tls: { mode: 'disable' },
},
pool: { maxConnections: 2, applicationName },
onPoolError(error) {
throw error;
},
});
}
const principal = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'automation-operator' }),
authenticationId: 'oidc:automation-live-contract',
authenticatedAtMs: 1,
expiresAtMs: 4_102_444_800_000,
assurance: 'hardware',
});
const identities = Object.freeze({
async reload() {
return Object.freeze({
schemaVersion: 1,
generation: 1,
digest: 'automation-live-contract',
issuer: 'https://identity.example.test/',
audience: 'qinglong3-automation-management',
activeKeyIds: Object.freeze(['automation-live-key']),
revokedKeyIds: Object.freeze([]),
});
},
bind() {
return Object.freeze({
async authenticate() {
return principal;
},
});
},
});
function privateWrite(filePath, value) {
writeFileSync(filePath, value, { mode: 0o600 });
chmodSync(filePath, 0o600);
}
function clientFiles(port, command, directories) {
const directory = realpathSync(
mkdtempSync(join(tmpdir(), 'ql3-automation-pg-client-')),
);
directories.push(directory);
const caFile = join(directory, 'ca.crt');
const clientCertificateFile = join(directory, 'client.crt');
const clientPrivateKeyFile = join(directory, 'client.key');
const paths = {
configFile: join(directory, 'client.json'),
commandFile: join(directory, 'command.json'),
assertionFile: join(directory, 'assertion.jwt'),
};
privateWrite(caFile, readFileSync(join(FIXTURES, 'ca-cert.pem')));
privateWrite(
clientCertificateFile,
readFileSync(join(FIXTURES, 'client-cert.pem')),
);
privateWrite(
clientPrivateKeyFile,
readFileSync(join(FIXTURES, 'client-key.pem')),
);
privateWrite(
paths.configFile,
`${JSON.stringify({
schemaVersion: 1,
endpoint: `https://localhost:${port}/api/v3/automations/management`,
servername: 'localhost',
caFile,
clientCertificateFile,
clientPrivateKeyFile,
requestTimeoutMs: 5_000,
})}\n`,
);
privateWrite(paths.commandFile, `${JSON.stringify(command)}\n`);
privateWrite(
paths.assertionFile,
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl',
);
return paths;
}
function envelope(operation, requestId, command) {
return Object.freeze({
schemaVersion: 1,
operation,
request: Object.freeze({ requestId, command }),
});
}
function readEnvelope(operation, request) {
return Object.freeze({
schemaVersion: 1,
operation,
request: Object.freeze(request),
});
}
function taskCommand(projectId, revision, suffix, value) {
return Object.freeze({
projectId,
taskId: 'managed-task',
expectedRevision: revision,
mutationId: `123e4567-e89b-42d3-a456-426614176${suffix}`,
name: 'Managed Task',
kind: 'command',
spec: Object.freeze({
schema: 'qinglong/command@v1',
config: Object.freeze({
command: Object.freeze({
kind: 'argv',
file: '/bin/echo',
args: Object.freeze([value]),
}),
}),
}),
labels: Object.freeze({}),
enabled: true,
occurredAtMs: 1_000 + Number(suffix),
});
}
function triggerCommand(projectId, revision, task, suffix) {
return Object.freeze({
projectId,
triggerId: 'managed-trigger',
expectedRevision: revision,
mutationId: `123e4567-e89b-42d3-a456-426614177${suffix}`,
taskId: task.taskId,
taskRevision: task.revision,
taskContentDigest: task.contentDigest,
spec: Object.freeze({
schema: 'qinglong/cron@v1',
config: Object.freeze({
expression: '*/5 * * * *',
timezone: 'UTC',
misfirePolicy: 'skip',
}),
}),
enabled: true,
occurredAtMs: 2_000 + Number(suffix),
});
}
function buildTransport(database) {
return createClusterAutomationManagementTransport({
service: createClusterAutomationManagementService({
policy: new ProjectPolicyEngine(
new PostgresProjectPolicyRepository(database.pool),
),
taskDefinitions: new PostgresTaskDefinitionAdministrationRepository(
database.pool,
),
triggers: new PostgresTriggerAdministrationRepository(database.pool),
}),
});
}
async function startManager(transport, sequence) {
return startClusterAutomationManagementHttp({
host: 'localhost',
port: 0,
tls: {
privateKey: Buffer.from(readFileSync(join(FIXTURES, 'server-key.pem'))),
certificate: Buffer.from(
readFileSync(join(FIXTURES, 'server-cert.pem')),
),
clientCertificateAuthority: Buffer.from(
readFileSync(join(FIXTURES, 'ca-cert.pem')),
),
clientCertificateRevocationList: Buffer.from(
readFileSync(join(FIXTURES, 'empty-crl.pem')),
),
},
transport,
identities,
createRequestId: () => `manager-${sequence.value++}`,
});
}
if (!MIGRATION_URL || !AUTOMATION_URL) {
test('automation PostgreSQL HTTPS integration requires migration and automation-manager URLs', {
skip: true,
});
} else {
test(
'two HTTPS managers converge concurrent publish and commit-response loss on real PostgreSQL',
{ timeout: 60_000 },
async () => {
const directories = [];
const sequence = { value: 1 };
const migration = await opener(
'migration',
MIGRATION_URL,
'ql3-automation-live-migration',
)();
const firstDatabase = await opener(
'automation-manager',
AUTOMATION_URL,
'ql3-automation-live-first',
)();
const secondDatabase = await opener(
'automation-manager',
AUTOMATION_URL,
'ql3-automation-live-second',
)();
let first;
let second;
let responseLoss;
try {
await runPostgresMigrations({ pool: migration.pool });
const readiness = await Promise.all([
assertPostgresAutomationManagerSchemaReady(firstDatabase.pool),
assertPostgresAutomationManagerSchemaReady(secondDatabase.pool),
]);
assert.deepEqual(
readiness.map((entry) => entry.currentUser),
['ql3_automation_manager', 'ql3_automation_manager'],
);
const observed = await migration.pool.query(
`SELECT floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint
AS "observedAtMs"`,
);
const occurredAtMs = Number(observed.rows[0].observedAtMs);
const projectId = `automation-https-${process.pid}`;
await migration.pool.query(
`DELETE FROM "ql3"."projects" WHERE id = $1`,
[projectId],
);
await migration.pool.query(
`INSERT INTO "ql3"."projects" (
id, name, slug, status, version, created_at_ms, updated_at_ms
) VALUES ($1, 'Automation HTTPS', $1, 'active', 1, $2, $2)`,
[projectId, occurredAtMs],
);
await migration.pool.query(
`INSERT INTO "ql3"."project_role_bindings" (
project_id, subject_type, subject_id, version, state, role,
mutation_id, changed_by_type, changed_by_id, created_at_ms
) VALUES ($1, 'user', $2, 1, 'active', 'owner', $3,
'user', $2, $4)`,
[
projectId,
principal.subject.id,
'123e4567-e89b-42d3-a456-426614176000',
occurredAtMs,
],
);
const firstTransport = buildTransport(firstDatabase);
const secondTransport = buildTransport(secondDatabase);
first = await startManager(firstTransport, sequence);
second = await startManager(secondTransport, sequence);
const createTask = envelope(
'task.publish',
'concurrent-task-create',
taskCommand(projectId, null, '001', 'v1'),
);
const concurrent = await Promise.all([
executeClusterAutomationManagementClient(
clientFiles(first.address.port, createTask, directories),
),
executeClusterAutomationManagementClient(
clientFiles(second.address.port, createTask, directories),
),
]);
assert.deepEqual(
concurrent.map((entry) => entry.result.status).sort(),
['created', 'existing'],
);
const taskV1 = concurrent[0].result.task;
let dropCommittedResponse = true;
responseLoss = await startManager(
Object.freeze({
async execute(command, authentication) {
const result = await firstTransport.execute(
command,
authentication,
);
if (dropCommittedResponse) {
dropCommittedResponse = false;
throw new Error('simulated post-commit response loss');
}
return result;
},
}),
sequence,
);
const updateTask = envelope(
'task.publish',
'response-loss-task-update',
taskCommand(projectId, 1, '002', 'v2'),
);
await assert.rejects(
executeClusterAutomationManagementClient(
clientFiles(responseLoss.address.port, updateTask, directories),
),
(error) =>
error?.statusCode === 500 &&
error?.responseCode === 'internal_error',
);
const converged = await executeClusterAutomationManagementClient(
clientFiles(second.address.port, updateTask, directories),
);
assert.equal(converged.result.status, 'existing');
assert.equal(converged.result.task.revision, 2);
const createTrigger = envelope(
'trigger.publish',
'trigger-create-v2',
triggerCommand(projectId, null, converged.result.task, '001'),
);
const triggerV1 = await executeClusterAutomationManagementClient(
clientFiles(first.address.port, createTrigger, directories),
);
assert.equal(triggerV1.result.status, 'created');
assert.equal(triggerV1.result.trigger.taskRevision, 2);
const taskV3 = await executeClusterAutomationManagementClient(
clientFiles(
second.address.port,
envelope(
'task.publish',
'task-update-v3',
taskCommand(projectId, 2, '003', 'v3'),
),
directories,
),
);
const triggerV2 = await executeClusterAutomationManagementClient(
clientFiles(
first.address.port,
envelope(
'trigger.publish',
'trigger-repin-v3',
triggerCommand(projectId, 1, taskV3.result.task, '002'),
),
directories,
),
);
assert.equal(triggerV2.result.status, 'updated');
assert.equal(triggerV2.result.trigger.taskRevision, 3);
const taskInspection = await executeClusterAutomationManagementClient(
clientFiles(
second.address.port,
readEnvelope('task.inspect', {
requestId: 'task-inspect-v3',
auditEventId: '123e4567-e89b-42d3-a456-426614178001',
projectId,
taskId: 'managed-task',
}),
directories,
),
);
assert.equal(taskInspection.result.status, 'found');
assert.equal(taskInspection.result.task.revision, 3);
const taskPage = await executeClusterAutomationManagementClient(
clientFiles(
first.address.port,
readEnvelope('task.list', {
requestId: 'task-list-v3',
auditEventId: '123e4567-e89b-42d3-a456-426614178002',
projectId,
limit: 1,
}),
directories,
),
);
assert.equal(taskPage.result.tasks.length, 1);
assert.equal(taskPage.result.next, null);
const triggerInspection =
await executeClusterAutomationManagementClient(
clientFiles(
first.address.port,
readEnvelope('trigger.inspect', {
requestId: 'trigger-inspect-v2',
auditEventId: '123e4567-e89b-42d3-a456-426614178003',
projectId,
triggerId: 'managed-trigger',
}),
directories,
),
);
assert.equal(triggerInspection.result.trigger.revision, 2);
const triggerPage = await executeClusterAutomationManagementClient(
clientFiles(
second.address.port,
readEnvelope('trigger.list', {
requestId: 'trigger-list-v2',
auditEventId: '123e4567-e89b-42d3-a456-426614178004',
projectId,
limit: 1,
}),
directories,
),
);
assert.equal(triggerPage.result.triggers.length, 1);
assert.equal(triggerPage.result.next, null);
assert.doesNotMatch(
JSON.stringify([
taskInspection.result,
taskPage.result,
triggerInspection.result,
triggerPage.result,
]),
/Managed Task|\/bin\/echo|expression|authenticationId|mutationId/,
);
const durable = await migration.pool.query(
`SELECT
(SELECT count(*)::integer FROM "ql3"."task_definition_revisions"
WHERE project_id = $1 AND task_id = 'managed-task') AS "taskRevisions",
(SELECT current_revision FROM "ql3"."task_definitions"
WHERE project_id = $1 AND task_id = 'managed-task') AS "taskHead",
(SELECT count(*)::integer FROM "ql3"."trigger_revisions"
WHERE project_id = $1 AND trigger_id = 'managed-trigger') AS "triggerRevisions",
(SELECT current_revision FROM "ql3"."triggers"
WHERE project_id = $1 AND trigger_id = 'managed-trigger') AS "triggerHead",
(SELECT count(*)::integer FROM "ql3"."security_audit_events"
WHERE project_id = $1 AND outcome = 'allowed') AS "allowedAudits"`,
[projectId],
);
assert.deepEqual(durable.rows, [
{
taskRevisions: 3,
taskHead: 3,
triggerRevisions: 2,
triggerHead: 2,
allowedAudits: 9,
},
]);
assert.equal(taskV1.revision, 1);
} finally {
await Promise.allSettled([
responseLoss?.close(),
first?.close(),
second?.close(),
]);
await Promise.allSettled([
firstDatabase.close(),
secondDatabase.close(),
migration.close(),
]);
for (const directory of directories) {
rmSync(directory, { recursive: true, force: true });
}
}
},
);
}
@@ -0,0 +1,211 @@
const assert = require('node:assert/strict');
const { readFile, writeFile, mkdtemp, rm } = require('node:fs/promises');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterAutomationManagementProcessConfigError,
loadClusterAutomationManagementProcessConfig,
startClusterAutomationManagementProcess,
} = require('@qinglong/cluster-admin/automation-management-process');
const FIXTURES = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls',
);
const NOW_MS = Date.UTC(2030, 0, 1);
function enabledEnvironment(paths, overrides = {}) {
return {
QL3_AUTOMATION_MANAGEMENT_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
QL3_AUTOMATION_MANAGEMENT_HOST: '127.0.0.1',
QL3_AUTOMATION_MANAGEMENT_PORT: '8445',
QL3_AUTOMATION_MANAGEMENT_TLS_CERT_FILE: paths.certificateFile,
QL3_AUTOMATION_MANAGEMENT_TLS_KEY_FILE: paths.privateKeyFile,
QL3_AUTOMATION_MANAGEMENT_CLIENT_CA_FILE:
paths.clientCertificateAuthorityFile,
QL3_AUTOMATION_MANAGEMENT_CLIENT_CRL_FILE:
paths.clientCertificateRevocationListFile,
QL3_AUTOMATION_MANAGEMENT_IDENTITY_KEYSET_FILE: paths.identityKeysetFile,
QL3_POSTGRES_AUTOMATION_MANAGER_URL:
'postgresql://ql3_automation_manager:secret@postgres.example.test/ql3',
QL3_POSTGRES_AUTOMATION_MANAGER_TLS_MODE: 'disable',
QL3_POSTGRES_AUTOMATION_MANAGER_ALLOW_INSECURE: 'true',
...overrides,
};
}
async function tlsFixture(run) {
const directory = await mkdtemp(join(tmpdir(), 'ql3-automation-manager-'));
const paths = {
certificateFile: join(directory, 'tls.crt'),
privateKeyFile: join(directory, 'tls.key'),
clientCertificateAuthorityFile: join(directory, 'client-ca.crt'),
clientCertificateRevocationListFile: join(directory, 'client.crl'),
identityKeysetFile: join(directory, 'keyset.json'),
};
try {
await writeFile(
paths.certificateFile,
await readFile(join(FIXTURES, 'server-cert.pem')),
{ mode: 0o644 },
);
await writeFile(
paths.privateKeyFile,
await readFile(join(FIXTURES, 'server-key.pem')),
{ mode: 0o640 },
);
await writeFile(
paths.clientCertificateAuthorityFile,
await readFile(join(FIXTURES, 'ca-cert.pem')),
{ mode: 0o644 },
);
await writeFile(
paths.clientCertificateRevocationListFile,
await readFile(join(FIXTURES, 'empty-crl.pem')),
{ mode: 0o644 },
);
await writeFile(paths.identityKeysetFile, '{}\n', { mode: 0o644 });
return await run(paths);
} finally {
await rm(directory, { recursive: true, force: true });
}
}
test('disabled automation manager opens no PostgreSQL or file authority', async () => {
let opened = 0;
const runtime = await startClusterAutomationManagementProcess({
environment: { QL3_AUTOMATION_MANAGEMENT_ENABLED: 'false' },
async openDatabase() {
opened += 1;
throw new Error('must not open');
},
});
assert.equal(runtime.status, 'disabled');
assert.equal(opened, 0);
await runtime.close();
});
test('loads bounded low-footprint automation-only HTTPS and PostgreSQL configuration', async () => {
await tlsFixture(async (paths) => {
const config = loadClusterAutomationManagementProcessConfig(
enabledEnvironment(paths),
);
assert.equal(config.enabled, true);
assert.equal(config.port, 8445);
assert.equal(config.database.pool.maxConnections, 2);
assert.equal(
config.database.pool.applicationName,
'qinglong3-automation-manager',
);
assert.equal(config.http.maxConnections, 32);
assert.equal(config.http.maxConcurrentRequests, 16);
assert.match(
config.database.connection.connectionString,
/^postgresql:\/\/ql3_automation_manager:/,
);
assert.equal(config.database.connection.tls.mode, 'disable');
});
assert.throws(
() =>
loadClusterAutomationManagementProcessConfig({
QL3_AUTOMATION_MANAGEMENT_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
QL3_POSTGRES_AUTOMATION_MANAGER_TLS_MODE: 'disable',
}),
ClusterAutomationManagementProcessConfigError,
);
});
test('starts only after automation readiness and identity validation then closes in order', async () => {
await tlsFixture(async (paths) => {
const order = [];
let privateKey;
let transport;
let httpClosed = 0;
let databaseClosed = 0;
const pool = {
async query() {
throw new Error('repositories must remain lazy during composition');
},
async connect() {
throw new Error('repositories must remain lazy during composition');
},
};
const runtime = await startClusterAutomationManagementProcess({
environment: enabledEnvironment(paths),
now: () => NOW_MS,
async openDatabase() {
order.push('open');
return {
pool,
async close() {
order.push('database-close');
databaseClosed += 1;
},
};
},
async assertReady(candidate) {
order.push('ready');
assert.equal(candidate, pool);
return {
ready: true,
writablePrimary: true,
serverVersionNum: 180004,
serverMajor: 18,
currentUser: 'ql3_automation_manager',
contractName: 'control-core',
contractVersion: 53,
migrationIds: [
'pg-0052-automation-management-identity-keyset-ledger',
],
};
},
identities: {
async reload() {
order.push('identity');
return {
schemaVersion: 1,
generation: 3,
digest: 'identity-digest',
issuer: 'https://identity.example.test/',
audience: 'qinglong3-automation-management',
activeKeyIds: ['key-3'],
revokedKeyIds: ['key-2'],
};
},
bind() {
throw new Error('HTTP stub does not authenticate');
},
},
async startHttp(options) {
order.push('http');
privateKey = options.tls.privateKey;
transport = options.transport;
assert.ok(options.tls.clientCertificateAuthority);
assert.ok(options.tls.clientCertificateRevocationList);
return {
status: 'active',
address: { host: options.host, port: options.port },
availabilityStatus: () => 'ready',
withdraw() {},
async close() {
order.push('http-close');
httpClosed += 1;
},
};
},
});
assert.equal(runtime.status, 'active');
assert.deepEqual(order, ['open', 'ready', 'identity', 'http']);
assert.equal(typeof transport.execute, 'function');
assert.equal(privateKey.every((byte) => byte === 0), true);
assert.equal(runtime.database.contractVersion, 53);
await Promise.all([runtime.close(), runtime.close()]);
assert.equal(httpClosed, 1);
assert.equal(databaseClosed, 1);
assert.deepEqual(order.slice(-2), ['http-close', 'database-close']);
});
});
@@ -0,0 +1,234 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createTaskDefinitionRecord,
} = require('@qinglong/runtime-core/task-definition');
const { createTriggerRecord } = require('@qinglong/runtime-core/trigger');
const {
ClusterAutomationManagementTransportAuthenticationError,
ClusterAutomationManagementTransportRequestError,
createClusterAutomationManagementTransport,
} = require('@qinglong/cluster-admin/automation-management-transport');
const principal = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'operator-a' }),
authenticationId: 'session-operator-a',
authenticatedAtMs: 900,
expiresAtMs: 10_000,
assurance: 'hardware',
});
const taskCommand = Object.freeze({
projectId: 'project-a',
taskId: 'task-a',
expectedRevision: null,
mutationId: '123e4567-e89b-42d3-a456-426614174000',
name: 'Sensitive task name',
kind: 'script',
spec: {
schema: 'qinglong/script@v1',
config: { source: 'sensitive script body' },
},
labels: {},
enabled: true,
occurredAtMs: 1_000,
});
const triggerCommand = Object.freeze({
projectId: 'project-a',
triggerId: 'trigger-a',
expectedRevision: null,
mutationId: '123e4567-e89b-42d3-a456-426614174002',
taskId: 'task-a',
taskRevision: 1,
taskContentDigest: 'a'.repeat(64),
spec: {
schema: 'qinglong/cron@v1',
config: {
expression: '*/5 * * * *',
timezone: 'UTC',
misfirePolicy: 'fire_once',
},
},
enabled: true,
occurredAtMs: 1_000,
});
test('routes strong User Task/Trigger publications and returns only low-sensitive summaries', async () => {
const calls = [];
const transport = createClusterAutomationManagementTransport({
service: {
async publishTask(request) {
calls.push(['task', request]);
return {
status: 'created',
definition: createTaskDefinitionRecord(request.command, 900),
};
},
async publishTrigger(request) {
calls.push(['trigger', request]);
return {
status: 'created',
trigger: createTriggerRecord(request.command, 900),
};
},
async inspectTask(request) {
calls.push(['task-inspect', request]);
return createTaskDefinitionRecord(taskCommand, 900);
},
async listTasks(request) {
calls.push(['task-list', request]);
return {
definitions: [createTaskDefinitionRecord(taskCommand, 900)],
truncated: true,
next: { taskId: 'task-a' },
};
},
async inspectTrigger(request) {
calls.push(['trigger-inspect', request]);
return null;
},
async listTriggers(request) {
calls.push(['trigger-list', request]);
return {
triggers: [createTriggerRecord(triggerCommand, 900)],
truncated: false,
};
},
},
now: () => 1_100,
});
const authentication = { async authenticate() { return principal; } };
const task = await transport.execute(
{
schemaVersion: 1,
operation: 'task.publish',
request: { requestId: 'request-task', command: taskCommand },
},
authentication,
);
const trigger = await transport.execute(
{
schemaVersion: 1,
operation: 'trigger.publish',
request: { requestId: 'request-trigger', command: triggerCommand },
},
authentication,
);
const taskInspection = await transport.execute(
{
schemaVersion: 1,
operation: 'task.inspect',
request: {
requestId: 'request-task-inspect',
auditEventId: '123e4567-e89b-42d3-a456-426614174010',
projectId: 'project-a',
taskId: 'task-a',
},
},
authentication,
);
const taskList = await transport.execute(
{
schemaVersion: 1,
operation: 'task.list',
request: {
requestId: 'request-task-list',
auditEventId: '123e4567-e89b-42d3-a456-426614174011',
projectId: 'project-a',
limit: 1,
},
},
authentication,
);
const triggerInspection = await transport.execute(
{
schemaVersion: 1,
operation: 'trigger.inspect',
request: {
requestId: 'request-trigger-inspect',
auditEventId: '123e4567-e89b-42d3-a456-426614174012',
projectId: 'project-a',
triggerId: 'trigger-a',
},
},
authentication,
);
const triggerList = await transport.execute(
{
schemaVersion: 1,
operation: 'trigger.list',
request: {
requestId: 'request-trigger-list',
auditEventId: '123e4567-e89b-42d3-a456-426614174013',
projectId: 'project-a',
limit: 1,
},
},
authentication,
);
assert.deepEqual(calls.map(([kind]) => kind), [
'task',
'trigger',
'task-inspect',
'task-list',
'trigger-inspect',
'trigger-list',
]);
assert.deepEqual(calls.map(([, request]) => request.principal), [
principal,
principal,
principal,
principal,
principal,
principal,
]);
assert.equal(task.task.contentDigest.length, 64);
assert.equal(trigger.trigger.taskContentDigest, 'a'.repeat(64));
assert.equal(taskInspection.status, 'found');
assert.equal(taskList.next.taskId, 'task-a');
assert.equal(triggerInspection.status, 'absent');
assert.equal(triggerList.next, null);
const serialized = JSON.stringify([
task,
trigger,
taskInspection,
taskList,
triggerInspection,
triggerList,
]);
assert.doesNotMatch(serialized, /Sensitive|script body|expression|authenticationId|mutationId/);
});
test('rejects malformed envelopes and weak identities before service authority', async () => {
let calls = 0;
const transport = createClusterAutomationManagementTransport({
service: {
async publishTask() { calls += 1; },
async publishTrigger() { calls += 1; },
async inspectTask() { calls += 1; },
async listTasks() { calls += 1; },
async inspectTrigger() { calls += 1; },
async listTriggers() { calls += 1; },
},
now: () => 1_100,
});
await assert.rejects(
transport.execute(
{ schemaVersion: 1, operation: 'task.publish', request: {}, extra: true },
{ async authenticate() { return principal; } },
),
ClusterAutomationManagementTransportRequestError,
);
await assert.rejects(
transport.execute(
{
schemaVersion: 1,
operation: 'task.publish',
request: { requestId: 'request-task', command: taskCommand },
},
{ async authenticate() { return { ...principal, assurance: 'single_factor' }; } },
),
ClusterAutomationManagementTransportAuthenticationError,
);
assert.equal(calls, 0);
});
@@ -0,0 +1,259 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
postgresqlControlSchemaContract,
postgresqlMainMigrationManifest,
} = require('@qinglong/cluster-postgres');
const { bootstrapClusterAdmin } = require('@qinglong/cluster-admin');
const PEPPER = 'A'.repeat(43);
const WORKER_PEPPER = Buffer.alloc(32, 0x42).toString('base64url');
function history() {
return postgresqlMainMigrationManifest.migrations.map((migration, index) => ({
streamId: postgresqlMainMigrationManifest.id,
dialect: postgresqlMainMigrationManifest.dialect,
migrationId: migration.id,
checksum: migration.checksum,
appliedAtMs: index + 1,
}));
}
function adminPrivileges() {
const writable = new Set([
'identity_subjects',
'api_credentials',
'security_audit_events',
'identity_subject_mutations',
'api_credential_mutations',
'worker_credentials',
'worker_credential_mutations',
'worker_credential_deliveries',
'worker_credential_stage_discards',
'tool_result_key_catalog_generations',
'tool_execution_result_rekey_overlays',
'tool_execution_result_rekey_heads',
'tool_result_key_retirement_receipts',
]);
const readable = new Set([
'schema_migrations',
'schema_capabilities',
'projects',
'plugin_package_task_ownerships',
'tool_execution_completions',
'tool_execution_result_key_bindings',
...writable,
]);
return postgresqlControlSchemaContract.tables.map(({ name: tableName }) => ({
tableName,
selectAllowed: readable.has(tableName),
insertAllowed: writable.has(tableName),
updateAllowed: [
'identity_subjects',
'worker_credentials',
'tool_execution_result_rekey_heads',
].includes(tableName),
deleteAllowed: false,
isOwner: false,
}));
}
function database(serverVersionNum = '160014') {
const contract = postgresqlControlSchemaContract;
let closes = 0;
const resource = {
pool: {
async query(text) {
if (text.includes("current_setting('server_version_num')")) {
return {
rows: [
{
serverVersionNum,
currentUser: 'ql3_admin',
inRecovery: false,
transactionReadOnly: 'off',
},
],
};
}
if (text.includes('FROM "ql3"."schema_migrations"')) {
return { rows: history() };
}
if (text.includes('FROM "ql3"."schema_capabilities"')) {
return {
rows: [
{
contractName: contract.contractName,
contractVersion: contract.contractVersion,
migrationId: contract.migrationId,
capabilities: contract.capabilities,
},
],
};
}
if (text.includes('FROM pg_class tables')) {
return {
rows: contract.tables.flatMap((table) =>
table.columns.map((columnName) => ({
tableName: table.name,
columnName,
})),
),
};
}
if (text.includes('FROM pg_indexes')) {
return { rows: contract.indexes.map((indexName) => ({ indexName })) };
}
if (text.includes('FROM pg_constraint')) {
return {
rows: [
...contract.checks.map((constraintName) => ({
constraintName,
constraintType: 'check',
})),
...contract.foreignKeys.map((constraintName) => ({
constraintName,
constraintType: 'foreign_key',
})),
],
};
}
if (text.includes('FROM pg_proc routines')) {
return {
rows: contract.functions.map((definition) => ({
functionName: definition.name,
identityArguments: definition.identityArguments,
owner: definition.owner,
securityDefiner: definition.securityDefiner,
volatility: definition.volatility,
configuration: definition.configuration,
publicExecute: false,
})),
};
}
if (text.includes('FROM pg_catalog.pg_roles')) {
return {
rows: [
{
canLogin: true,
superuser: false,
createDatabase: false,
createRole: false,
replication: false,
bypassRowLevelSecurity: false,
databaseConnect: true,
},
],
};
}
if (text.includes('has_schema_privilege')) {
return { rows: [{ schemaUsage: true, schemaCreate: false }] };
}
if (text.includes('has_table_privilege')) {
return { rows: adminPrivileges() };
}
if (text.includes('has_function_privilege')) {
return {
rows: contract.functions.map(({ name: functionName }) => ({
functionName,
executeAllowed: ![
'commit_plugin_package_lifecycle',
'commit_plugin_package_task_reconciliation',
'commit_plugin_package_quarantine',
'enforce_plugin_package_stage_provenance',
'lock_active_plugin_package_project',
'lock_approval_policy_fence',
'plugin_package_lifecycle_blocking_runs',
'plugin_package_automation_start_allowed',
'plugin_package_run_start_allowed',
'plugin_package_tool_start_allowed',
'plugin_package_workflow_admission_snapshot',
'plugin_package_workflow_task_attempt_snapshot',
].includes(functionName),
isOwner: false,
})),
};
}
throw new Error(`unexpected query: ${text}`);
},
async connect() {
throw new Error('not used during bootstrap');
},
},
async close() {
closes += 1;
},
};
return { resource, closes: () => closes };
}
test('rejects an invalid pepper before opening PostgreSQL', async () => {
let opens = 0;
await assert.rejects(
bootstrapClusterAdmin({
apiCredentialPepper: 'invalid',
workerCredentialPepper: WORKER_PEPPER,
async openDatabase() {
opens += 1;
return database().resource;
},
}),
/pepper is invalid/,
);
assert.equal(opens, 0);
});
test('rejects invalid optional configuration before opening PostgreSQL', async () => {
let opens = 0;
await assert.rejects(
bootstrapClusterAdmin({
apiCredentialPepper: PEPPER,
workerCredentialPepper: WORKER_PEPPER,
now: 1,
async openDatabase() {
opens += 1;
return database().resource;
},
}),
/clock is invalid/,
);
assert.equal(opens, 0);
});
test('closes PostgreSQL after readiness failure', async () => {
const db = database('150018');
await assert.rejects(
bootstrapClusterAdmin({
apiCredentialPepper: PEPPER,
workerCredentialPepper: WORKER_PEPPER,
async openDatabase() {
return db.resource;
},
}),
(error) => error.code === 'server_version_unsupported',
);
assert.equal(db.closes(), 1);
});
test('assembles isolated administration and audit ports after readiness', async () => {
const db = database();
const runtime = await bootstrapClusterAdmin({
apiCredentialPepper: PEPPER,
workerCredentialPepper: WORKER_PEPPER,
async openDatabase() {
return db.resource;
},
});
assert.equal(runtime.evidence.currentUser, 'ql3_admin');
assert.equal(
runtime.evidence.contractVersion,
postgresqlControlSchemaContract.contractVersion,
);
assert.equal(typeof runtime.administration.issueCredential, 'function');
assert.equal(typeof runtime.audit.list, 'function');
assert.equal(typeof runtime.workerCredentials.issue, 'function');
assert.equal('taskDefinitions' in runtime, false);
assert.equal('triggers' in runtime, false);
await Promise.all([runtime.close(), runtime.close()]);
assert.equal(db.closes(), 1);
});
@@ -0,0 +1,22 @@
-----BEGIN CERTIFICATE-----
MIIDnDCCAoSgAwIBAgIUSqFCIWmVHiQPuPrRwZNFpb5zwbAwDQYJKoZIhvcNAQEL
BQAwPTE7MDkGA1UEAwwycWwzLXBsdWdpbi1wYWNrYWdlLW1hbmFnZW1lbnQucWlu
Z2xvbmczLXN5c3RlbS5zdmMwHhcNMjYwNzI4MTgxODE5WhcNMzYwNzI1MTgxODE5
WjA9MTswOQYDVQQDDDJxbDMtcGx1Z2luLXBhY2thZ2UtbWFuYWdlbWVudC5xaW5n
bG9uZzMtc3lzdGVtLnN2YzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AL3w9awjAP6KwlM4sSBrp1NpG6w/FtG6B3/YS7bq4IWDGAKo9OTGxiRdEUHQCB0T
GzvkL1Xz6xQx6TI0jZtOAs9MJf9/uSH7+Dpu6eKjF2B6yThLFAyUvjmWTkFTJM/c
YPrh5B52eAGEkDmufGvPpIoxM2t6H+p/Ka90svmJ92264DFQCNGZCqBTe9UWmF72
n79EXvlVnHYncVqf5Q2kHZHZK2CYkKBwB4Q9NyHmkuv7onM+zWOVdPk6+n0d/C6i
a87vfy2dneCV1kgvSXJKk7Bq0OeSFaFhL3l4iAQIDlk9/HxfpTw9dCW7497JFbzf
ZDGxgR+GgTLjwihZ3ZXNr50CAwEAAaOBkzCBkDAdBgNVHQ4EFgQU7IspgeYtkxgS
JzrpE/4D0DICJcswHwYDVR0jBBgwFoAU7IspgeYtkxgSJzrpE/4D0DICJcswPQYD
VR0RBDYwNIIycWwzLXBsdWdpbi1wYWNrYWdlLW1hbmFnZW1lbnQucWluZ2xvbmcz
LXN5c3RlbS5zdmMwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEA
IX5TeIHd6nWu93VBRnK+myXvIFJA8V3uT/Av2klDfpxucd6VnUnT7y+tlCuD/69l
vnWz2GFstdS9fqOLcUIjqiNky/cwM9OPOoSU7GUe50Umd/7+rfmp2QzPJ9XTO3Lf
IA7m3gYaUzFX9INx22OcKpG4mIiWdk4CGwmb/ydyoRfbQGF3CO2roaipLcBfvNGv
8y/mGT44aa/3ZOiWdhbamkmzsWBSr8X2EajmS+jQXbk3UPgYg1M6XPMaHf4T+vym
jnhHvO7lXAW6HZJG9qIf2E0xY66BWCLSS2MI0Prm3rcmv0f+8R9X6jWmc3SWc/M+
Xh/sql7vvusdH8nCr6Ul9Q==
-----END CERTIFICATE-----
@@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC98PWsIwD+isJT
OLEga6dTaRusPxbRugd/2Eu26uCFgxgCqPTkxsYkXRFB0AgdExs75C9V8+sUMeky
NI2bTgLPTCX/f7kh+/g6bunioxdgesk4SxQMlL45lk5BUyTP3GD64eQedngBhJA5
rnxrz6SKMTNreh/qfymvdLL5ifdtuuAxUAjRmQqgU3vVFphe9p+/RF75VZx2J3Fa
n+UNpB2R2StgmJCgcAeEPTch5pLr+6JzPs1jlXT5Ovp9HfwuomvO738tnZ3gldZI
L0lySpOwatDnkhWhYS95eIgECA5ZPfx8X6U8PXQlu+PeyRW832QxsYEfhoEy48Io
Wd2Vza+dAgMBAAECggEAA3bfpDzHNIf8tLPgrIIKJ+qk3JFnWHqXecIiL2Xn+RzD
qq4WOWNUEy8MtNyXKUtDQJEQOUsc9d4Ag4U93Uqg9n2hu8qX6oIZVvIq8JKfnIeL
dVa5OOQwTAXtIM05a1POWQUXD2SDGn6mesbZ8hrJb35855rvS2xNAFbqj26axV+O
fIVbzhPq0BXVM+vrv0uyck3+6eatzEZ774zweZxIeQ1dwN1b3D0/4Tv56PiW6ydo
JVXoHq+WS4eWFy2qngEoM2YcOBkHELagYz2atGc6lCSn9DES9kzFrARJtZaxtg1v
84M5ylEsBg9g3Nqb9h+30z/1DncWe5oPQVGzGehXQQKBgQDi1i/BLjJn9HdZkaH5
WxJJ8DTTlZwz3BYoWTnbaxV/bwmc/mYWgk9wy7R+TXbfSr5UA4XN21JsESBrBqqw
iWwC5A33vrtvPv8tK5zU4NhN1Gqp3wL905fUpCktOSmd5Yy17ChxAG1T0H4Z+Q8f
btRX3uAqDY6vjEA7KIQtqXrwuQKBgQDWXHG8aH5J3PrXNxQsrOqsMFcZwsIutyI7
AX8pNr2S2yvCu8QY+OkF10MWSfpcu7n1U7odPu/e+fnyGdb/eP/3t/xPa3m0nKhF
n8lLKTuqFLRRHS8vWnqwwlREi6zCiCY1p6ole5I7DdaBdZ3NmVsDsT1/9PQL5jOM
jtYGh8jcBQKBgQDXstKAQRyfa4DeRDSgt/AhLPAezqJVUhAj2AzDUAWGQyECD4sm
Fk3SNXJxs9m4pQttOlhPEyJCLtsDyrge3N4/tXpuvgjf1SizXEhqyVAGWln/JFhk
44L6KgwZu8SOJ8zw5RrjsYNEcvqmWgX+XtY+pGnGs1OeLKCbYICoJwQHyQKBgAlI
W4yDIeTk8t/a/L6qhkcKmNr+uhX3zD2t5OnN+wue/hgitW03ai/ckIUokvTtFDJx
e7/Ed/K59H7ta4gIn10E1KJDzzNpDUhmkPr2QCUvXFee4eo1CtcYszl8qvCJoM32
AsI4xa5U/RMCGuFKYMyaIkWmP+M8BsNxdAc4XhRZAoGAWz1Unno4av/b+h0AKXR2
+07KTJaJqm99sg3XW9Zdqv4oh/TQBfDxzBIqP6nqC0FhNqw5rvgrPSmIy1TXwCCq
lsde21N/ICctnn1E4r+eddbteWqfXNZN85s7wt4KyIBkkXKT6J8dE71UeHaNPxhw
OPxbkZsPTPXmc9g82T0eFD4=
-----END PRIVATE KEY-----
@@ -0,0 +1,331 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const {
ModelProviderCredentialAdministrationAuthorizationFenceConflictError,
} = require('@qinglong/ai/model-provider-credential-administration');
const {
createModelProviderCredentialTransition,
} = require('@qinglong/ai/model-provider-credential-catalog');
const {
createModelProviderCredentialTestAllowlist,
} = require('@qinglong/ai/model-provider-credential-test-connection');
const {
ClusterModelProviderCredentialManagementAuthenticationError,
ClusterModelProviderCredentialManagementAuthorizationError,
ClusterModelProviderCredentialManagementConflictError,
ClusterModelProviderCredentialManagementRequestError,
createClusterModelProviderCredentialManagementService,
} = require('../dist/model-provider-credential/modelProviderCredentialManagement.js');
const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
const NOW_MS = 1_000_000;
const MUTATION_ID = '019f7094-a853-4f3b-82ab-dfa08e6bd1c1';
function principal(overrides = {}) {
return {
subject: { type: 'user', id: 'owner-a' },
authenticationId: 'authentication-1',
authenticatedAtMs: NOW_MS - 1_000,
expiresAtMs: NOW_MS + 60_000,
assurance: 'multi_factor',
...overrides,
};
}
function bindRequest(overrides = {}) {
return {
requestId: 'request-1',
mutationId: MUTATION_ID,
projectId: 'project-a',
provider: 'openai-compatible',
expectedGeneration: 0,
revision: 'credential-v1',
secretRef: createSecretRef({
projectId: 'project-a',
name: 'openai-token',
}),
principal: principal(),
...overrides,
};
}
function fixture(overrides = {}) {
const calls = { policy: [], mutations: [], audits: [], testPlans: [] };
const policy = {
async authorize(...args) {
calls.policy.push(args);
return (
overrides.decision ?? {
effect: 'allow',
reasons: ['project_owner'],
fence: { projectVersion: 3, bindingVersion: 7 },
}
);
},
};
const credentials = {
async findCurrentTransition() {
return null;
},
async commit() {
throw new Error('raw commit must not be used');
},
async commitAuthorized(mutation) {
calls.mutations.push(mutation);
if (overrides.repositoryError) throw overrides.repositoryError;
return {
status: 'created',
transition: createModelProviderCredentialTransition(
mutation.command,
null,
NOW_MS,
),
};
},
};
const audit = {
async listAuthorized(query) {
calls.audits.push(query);
if (overrides.auditError) throw overrides.auditError;
return {
projectId: query.query.projectId,
records: [
{
eventId: MUTATION_ID,
requestId: 'request-1',
operation: 'provider-credential.bind',
actor: { type: 'user', id: 'owner-a' },
fence: { projectVersion: 3, bindingVersion: 7 },
occurredAtMs: NOW_MS - 1,
},
],
nextCursor: null,
};
},
};
const testAllowlist = createModelProviderCredentialTestAllowlist({
revision: 'catalog-v1',
providers: [
{
provider: 'openai-compatible',
adapter: 'openai-compatible',
baseUrl: 'https://provider.example.test/v1/',
revision: 'endpoint-v1',
deadlineMs: 5_000,
maxResponseBytes: 64 * 1_024,
maxModels: 64,
maxCostMicrousd: 0,
retryLimit: 0,
},
],
});
const testPlans = {
async createAuthorized(value) {
calls.testPlans.push(value);
if (overrides.testPlanError) throw overrides.testPlanError;
return { status: 'created', plan: value.plan };
},
};
return {
calls,
service: createClusterModelProviderCredentialManagementService({
policy,
credentials,
audit,
testPlans,
testAllowlist,
testPlanLifetimeMs: 60_000,
now: () => NOW_MS,
}),
};
}
test('test connection plan selects endpoint and hard budgets server-side', async () => {
const { service, calls } = fixture();
const testId = '319f7094-a853-4f3b-82ab-dfa08e6bd1c4';
const result = await service.planTestConnection({
requestId: 'test-request-1',
testId,
projectId: 'project-a',
provider: 'openai-compatible',
principal: principal(),
});
assert.equal(result.status, 'created');
assert.equal(result.plan.testId, testId);
assert.equal(
result.plan.endpoint.baseUrl,
'https://provider.example.test/v1/',
);
assert.equal(result.plan.endpoint.maxCostMicrousd, 0);
assert.equal(result.plan.endpoint.retryLimit, 0);
assert.equal(result.plan.expiresAtMs - result.plan.plannedAtMs, 60_000);
assert.equal(calls.policy[0][2], 'secret.manage');
assert.equal(calls.testPlans.length, 1);
assert.equal(
calls.testPlans[0].audit.operationId,
'model_provider_credential.test.plan',
);
assert.doesNotMatch(JSON.stringify(calls.testPlans[0]), /secretRef|token/i);
});
test('test connection request cannot supply URL, budgets or SecretRef', async () => {
const { service, calls } = fixture();
const base = {
requestId: 'test-request-1',
testId: '419f7094-a853-4f3b-82ab-dfa08e6bd1c5',
projectId: 'project-a',
provider: 'openai-compatible',
principal: principal(),
};
for (const widened of [
{ baseUrl: 'https://attacker.example/v1/' },
{ deadlineMs: 60_000 },
{ secretRef: 'qlsecret:project-a/openai-token' },
]) {
await assert.rejects(
service.planTestConnection({ ...base, ...widened }),
ClusterModelProviderCredentialManagementRequestError,
);
}
assert.equal(calls.policy.length, 0);
assert.equal(calls.testPlans.length, 0);
});
test('bind derives actor and exact secret.manage authority server-side', async () => {
const { service, calls } = fixture();
const result = await service.bind(bindRequest());
assert.equal(result.status, 'created');
assert.equal(result.transition.activeBindingRevision, 'credential-v1');
assert.equal(calls.policy.length, 1);
assert.equal(calls.policy[0][1], 'project-a');
assert.equal(calls.policy[0][2], 'secret.manage');
assert.equal(calls.mutations.length, 1);
assert.deepEqual(calls.mutations[0].command.changedBy, {
type: 'user',
id: 'owner-a',
});
assert.equal(calls.mutations[0].audit.outcome, 'allowed');
assert.equal(
calls.mutations[0].audit.operationId,
'model_provider_credential.bind',
);
assert.equal(
JSON.stringify(calls.mutations[0]).includes('authorization'),
false,
);
});
test('revoke is Project-fenced and cannot carry binding material', async () => {
const { service, calls } = fixture();
const request = bindRequest();
const result = await service.revoke({
requestId: 'request-2',
mutationId: '119f7094-a853-4f3b-82ab-dfa08e6bd1c2',
projectId: request.projectId,
provider: request.provider,
expectedGeneration: 0,
principal: request.principal,
});
assert.equal(result.transition.action, 'revoke');
assert.equal(calls.mutations[0].command.binding, null);
assert.equal(
calls.mutations[0].audit.operationId,
'model_provider_credential.revoke',
);
});
test('weak, stale and denied principals fail before repository mutation', async () => {
for (const scenario of [
{
request: bindRequest({
principal: principal({ assurance: 'single_factor' }),
}),
error: ClusterModelProviderCredentialManagementAuthenticationError,
},
{
request: bindRequest({
principal: principal({ authenticatedAtMs: NOW_MS - 300_001 }),
}),
error: ClusterModelProviderCredentialManagementAuthenticationError,
},
{
decision: { effect: 'deny', reasons: ['policy_denied'], fence: null },
request: bindRequest(),
error: ClusterModelProviderCredentialManagementAuthorizationError,
},
]) {
const { service, calls } = fixture({ decision: scenario.decision });
await assert.rejects(service.bind(scenario.request), scenario.error);
assert.equal(calls.mutations.length, 0);
}
});
test('request widening and durable fence drift fail closed', async () => {
const { service } = fixture();
await assert.rejects(
service.bind({
...bindRequest(),
changedBy: { type: 'user', id: 'owner-b' },
}),
ClusterModelProviderCredentialManagementRequestError,
);
await assert.rejects(
service.bind({ ...bindRequest(), secretRef: 'not-a-canonical-secret-ref' }),
ClusterModelProviderCredentialManagementRequestError,
);
const conflict = fixture({
repositoryError:
new ModelProviderCredentialAdministrationAuthorizationFenceConflictError(),
});
await assert.rejects(
conflict.service.bind(bindRequest()),
ClusterModelProviderCredentialManagementConflictError,
);
});
test('audit query reuses secret.manage and returns only content-free events', async () => {
const { service, calls } = fixture();
const result = await service.listAudit({
requestId: 'audit-request-1',
queryId: '219f7094-a853-4f3b-82ab-dfa08e6bd1c3',
projectId: 'project-a',
limit: 8,
principal: principal(),
});
assert.equal(calls.policy.length, 1);
assert.equal(calls.policy[0][2], 'secret.manage');
assert.equal(calls.audits.length, 1);
assert.equal(
calls.audits[0].audit.operationId,
'model_provider_credential.audit.list',
);
assert.deepEqual(calls.audits[0].fence, {
projectVersion: 3,
bindingVersion: 7,
});
assert.equal(result.records[0].operation, 'provider-credential.bind');
assert.doesNotMatch(
JSON.stringify(result),
/secretRef|provider-token|bindingDigest|authenticationId|openai/i,
);
});
test('invalid audit cursors fail before policy or storage', async () => {
const { service, calls } = fixture();
await assert.rejects(
service.listAudit({
requestId: 'audit-request-1',
queryId: '219f7094-a853-4f3b-82ab-dfa08e6bd1c3',
projectId: 'project-a',
limit: 8,
before: { occurredAtMs: 1, eventId: 'not-a-uuid' },
principal: principal(),
}),
ClusterModelProviderCredentialManagementRequestError,
);
assert.equal(calls.policy.length, 0);
assert.equal(calls.audits.length, 0);
});
@@ -0,0 +1,363 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const {
chmodSync,
mkdtempSync,
readFileSync,
realpathSync,
rmSync,
writeFileSync,
} = require('node:fs');
const { createServer } = require('node:https');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { afterEach, test } = require('node:test');
const {
executeClusterModelProviderCredentialManagementClient,
validateClusterModelProviderCredentialManagementClientResult,
} = require('@qinglong/cluster-admin/model-provider-credential-management-client');
const {
createModelProviderCredentialTestAllowlist,
createModelProviderCredentialTestPlan,
} = require('@qinglong/ai/model-provider-credential-test-connection');
const FIXTURES = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls',
);
const CLI = resolve(
__dirname,
'../dist/model-provider-credential/modelProviderCredentialManagementClientCli.js',
);
const temporaryDirectories = [];
const bindCommand = Object.freeze({
schemaVersion: 1,
operation: 'provider-credential.bind',
request: Object.freeze({
requestId: 'request-bind-1',
mutationId: '123e4567-e89b-42d3-a456-426614174000',
projectId: 'project-a',
provider: 'openai-compatible',
expectedGeneration: 0,
revision: 'credential-v1',
secretRef: 'project/project-a/provider-token',
}),
});
const auditCommand = Object.freeze({
schemaVersion: 1,
operation: 'provider-credential.audit.list',
request: Object.freeze({
requestId: 'audit-request-1',
queryId: '219f7094-a853-4f3b-82ab-dfa08e6bd1c3',
projectId: 'project-a',
limit: 2,
}),
});
const testPlanCommand = Object.freeze({
schemaVersion: 1,
operation: 'provider-credential.test.plan',
request: Object.freeze({
requestId: 'test-request-1',
testId: '319f7094-a853-4f3b-82ab-dfa08e6bd1c4',
projectId: 'project-a',
provider: 'openai-compatible',
}),
});
function testPlanResult(overrides = {}) {
const allowlist = createModelProviderCredentialTestAllowlist({
revision: 'catalog-v1',
providers: [
{
provider: 'openai-compatible',
adapter: 'openai-compatible',
baseUrl: 'https://provider.example.test/v1/',
revision: 'endpoint-v1',
deadlineMs: 5_000,
maxResponseBytes: 64 * 1_024,
maxModels: 64,
maxCostMicrousd: 0,
retryLimit: 0,
},
],
});
const plan = createModelProviderCredentialTestPlan({
testId: testPlanCommand.request.testId,
requestId: testPlanCommand.request.requestId,
projectId: testPlanCommand.request.projectId,
provider: testPlanCommand.request.provider,
endpoint: allowlist.providers[0],
requestedBy: { type: 'user', id: 'owner-a' },
fence: { projectVersion: 3, bindingVersion: 7 },
plannedAtMs: 1_000,
expiresAtMs: 61_000,
...overrides,
});
return {
schemaVersion: 1,
operation: 'provider-credential.test.plan',
status: 'created',
plan,
};
}
function bindResult(overrides = {}) {
return {
schemaVersion: 1,
operation: 'provider-credential.bind',
status: 'created',
credential: {
projectId: 'project-a',
provider: 'openai-compatible',
generation: 1,
action: 'bind',
activeBindingRevision: 'credential-v1',
activeBindingDigest: 'a'.repeat(64),
transitionDigest: 'b'.repeat(64),
changedAtMs: 1_001,
...overrides,
},
};
}
function auditResult(overrides = {}) {
return {
schemaVersion: 1,
operation: 'provider-credential.audit.list',
audit: {
projectId: 'project-a',
records: [
{
eventId: '019f7094-a853-4f3b-82ab-dfa08e6bd1c1',
requestId: 'request-bind-1',
operation: 'provider-credential.bind',
actor: { type: 'user', id: 'owner-a' },
fence: { projectVersion: 3, bindingVersion: 7 },
occurredAtMs: 1_001,
},
],
nextCursor: null,
...overrides,
},
};
}
function privateWrite(filePath, value) {
writeFileSync(filePath, value, { mode: 0o600 });
chmodSync(filePath, 0o600);
}
function clientFiles(port, command) {
const directory = realpathSync(
mkdtempSync(join(tmpdir(), 'ql3-provider-credential-client-')),
);
temporaryDirectories.push(directory);
const paths = {
configFile: join(directory, 'client.json'),
commandFile: join(directory, 'command.json'),
assertionFile: join(directory, 'assertion.jwt'),
};
const caFile = join(directory, 'ca.crt');
const clientCertificateFile = join(directory, 'client.crt');
const clientPrivateKeyFile = join(directory, 'client.key');
privateWrite(caFile, readFileSync(join(FIXTURES, 'ca-cert.pem')));
privateWrite(
clientCertificateFile,
readFileSync(join(FIXTURES, 'client-cert.pem')),
);
privateWrite(
clientPrivateKeyFile,
readFileSync(join(FIXTURES, 'client-key.pem')),
);
privateWrite(
paths.configFile,
`${JSON.stringify({
schemaVersion: 1,
endpoint: `https://localhost:${port}/api/v3/provider-credentials/management`,
servername: 'localhost',
caFile,
clientCertificateFile,
clientPrivateKeyFile,
requestTimeoutMs: 2_000,
})}\n`,
);
privateWrite(paths.commandFile, `${JSON.stringify(command)}\n`);
privateWrite(
paths.assertionFile,
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl',
);
return paths;
}
function listen(server) {
return new Promise((resolvePromise, reject) => {
server.once('error', reject);
server.listen(0, 'localhost', () => resolvePromise(server.address()));
});
}
function close(server) {
return new Promise((resolvePromise, reject) => {
server.close((error) => (error ? reject(error) : resolvePromise()));
});
}
afterEach(() => {
for (const directory of temporaryDirectories.splice(0)) {
rmSync(directory, { recursive: true, force: true });
}
});
test('validates an exact content-free result against the request fence', () => {
const result = validateClusterModelProviderCredentialManagementClientResult(
bindResult(),
bindCommand,
);
assert.equal(result.credential.generation, 1);
assert.doesNotMatch(
JSON.stringify(result),
/secretRef|provider-token|mutationId|authenticationId/,
);
assert.throws(() =>
validateClusterModelProviderCredentialManagementClientResult(
bindResult({ projectId: 'project-b' }),
bindCommand,
),
);
assert.throws(() =>
validateClusterModelProviderCredentialManagementClientResult(
{ ...bindResult(), secretRef: 'must-not-leak' },
bindCommand,
),
);
});
test('sends one TLS 1.3 mTLS provider credential command', async () => {
let observed;
const server = createServer(
{
key: readFileSync(join(FIXTURES, 'server-key.pem')),
cert: readFileSync(join(FIXTURES, 'server-cert.pem')),
ca: readFileSync(join(FIXTURES, 'ca-cert.pem')),
requestCert: true,
rejectUnauthorized: true,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
},
(request, response) => {
const chunks = [];
request.on('data', (chunk) => chunks.push(chunk));
request.once('end', () => {
observed = {
method: request.method,
path: request.url,
authorization: request.headers.authorization,
authorized: request.socket.authorized,
protocol: request.socket.getProtocol(),
command: JSON.parse(Buffer.concat(chunks).toString('utf8')),
};
const body = Buffer.from(
JSON.stringify({
schemaVersion: 1,
requestId: 'http-request-1',
result: bindResult(),
}),
);
response.writeHead(200, {
'content-type': 'application/json; charset=utf-8',
'content-length': String(body.length),
});
response.end(body);
});
},
);
const address = await listen(server);
try {
const result = await executeClusterModelProviderCredentialManagementClient(
clientFiles(address.port, bindCommand),
);
assert.equal(result.requestId, 'http-request-1');
assert.equal(result.result.credential.generation, 1);
assert.deepEqual(observed, {
method: 'POST',
path: '/api/v3/provider-credentials/management',
authorization: 'Bearer eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl',
authorized: true,
protocol: 'TLSv1.3',
command: bindCommand,
});
} finally {
await close(server);
}
});
test('validates a bounded content-free audit page and rejects widening', () => {
const result = validateClusterModelProviderCredentialManagementClientResult(
auditResult(),
auditCommand,
);
assert.equal(result.audit.records.length, 1);
assert.doesNotMatch(
JSON.stringify(result),
/secretRef|provider-token|bindingDigest|transitionDigest|authenticationId|openai/i,
);
assert.throws(() =>
validateClusterModelProviderCredentialManagementClientResult(
auditResult({ provider: 'openai-compatible' }),
auditCommand,
),
);
assert.throws(() =>
validateClusterModelProviderCredentialManagementClientResult(
auditResult({
nextCursor: {
occurredAtMs: 1_001,
eventId: '019f7094-a853-4f3b-82ab-dfa08e6bd1c1',
},
}),
auditCommand,
),
);
});
test('validates the exact server-selected test plan identity', () => {
const result = validateClusterModelProviderCredentialManagementClientResult(
testPlanResult(),
testPlanCommand,
);
assert.equal(result.plan.endpoint.maxCostMicrousd, 0);
assert.equal(result.plan.endpoint.retryLimit, 0);
assert.doesNotMatch(JSON.stringify(result), /secretRef|token/i);
assert.throws(() =>
validateClusterModelProviderCredentialManagementClientResult(
testPlanResult({ projectId: 'project-b' }),
testPlanCommand,
),
);
assert.throws(() =>
validateClusterModelProviderCredentialManagementClientResult(
{ ...testPlanResult(), secretRef: 'must-not-leak' },
testPlanCommand,
),
);
});
test('client CLI exposes only private file paths and stable errors', () => {
const help = spawnSync(process.execPath, [CLI, '--help'], {
encoding: 'utf8',
});
assert.equal(help.status, 0);
assert.match(help.stdout, /^Usage: ql3-provider-credential-client /);
assert.doesNotMatch(help.stdout, /token value|secret value/i);
const invalid = spawnSync(process.execPath, [CLI, '--assertion=value'], {
encoding: 'utf8',
});
assert.equal(invalid.status, 64);
assert.match(invalid.stderr, /USAGE_INVALID/);
assert.doesNotMatch(invalid.stderr, /assertion=value/);
});
@@ -0,0 +1,134 @@
const assert = require('node:assert/strict');
const { readFileSync } = require('node:fs');
const { request: httpsRequest } = require('node:https');
const { resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterModelProviderCredentialManagementConflictError,
} = require('@qinglong/cluster-admin/model-provider-credential-management');
const {
startClusterModelProviderCredentialManagementHttp,
} = require('@qinglong/cluster-admin/model-provider-credential-management-http');
const SERVER_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-key.pem',
);
const SERVER_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem',
);
const PATH = '/api/v3/provider-credentials/management';
function post(port, path = PATH) {
const body = Buffer.from(
JSON.stringify({
schemaVersion: 1,
operation: 'provider-credential.revoke',
request: {
requestId: 'request-provider-revoke',
mutationId: '1e828c8a-86d7-4b24-bbe0-2d9ecf1c1eab',
projectId: 'project-a',
provider: 'openai-compatible',
expectedGeneration: 1,
},
}),
);
return new Promise((resolvePromise, reject) => {
const outgoing = httpsRequest(
{
host: '127.0.0.1',
port,
path,
method: 'POST',
rejectUnauthorized: false,
agent: false,
headers: {
authorization: 'Bearer assertion',
'content-type': 'application/json',
'content-length': String(body.length),
},
},
(incoming) => {
const chunks = [];
incoming.on('data', (chunk) => chunks.push(chunk));
incoming.once('end', () => {
const bytes = Buffer.concat(chunks);
resolvePromise({
statusCode: incoming.statusCode,
body: bytes.length ? JSON.parse(bytes.toString('utf8')) : null,
});
});
},
);
outgoing.once('error', reject);
outgoing.end(body);
});
}
test('serves only the provider credential path and maps conflicts to low-sensitive HTTP facts', async () => {
const privateKey = Buffer.from(readFileSync(SERVER_KEY));
const application = await startClusterModelProviderCredentialManagementHttp({
host: '127.0.0.1',
port: 0,
tls: {
privateKey,
certificate: Buffer.from(readFileSync(SERVER_CERT)),
},
identities: {
async reload() {
return {
schemaVersion: 1,
generation: 1,
digest: 'digest',
issuer: 'https://identity.example.test/',
audience: 'qinglong3-model-provider-credential-management',
activeKeyIds: ['provider-key-1'],
revokedKeyIds: [],
};
},
bind() {
return {
async authenticate() {
return {
subject: { type: 'user', id: 'operator-a' },
authenticationId: 'session-operator-a',
authenticatedAtMs: 900,
expiresAtMs: 10_000,
assurance: 'multi_factor',
};
},
};
},
},
transport: {
async execute(_command, authentication) {
await authentication.authenticate();
throw new ClusterModelProviderCredentialManagementConflictError();
},
},
now: () => 1_000,
});
try {
assert.equal(privateKey.every((value) => value === 0), true);
const conflict = await post(application.address.port);
assert.deepEqual(conflict, {
statusCode: 409,
body: {
schemaVersion: 1,
requestId: conflict.body.requestId,
error: { code: 'conflict' },
},
});
assert.match(conflict.body.requestId, /^[0-9a-f-]{36}$/);
const absent = await post(
application.address.port,
'/api/v3/plugin-packages/management',
);
assert.equal(absent.statusCode, 404);
assert.equal(absent.body.error.code, 'not_found');
} finally {
await application.close();
}
});
@@ -0,0 +1,246 @@
const assert = require('node:assert/strict');
const { readFile, writeFile, mkdtemp, rm } = require('node:fs/promises');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterModelProviderCredentialManagementProcessConfigError,
loadClusterModelProviderCredentialManagementProcessConfig,
startClusterModelProviderCredentialManagementProcess,
} = require('@qinglong/cluster-admin/model-provider-credential-management-process');
const {
createModelProviderCredentialTestAllowlist,
} = require('@qinglong/ai/model-provider-credential-test-connection');
const FIXTURES = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls',
);
const NOW_MS = Date.UTC(2030, 0, 1);
function enabledEnvironment(paths, overrides = {}) {
return {
QL3_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
QL3_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_HOST: '127.0.0.1',
QL3_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PORT: '8446',
QL3_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_TLS_CERT_FILE:
paths.certificateFile,
QL3_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_TLS_KEY_FILE: paths.privateKeyFile,
QL3_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_CLIENT_CA_FILE:
paths.clientCertificateAuthorityFile,
QL3_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_CLIENT_CRL_FILE:
paths.clientCertificateRevocationListFile,
QL3_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_KEYSET_FILE:
paths.identityKeysetFile,
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_ALLOWLIST_FILE: paths.testAllowlistFile,
QL3_POSTGRES_AI_CREDENTIAL_MANAGER_URL:
'postgresql://ql3_ai_credential_manager:secret@postgres.example.test/ql3',
QL3_POSTGRES_AI_CREDENTIAL_MANAGER_TLS_MODE: 'disable',
QL3_POSTGRES_AI_CREDENTIAL_MANAGER_ALLOW_INSECURE: 'true',
...overrides,
};
}
async function tlsFixture(run) {
const directory = await mkdtemp(join(tmpdir(), 'ql3-provider-manager-'));
const paths = {
certificateFile: join(directory, 'tls.crt'),
privateKeyFile: join(directory, 'tls.key'),
clientCertificateAuthorityFile: join(directory, 'client-ca.crt'),
clientCertificateRevocationListFile: join(directory, 'client.crl'),
identityKeysetFile: join(directory, 'keyset.json'),
testAllowlistFile: join(directory, 'test-allowlist.json'),
};
try {
await writeFile(
paths.certificateFile,
await readFile(join(FIXTURES, 'server-cert.pem')),
{ mode: 0o644 },
);
await writeFile(
paths.privateKeyFile,
await readFile(join(FIXTURES, 'server-key.pem')),
{ mode: 0o640 },
);
await writeFile(
paths.clientCertificateAuthorityFile,
await readFile(join(FIXTURES, 'ca-cert.pem')),
{ mode: 0o644 },
);
await writeFile(
paths.clientCertificateRevocationListFile,
await readFile(join(FIXTURES, 'empty-crl.pem')),
{ mode: 0o644 },
);
await writeFile(paths.identityKeysetFile, '{}\n', { mode: 0o644 });
await writeFile(
paths.testAllowlistFile,
`${JSON.stringify(
createModelProviderCredentialTestAllowlist({
revision: 'catalog-v1',
providers: [
{
provider: 'openai-compatible',
adapter: 'openai-compatible',
baseUrl: 'https://provider.example.test/v1/',
revision: 'endpoint-v1',
deadlineMs: 5_000,
maxResponseBytes: 64 * 1_024,
maxModels: 64,
maxCostMicrousd: 0,
retryLimit: 0,
},
],
}),
)}\n`,
{ mode: 0o644 },
);
return await run(paths);
} finally {
await rm(directory, { recursive: true, force: true });
}
}
test('disabled provider credential manager opens no database or files', async () => {
let opened = 0;
const runtime = await startClusterModelProviderCredentialManagementProcess({
environment: {
QL3_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_ENABLED: 'false',
},
async openDatabase() {
opened += 1;
throw new Error('must not open');
},
});
assert.equal(runtime.status, 'disabled');
assert.equal(opened, 0);
await runtime.close();
});
test('loads bounded provider-only network and PostgreSQL configuration', async () => {
await tlsFixture(async (paths) => {
const config = loadClusterModelProviderCredentialManagementProcessConfig(
enabledEnvironment(paths),
);
assert.equal(config.enabled, true);
assert.equal(config.port, 8446);
assert.equal(config.database.pool.maxConnections, 2);
assert.equal(
config.database.pool.applicationName,
'qinglong3-ai-credential-manager',
);
assert.equal(config.http.maxConnections, 32);
assert.equal(config.http.maxConcurrentRequests, 8);
assert.equal(config.http.maxBodyBytes, 32 * 1024);
assert.equal(config.testConnection.planLifetimeMs, 60_000);
assert.equal(config.testConnection.quotaWindowMs, 60_000);
assert.equal(config.testConnection.quotaLimit, 5);
assert.equal(config.testConnection.allowlistFile, paths.testAllowlistFile);
assert.match(
config.database.connection.connectionString,
/^postgresql:\/\/ql3_ai_credential_manager:/,
);
});
assert.throws(
() =>
loadClusterModelProviderCredentialManagementProcessConfig({
QL3_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_ENABLED: 'true',
QL3_PROFILE: 'edge',
}),
ClusterModelProviderCredentialManagementProcessConfigError,
);
});
test('starts after exact readiness and identity, wipes key and closes in order', async () => {
await tlsFixture(async (paths) => {
const order = [];
let privateKey;
let transport;
let httpClosed = 0;
let databaseClosed = 0;
const pool = {
async query() {
throw new Error('repositories remain lazy during composition');
},
async connect() {
throw new Error('repositories remain lazy during composition');
},
};
const runtime = await startClusterModelProviderCredentialManagementProcess({
environment: enabledEnvironment(paths),
now: () => NOW_MS,
async openDatabase() {
order.push('open');
return {
pool,
async close() {
order.push('database-close');
databaseClosed += 1;
},
};
},
async assertReady(candidate) {
order.push('ready');
assert.equal(candidate, pool);
return {
ready: true,
currentUser: 'ql3_ai_credential_manager',
migrationIds: [
'pg-9014-ai-model-provider-credential-management-identity-ledger',
],
writablePrimary: true,
managerAuthority: true,
leastPrivilege: true,
};
},
identities: {
async reload() {
order.push('identity');
return {
schemaVersion: 1,
generation: 3,
digest: 'identity-digest',
issuer: 'https://identity.example.test/',
audience: 'qinglong3-model-provider-credential-management',
activeKeyIds: ['key-3'],
revokedKeyIds: ['key-2'],
};
},
bind() {
throw new Error('HTTP stub does not authenticate');
},
},
async startHttp(options) {
order.push('http');
privateKey = options.tls.privateKey;
transport = options.transport;
assert.equal(options.limits.maxConcurrentRequests, 8);
assert.ok(options.tls.clientCertificateAuthority);
assert.ok(options.tls.clientCertificateRevocationList);
return {
status: 'active',
address: { host: options.host, port: options.port },
availabilityStatus: () => 'ready',
withdraw() {},
async close() {
order.push('http-close');
httpClosed += 1;
},
};
},
});
assert.equal(runtime.status, 'active');
assert.deepEqual(order, ['open', 'ready', 'identity', 'http']);
assert.equal(typeof transport.execute, 'function');
assert.equal(
privateKey.every((byte) => byte === 0),
true,
);
await Promise.all([runtime.close(), runtime.close()]);
assert.equal(httpClosed, 1);
assert.equal(databaseClosed, 1);
assert.deepEqual(order.slice(-2), ['http-close', 'database-close']);
});
});
@@ -0,0 +1,298 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const {
createModelProviderCredentialTransition,
createModelProviderCredentialTransitionCommand,
MODEL_PROVIDER_CREDENTIAL_TRANSITION_COMMAND_SCHEMA,
} = require('@qinglong/ai/model-provider-credential-catalog');
const {
MODEL_PROVIDER_CREDENTIAL_BINDING_SCHEMA,
} = require('@qinglong/ai/provider-credential');
const {
createModelProviderCredentialTestAllowlist,
createModelProviderCredentialTestPlan,
} = require('@qinglong/ai/model-provider-credential-test-connection');
const {
ClusterModelProviderCredentialManagementTransportAuthenticationError,
ClusterModelProviderCredentialManagementTransportRequestError,
createClusterModelProviderCredentialManagementTransport,
normalizeClusterModelProviderCredentialManagementCommand,
} = require('../dist/model-provider-credential/modelProviderCredentialManagementTransport.js');
const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
const NOW_MS = 1_000_000;
const MUTATION_ID = '019f7094-a853-4f3b-82ab-dfa08e6bd1c1';
function principal(overrides = {}) {
return {
subject: { type: 'user', id: 'owner-a' },
authenticationId: 'authentication-1',
authenticatedAtMs: NOW_MS - 1_000,
expiresAtMs: NOW_MS + 60_000,
assurance: 'hardware',
...overrides,
};
}
function bindCommand(overrides = {}) {
return {
schemaVersion: 1,
operation: 'provider-credential.bind',
request: {
requestId: 'request-1',
mutationId: MUTATION_ID,
projectId: 'project-a',
provider: 'openai-compatible',
expectedGeneration: 0,
revision: 'credential-v1',
secretRef: createSecretRef({
projectId: 'project-a',
name: 'openai-token',
}),
},
...overrides,
};
}
function auditCommand(overrides = {}) {
return {
schemaVersion: 1,
operation: 'provider-credential.audit.list',
request: {
requestId: 'audit-request-1',
queryId: '219f7094-a853-4f3b-82ab-dfa08e6bd1c3',
projectId: 'project-a',
limit: 8,
},
...overrides,
};
}
function testPlanCommand(overrides = {}) {
return {
schemaVersion: 1,
operation: 'provider-credential.test.plan',
request: {
requestId: 'test-request-1',
testId: '319f7094-a853-4f3b-82ab-dfa08e6bd1c4',
projectId: 'project-a',
provider: 'openai-compatible',
},
...overrides,
};
}
function testPlan(request) {
const allowlist = createModelProviderCredentialTestAllowlist({
revision: 'catalog-v1',
providers: [
{
provider: request.provider,
adapter: 'openai-compatible',
baseUrl: 'https://provider.example.test/v1/',
revision: 'endpoint-v1',
deadlineMs: 5_000,
maxResponseBytes: 64 * 1_024,
maxModels: 64,
maxCostMicrousd: 0,
retryLimit: 0,
},
],
});
return createModelProviderCredentialTestPlan({
testId: request.testId,
requestId: request.requestId,
projectId: request.projectId,
provider: request.provider,
endpoint: allowlist.providers[0],
requestedBy: { type: 'user', id: 'owner-a' },
fence: { projectVersion: 3, bindingVersion: 7 },
plannedAtMs: NOW_MS,
expiresAtMs: NOW_MS + 60_000,
});
}
function transition(request, action) {
const command = createModelProviderCredentialTransitionCommand({
schema: MODEL_PROVIDER_CREDENTIAL_TRANSITION_COMMAND_SCHEMA,
mutationId: request.mutationId,
projectId: request.projectId,
provider: request.provider,
expectedGeneration: request.expectedGeneration,
action,
binding:
action === 'bind'
? {
schema: MODEL_PROVIDER_CREDENTIAL_BINDING_SCHEMA,
projectId: request.projectId,
provider: request.provider,
revision: request.revision,
secretRef: request.secretRef,
scheme: 'bearer',
}
: null,
changedBy: { type: 'user', id: 'owner-a' },
});
return createModelProviderCredentialTransition(command, null, NOW_MS);
}
function fixture() {
const calls = [];
const service = {
async bind(request) {
calls.push({ operation: 'bind', request });
return {
status: 'created',
transition: transition(request, 'bind'),
};
},
async revoke(request) {
calls.push({ operation: 'revoke', request });
return {
status: 'created',
transition: transition(request, 'revoke'),
};
},
async listAudit(request) {
calls.push({ operation: 'audit.list', request });
return {
projectId: request.projectId,
records: [
{
eventId: MUTATION_ID,
requestId: 'request-1',
operation: 'provider-credential.bind',
actor: { type: 'user', id: 'owner-a' },
fence: { projectVersion: 3, bindingVersion: 7 },
occurredAtMs: NOW_MS - 1,
},
],
nextCursor: null,
};
},
async planTestConnection(request) {
calls.push({ operation: 'test.plan', request });
return { status: 'created', plan: testPlan(request) };
},
};
return {
calls,
transport: createClusterModelProviderCredentialManagementTransport({
service,
now: () => NOW_MS,
}),
};
}
test('transport injects authenticated principal and returns content-free summary', async () => {
const { transport, calls } = fixture();
const result = await transport.execute(bindCommand(), {
async authenticate() {
return principal();
},
});
assert.equal(calls.length, 1);
assert.equal(calls[0].request.principal.subject.id, 'owner-a');
assert.equal(result.operation, 'provider-credential.bind');
assert.equal(result.credential.activeBindingRevision, 'credential-v1');
assert.equal(JSON.stringify(result).includes('qlsecret:'), false);
assert.equal(JSON.stringify(result).includes('openai-token'), false);
});
test('transport rejects caller-supplied identity and unknown operations', () => {
assert.throws(
() =>
normalizeClusterModelProviderCredentialManagementCommand({
...bindCommand(),
request: { ...bindCommand().request, principal: principal() },
}),
ClusterModelProviderCredentialManagementTransportRequestError,
);
assert.throws(
() =>
normalizeClusterModelProviderCredentialManagementCommand({
...bindCommand(),
operation: 'provider-credential.inspect',
}),
ClusterModelProviderCredentialManagementTransportRequestError,
);
});
test('transport rejects weak or stale authentication before service use', async () => {
for (const candidate of [
principal({ assurance: 'single_factor' }),
principal({ authenticatedAtMs: NOW_MS - 300_001 }),
]) {
const { transport, calls } = fixture();
await assert.rejects(
transport.execute(bindCommand(), {
async authenticate() {
return candidate;
},
}),
ClusterModelProviderCredentialManagementTransportAuthenticationError,
);
assert.equal(calls.length, 0);
}
});
test('transport returns an exact content-free audit page', async () => {
const { transport, calls } = fixture();
const result = await transport.execute(auditCommand(), {
async authenticate() {
return principal();
},
});
assert.equal(calls[0].operation, 'audit.list');
assert.equal(calls[0].request.principal.subject.id, 'owner-a');
assert.equal(result.operation, 'provider-credential.audit.list');
assert.equal(result.audit.records[0].operation, 'provider-credential.bind');
assert.doesNotMatch(
JSON.stringify(result),
/secretRef|bindingDigest|transitionDigest|authenticationId|openai/i,
);
});
test('transport injects identity into a server-bounded test plan', async () => {
const { transport, calls } = fixture();
const result = await transport.execute(testPlanCommand(), {
async authenticate() {
return principal();
},
});
assert.equal(calls[0].operation, 'test.plan');
assert.equal(calls[0].request.principal.subject.id, 'owner-a');
assert.equal(result.operation, 'provider-credential.test.plan');
assert.equal(result.plan.endpoint.maxCostMicrousd, 0);
assert.equal(result.plan.endpoint.retryLimit, 0);
assert.doesNotMatch(JSON.stringify(result), /secretRef|token/i);
});
test('transport rejects caller-controlled test endpoint and budgets', () => {
for (const widened of [
{ baseUrl: 'https://attacker.example/v1/' },
{ deadlineMs: 60_000 },
{ secretRef: 'qlsecret:project-a/openai-token' },
]) {
assert.throws(
() =>
normalizeClusterModelProviderCredentialManagementCommand({
...testPlanCommand(),
request: { ...testPlanCommand().request, ...widened },
}),
ClusterModelProviderCredentialManagementTransportRequestError,
);
}
});
test('transport rejects widened audit filters before authentication', () => {
assert.throws(
() =>
normalizeClusterModelProviderCredentialManagementCommand({
...auditCommand(),
request: { ...auditCommand().request, provider: 'openai-compatible' },
}),
ClusterModelProviderCredentialManagementTransportRequestError,
);
});
@@ -0,0 +1,279 @@
const assert = require('node:assert/strict');
const { randomUUID } = require('node:crypto');
const test = require('node:test');
const {
createModelProviderCredentialTestAllowlist,
createModelProviderCredentialTestExecution,
createModelProviderCredentialTestPlan,
} = require('@qinglong/ai/model-provider-credential-test-connection');
const {
ModelProviderCredentialTestExecutionUnavailableError,
} = require('@qinglong/ai/postgres-model-provider-credential-test-connection');
const {
MODEL_PROVIDER_CREDENTIAL_BINDING_SCHEMA,
} = require('@qinglong/ai/provider-credential');
const {
createModelProviderCredentialTestExecutor,
} = require('@qinglong/cluster-admin/model-provider-credential-test-executor');
const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
function fixture({
maxModels = 4,
providerFailure = false,
transportReadiness = false,
useDefaultClock = false,
} = {}) {
const events = [];
const allowlist = createModelProviderCredentialTestAllowlist({
revision: 'catalog-v1',
providers: [
{
provider: 'openai-compatible',
adapter: 'openai-compatible',
baseUrl: 'https://provider.example.test/v1/',
revision: 'endpoint-v1',
deadlineMs: 5_000,
maxResponseBytes: 64 * 1_024,
maxModels,
maxCostMicrousd: 0,
retryLimit: 0,
},
],
});
const plan = createModelProviderCredentialTestPlan({
testId: randomUUID(),
requestId: `request-${randomUUID()}`,
projectId: 'project-a',
provider: 'openai-compatible',
endpoint: allowlist.providers[0],
requestedBy: { type: 'user', id: 'owner-a' },
fence: { projectVersion: 3, bindingVersion: 7 },
plannedAtMs: 100,
expiresAtMs: 60_100,
});
const executionId = randomUUID();
const state = { execution: null, result: null, loseCompletion: false };
const repository = {
async beginExecution(input) {
events.push('begin');
if (state.execution) {
return {
status: 'existing',
plan,
execution: state.execution,
result: state.result,
};
}
state.execution = createModelProviderCredentialTestExecution({
executionId: input.executionId,
testId: input.testId,
planDigest: plan.planDigest,
startedAtMs: 200,
});
events.push('intent-committed');
return {
status: 'created',
plan,
execution: state.execution,
result: null,
};
},
async complete(result) {
events.push('complete');
if (!state.result) state.result = result;
if (state.loseCompletion) {
state.loseCompletion = false;
throw new ModelProviderCredentialTestExecutionUnavailableError();
}
return {
status: state.result === result ? 'created' : 'existing',
result: state.result,
};
},
};
const secretRef = createSecretRef({
projectId: 'project-a',
name: 'openai-token',
});
const credentials = {
async resolveModelProviderCredentialBinding() {
events.push('binding');
return {
schema: MODEL_PROVIDER_CREDENTIAL_BINDING_SCHEMA,
projectId: 'project-a',
provider: 'openai-compatible',
revision: 'credential-v1',
secretRef,
scheme: 'bearer',
};
},
async record(record) {
events.push('audit');
assert.equal(record.requestId, executionId);
assert.equal(record.operation, 'list_models');
},
};
const secrets = {
async resolveProjectSecretMaterial() {
events.push('secret');
const bytes = Buffer.from('provider-token');
return {
secretRef,
bytes,
dispose() {
events.push('secret-disposed');
bytes.fill(0);
},
};
},
};
let fetchCalls = 0;
let modelCount = 2;
let monotonic = 0;
const executor = createModelProviderCredentialTestExecutor({
repository,
credentials,
secrets,
now: () => 1_000,
...(useDefaultClock
? {}
: {
monotonicNow: () => {
monotonic += 10;
return monotonic;
},
}),
...(transportReadiness
? {
async transportReady(baseUrl, signal) {
events.push('transport-ready');
assert.equal(baseUrl, 'https://provider.example.test/v1/');
assert.equal(signal.aborted, false);
},
}
: {}),
async fetch(url, init) {
events.push('fetch');
fetchCalls += 1;
assert.equal(
events.indexOf('intent-committed') < events.indexOf('fetch'),
true,
);
assert.equal(events.indexOf('audit') < events.indexOf('fetch'), true);
assert.equal(url.toString(), 'https://provider.example.test/v1/models');
assert.equal(init.method, 'GET');
assert.equal(init.headers.authorization, 'Bearer provider-token');
if (providerFailure) throw new Error('provider unavailable');
return new Response(
JSON.stringify({
data: Array.from({ length: modelCount }, (_, index) => ({
id: `model-${index + 1}`,
})),
}),
{
status: 200,
headers: { 'content-type': 'application/json' },
},
);
},
});
return {
allowlist,
plan,
executionId,
events,
executor,
state,
fetchCalls: () => fetchCalls,
setModelCount(value) {
modelCount = value;
},
};
}
function input(value) {
return {
executionId: value.executionId,
testId: value.plan.testId,
allowlist: value.allowlist,
};
}
test('commits intent and credential-use audit before exactly one provider call', async () => {
const value = fixture();
const result = await value.executor.execute(input(value));
assert.equal(result.status, 'completed');
assert.equal(result.result.outcome, 'reachable');
assert.equal(result.result.modelCount, 2);
assert.equal(value.fetchCalls(), 1);
assert.equal(value.events.filter((event) => event === 'complete').length, 1);
assert.doesNotMatch(JSON.stringify(result), /provider-token|secretRef/i);
});
test('default monotonic clock retains its performance receiver', async () => {
const value = fixture({ useDefaultClock: true });
const result = await value.executor.execute(input(value));
assert.equal(result.status, 'completed');
assert.equal(result.result.outcome, 'reachable');
assert.equal(Number.isSafeInteger(result.result.durationMs), true);
});
test('waits for credential-free transport readiness after durable intent', async () => {
const value = fixture({ transportReadiness: true });
const result = await value.executor.execute(input(value));
assert.equal(result.result.outcome, 'reachable');
assert.equal(
value.events.indexOf('intent-committed') <
value.events.indexOf('transport-ready'),
true,
);
assert.equal(
value.events.indexOf('transport-ready') < value.events.indexOf('binding'),
true,
);
});
test('existing intent without result is outcome_unknown and never reaches provider', async () => {
const value = fixture();
value.state.execution = createModelProviderCredentialTestExecution({
executionId: value.executionId,
testId: value.plan.testId,
planDigest: value.plan.planDigest,
startedAtMs: 200,
});
const result = await value.executor.execute(input(value));
assert.equal(result.status, 'outcome_unknown');
assert.equal(result.result, null);
assert.equal(value.fetchCalls(), 0);
assert.deepEqual(value.events, ['begin']);
});
test('provider failure and response budget excess persist unreachable only', async () => {
const failed = fixture({ providerFailure: true });
const unavailable = await failed.executor.execute(input(failed));
assert.equal(unavailable.result.outcome, 'unreachable');
assert.equal(unavailable.result.modelCount, null);
const oversized = fixture({ maxModels: 1 });
oversized.setModelCount(2);
const bounded = await oversized.executor.execute(input(oversized));
assert.equal(bounded.result.outcome, 'unreachable');
assert.equal(bounded.result.modelCount, null);
assert.equal(oversized.fetchCalls(), 1);
});
test('completion COMMIT response loss retries only the exact result', async () => {
const value = fixture();
value.state.loseCompletion = true;
const result = await value.executor.execute(input(value));
assert.equal(result.status, 'completed');
assert.equal(value.fetchCalls(), 1);
assert.equal(value.events.filter((event) => event === 'complete').length, 2);
const replay = await value.executor.execute(input(value));
assert.equal(replay.status, 'existing');
assert.deepEqual(replay.result, result.result);
assert.equal(value.fetchCalls(), 1);
});
@@ -0,0 +1,288 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const { resolve } = require('node:path');
const test = require('node:test');
const {
createModelProviderCredentialTestAllowlist,
createModelProviderCredentialTestExecution,
createModelProviderCredentialTestPlan,
createModelProviderCredentialTestResult,
} = require('@qinglong/ai/model-provider-credential-test-connection');
const {
ModelProviderCredentialTestExecutorProcessConfigError,
loadModelProviderCredentialTestExecutorProcessConfig,
runModelProviderCredentialTestExecutorProcess,
} = require('@qinglong/cluster-admin/model-provider-credential-test-executor-process');
const COMMAND = Object.freeze({
schemaVersion: 1,
executionId: '319f7094-a853-4f3b-82ab-dfa08e6bd1c4',
testId: '419f7094-a853-4f3b-82ab-dfa08e6bd1c5',
});
const CLI = resolve(
__dirname,
'../dist/model-provider-credential/modelProviderCredentialTestExecutorCli.js',
);
function environment(overrides = {}) {
return {
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_EXECUTOR_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_COMMAND_FILE:
'/run/ql3-provider-test/command.json',
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_ALLOWLIST_FILE:
'/run/ql3-provider-test/allowlist.json',
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_SECRET_ROOT:
'/run/ql3-provider-test/secrets',
QL3_POSTGRES_AI_CREDENTIAL_TESTER_URL:
'postgresql://ql3_ai_credential_tester:secret@postgres.example.test/ql3',
QL3_POSTGRES_AI_CREDENTIAL_TESTER_TLS_MODE: 'disable',
QL3_POSTGRES_AI_CREDENTIAL_TESTER_ALLOW_INSECURE: 'true',
...overrides,
};
}
function evidence() {
return {
ready: true,
currentUser: 'ql3_ai_credential_tester',
migrationIds: ['pg-9015-ai-model-provider-credential-test-connection'],
writablePrimary: true,
testerAuthority: true,
leastPrivilege: true,
};
}
function testData() {
const allowlist = createModelProviderCredentialTestAllowlist({
revision: 'catalog-v1',
providers: [
{
provider: 'openai-compatible',
adapter: 'openai-compatible',
baseUrl: 'https://provider.example.test/v1/',
revision: 'endpoint-v1',
deadlineMs: 5_000,
maxResponseBytes: 64 * 1_024,
maxModels: 64,
maxCostMicrousd: 0,
retryLimit: 0,
},
],
});
const plan = createModelProviderCredentialTestPlan({
testId: COMMAND.testId,
requestId: 'test-request-1',
projectId: 'project-a',
provider: 'openai-compatible',
endpoint: allowlist.providers[0],
requestedBy: { type: 'user', id: 'owner-a' },
fence: { projectVersion: 3, bindingVersion: 7 },
plannedAtMs: 100,
expiresAtMs: 60_100,
});
const execution = createModelProviderCredentialTestExecution({
executionId: COMMAND.executionId,
testId: COMMAND.testId,
planDigest: plan.planDigest,
startedAtMs: 200,
});
const result = createModelProviderCredentialTestResult({
executionId: COMMAND.executionId,
testId: COMMAND.testId,
planDigest: plan.planDigest,
outcome: 'reachable',
modelCount: 2,
durationMs: 40,
completedAtMs: 240,
});
return { allowlist, plan, execution, result };
}
test('disabled one-shot tester reads no database, file or Secret authority', async () => {
const reads = [];
const disabled = new Proxy(
{ QL3_MODEL_PROVIDER_CREDENTIAL_TEST_EXECUTOR_ENABLED: 'false' },
{
get(target, property) {
reads.push(property);
if (
property === 'QL3_MODEL_PROVIDER_CREDENTIAL_TEST_EXECUTOR_ENABLED'
) {
return target[property];
}
throw new Error(`unexpected read: ${String(property)}`);
},
},
);
const result = await runModelProviderCredentialTestExecutorProcess({
environment: disabled,
});
assert.deepEqual(result, { status: 'disabled' });
assert.deepEqual(reads, [
'QL3_MODEL_PROVIDER_CREDENTIAL_TEST_EXECUTOR_ENABLED',
]);
});
test('loads a bounded single-connection tester configuration', () => {
const config = loadModelProviderCredentialTestExecutorProcessConfig(
environment({
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_DENY_CANARY_HOST:
'kubernetes.default.svc',
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_DENY_CANARY_PORT: '443',
}),
);
assert.equal(config.enabled, true);
assert.equal(config.profile, 'cluster-admin');
assert.equal(config.database.pool.maxConnections, 1);
assert.equal(
config.database.pool.applicationName,
'qinglong3-ai-credential-tester',
);
assert.equal(config.database.connection.tls.mode, 'disable');
assert.deepEqual(config.networkPolicyDenyCanary, {
host: 'kubernetes.default.svc',
port: 443,
});
assert.equal(config.database.connection.applicationName, undefined);
});
test('rejects profile, path, pool and implicit TLS widening', () => {
for (const candidate of [
environment({ QL3_PROFILE: 'cluster-control' }),
environment({
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_COMMAND_FILE: 'command.json',
}),
environment({ QL3_POSTGRES_AI_CREDENTIAL_TESTER_POOL_MAX: '2' }),
environment({
QL3_POSTGRES_AI_CREDENTIAL_TESTER_ALLOW_INSECURE: 'false',
}),
environment({
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_DENY_CANARY_HOST:
'kubernetes.default.svc',
}),
environment({
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_DENY_CANARY_HOST:
'kubernetes.default.svc',
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_DENY_CANARY_PORT: '70000',
}),
]) {
assert.throws(
() => loadModelProviderCredentialTestExecutorProcessConfig(candidate),
ModelProviderCredentialTestExecutorProcessConfigError,
);
}
});
test('composes one execution, exact readiness and guaranteed database close', async () => {
const data = testData();
const pool = { async query() {}, async connect() {} };
const calls = [];
let closed = 0;
const processResult = await runModelProviderCredentialTestExecutorProcess({
environment: environment(),
command: COMMAND,
allowlist: data.allowlist,
async openDatabase() {
calls.push('open');
return {
pool,
async close() {
calls.push('close');
closed += 1;
},
};
},
async assertReady(candidate) {
calls.push('ready');
assert.equal(candidate, pool);
return evidence();
},
secrets: {
async verify() {},
async resolveProjectSecretMaterial() {
throw new Error('injected executor must own secret use');
},
},
executor: {
async execute(input) {
calls.push('execute');
assert.deepEqual(input, {
executionId: COMMAND.executionId,
testId: COMMAND.testId,
allowlist: data.allowlist,
});
return {
status: 'completed',
plan: data.plan,
execution: data.execution,
result: data.result,
};
},
},
});
assert.equal(processResult.status, 'completed');
assert.equal(processResult.test.result.modelCount, 2);
assert.equal(closed, 1);
assert.deepEqual(calls, ['open', 'ready', 'execute', 'close']);
assert.doesNotMatch(JSON.stringify(processResult), /secretRef|token/i);
});
test('closes the tester database when execution fails', async () => {
const data = testData();
let closed = 0;
const failure = new Error('execution failed');
await assert.rejects(
runModelProviderCredentialTestExecutorProcess({
environment: environment(),
command: COMMAND,
allowlist: data.allowlist,
async openDatabase() {
return {
pool: { async query() {}, async connect() {} },
async close() {
closed += 1;
},
};
},
async assertReady() {
return evidence();
},
secrets: {
async verify() {},
async resolveProjectSecretMaterial() {},
},
executor: {
async execute() {
throw failure;
},
},
}),
failure,
);
assert.equal(closed, 1);
});
test('CLI emits only content-free stable facts', () => {
const help = spawnSync(process.execPath, [CLI, '--help'], {
encoding: 'utf8',
});
assert.equal(help.status, 0);
assert.equal(help.stdout, 'Usage: ql3-provider-credential-test-execute\n');
const disabled = spawnSync(process.execPath, [CLI], {
encoding: 'utf8',
env: {
...process.env,
QL3_MODEL_PROVIDER_CREDENTIAL_TEST_EXECUTOR_ENABLED: 'false',
},
});
assert.equal(disabled.status, 0);
assert.deepEqual(JSON.parse(disabled.stdout), {
schemaVersion: 1,
component: 'qinglong3-model-provider-credential-test-executor',
event: 'execution_disabled',
});
assert.doesNotMatch(disabled.stdout + disabled.stderr, /secret|token/i);
});
@@ -0,0 +1,38 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
CLUSTER_PLUGIN_PACKAGE_DISPATCH_BATCH_LIMIT,
createClusterPluginPackageApprovedActionDispatcher,
} = require('@qinglong/cluster-admin/plugin-package-approved-action');
test('composes one bounded caller-driven cluster Package dispatcher', async () => {
const pool = {
async query() {
throw new Error('construction must not touch PostgreSQL');
},
async connect() {
throw new Error('construction must not open PostgreSQL');
},
};
const dispatcher = createClusterPluginPackageApprovedActionDispatcher({
pool,
owner: 'cluster_package_dispatcher_1',
clock: () => 100,
createId: () => 'dispatcher-id-1',
});
let observedLimit = null;
dispatcher.repository.listDueExecutions = async (query) => {
observedLimit = query.limit;
return { executions: [], truncated: false };
};
const summary = await dispatcher.dispatchBatch();
assert.equal(summary.scanned, 0);
assert.equal(summary.truncated, false);
assert.equal(observedLimit, CLUSTER_PLUGIN_PACKAGE_DISPATCH_BATCH_LIMIT);
assert.equal(
require('@qinglong/cluster-admin')
.createClusterPluginPackageApprovedActionDispatcher,
undefined,
);
});
@@ -0,0 +1,76 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ClusterPluginPackageExecutorProcessConfigError,
loadClusterPluginPackageExecutorProcessConfig,
runClusterPluginPackageExecutorProcess,
} = require('@qinglong/cluster-admin/plugin-package-executor-process');
function environment(overrides = {}) {
return {
QL3_PLUGIN_PACKAGE_EXECUTOR_ENABLED: 'true',
QL3_PLUGIN_PACKAGE_EXECUTOR_OWNER: 'cluster_package_executor_1',
QL3_PLUGIN_PACKAGE_EXECUTOR_APPROVAL_BATCH_SIZE: '4',
QL3_PLUGIN_PACKAGE_EXECUTOR_DISPATCH_BATCH_SIZE: '4',
QL3_PLUGIN_PACKAGE_EXECUTOR_MAX_BATCHES: '2',
QL3_PLUGIN_PACKAGE_EXECUTOR_LEASE_DURATION_MS: '600000',
QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_PAGE_SIZE: '8',
QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_MAX_PAGES: '4',
QL3_POSTGRES_PACKAGE_EXECUTOR_URL:
'postgresql://ql3_package_executor:secret@postgres/qinglong',
QL3_POSTGRES_TLS_MODE: 'disable',
QL3_POSTGRES_ALLOW_INSECURE: 'true',
...overrides,
};
}
test('disabled executor opens no PostgreSQL authority', async () => {
let opened = 0;
const result = await runClusterPluginPackageExecutorProcess({
environment: { QL3_PLUGIN_PACKAGE_EXECUTOR_ENABLED: 'false' },
async openDatabase() {
opened += 1;
throw new Error('must not open');
},
});
assert.deepEqual(result, { status: 'disabled' });
assert.equal(opened, 0);
});
test('loads bounded low-footprint Package-executor configuration', () => {
const config = loadClusterPluginPackageExecutorProcessConfig(environment());
assert.equal(config.enabled, true);
assert.equal(config.owner, 'cluster_package_executor_1');
assert.equal(config.approvalBatchSize, 4);
assert.equal(config.dispatchBatchSize, 4);
assert.equal(config.maxBatches, 2);
assert.equal(config.revocationPageSize, 8);
assert.equal(config.revocationMaxPages, 4);
assert.equal(config.database.pool.maxConnections, 2);
assert.equal(config.database.connection.tls.mode, 'disable');
});
test('rejects implicit insecure PostgreSQL and unbounded work', () => {
for (const invalid of [
environment({ QL3_POSTGRES_ALLOW_INSECURE: undefined }),
environment({ QL3_PLUGIN_PACKAGE_EXECUTOR_MAX_BATCHES: '65' }),
environment({ QL3_PLUGIN_PACKAGE_EXECUTOR_REVOCATION_PAGE_SIZE: '129' }),
environment({ QL3_PLUGIN_PACKAGE_EXECUTOR_OWNER: 'not safe' }),
]) {
assert.throws(
() => loadClusterPluginPackageExecutorProcessConfig(invalid),
ClusterPluginPackageExecutorProcessConfigError,
);
}
});
test('keeps executor authority off the cluster-admin root', () => {
const root = require('@qinglong/cluster-admin');
const manifest = require('../package.json');
assert.equal(root.runClusterPluginPackageExecutorProcess, undefined);
assert.equal(
manifest.bin['ql3-plugin-package-execute'],
'dist/plugin-package/executor/pluginPackageExecutorCli.js',
);
});
@@ -0,0 +1,386 @@
const assert = require('node:assert/strict');
const { createHash, generateKeyPairSync, sign } = require('node:crypto');
const { test } = require('node:test');
const {
ClusterPluginPackageIdentityAssertionAuthenticationError,
ClusterPluginPackageIdentityAssertionConfigurationError,
createClusterPluginPackageIdentityAssertionVerifier,
} = require('@qinglong/cluster-admin/plugin-package-identity-assertion');
const {
createClusterPluginPackageManagementTransport,
} = require('@qinglong/cluster-admin/plugin-package-management-transport');
const NOW = 1_700_000_000_000;
const ISSUER = 'https://identity.example.test/ql3';
const AUDIENCE = 'qinglong3-plugin-package-management';
const PURPOSE = 'plugin-package-management';
const TYPE = 'ql3-plugin-package-management+jwt';
function reviewedKey(algorithm = 'EdDSA', kid = 'identity-key-1') {
const pair =
algorithm === 'RS256'
? generateKeyPairSync('rsa', {
modulusLength: 2048,
publicExponent: 0x10001,
})
: algorithm === 'ES256'
? generateKeyPairSync('ec', { namedCurve: 'P-256' })
: generateKeyPairSync('ed25519');
return {
algorithm,
kid,
privateKey: pair.privateKey,
publicJwk: {
...pair.publicKey.export({ format: 'jwk' }),
kid,
use: 'sig',
alg: algorithm,
},
};
}
function encode(value) {
return Buffer.from(JSON.stringify(value)).toString('base64url');
}
function assertion(key, claimOverrides = {}, headerOverrides = {}) {
const header = {
typ: TYPE,
alg: key.algorithm,
kid: key.kid,
...headerOverrides,
};
const claims = {
iss: ISSUER,
aud: AUDIENCE,
sub: 'cluster-reviewer',
jti: 'assertion-session-1',
iat: NOW / 1_000 - 10,
auth_time: NOW / 1_000 - 20,
exp: NOW / 1_000 + 120,
acr: 'urn:example:assurance:mfa',
amr: ['pwd', 'otp'],
ql3_purpose: PURPOSE,
...claimOverrides,
};
const protectedSegment = encode(header);
const payloadSegment = encode(claims);
const signed = Buffer.from(`${protectedSegment}.${payloadSegment}`, 'ascii');
const signature =
key.algorithm === 'EdDSA'
? sign(null, signed, key.privateKey)
: key.algorithm === 'ES256'
? sign('sha256', signed, {
key: key.privateKey,
dsaEncoding: 'ieee-p1363',
})
: sign('RSA-SHA256', signed, key.privateKey);
return `${protectedSegment}.${payloadSegment}.${signature.toString(
'base64url',
)}`;
}
function verifier(key, overrides = {}) {
return createClusterPluginPackageIdentityAssertionVerifier({
issuer: ISSUER,
audience: AUDIENCE,
keys: [key.publicJwk],
assuranceMappings: [
{
acr: 'urn:example:assurance:mfa',
assurance: 'multi_factor',
requiredAmr: ['pwd', 'otp'],
},
{
acr: 'urn:example:assurance:hardware',
assurance: 'hardware',
requiredAmr: ['fido', 'hwk'],
},
],
now: () => NOW,
...overrides,
});
}
function fakeManagementService() {
const calls = [];
return {
calls,
service: {
async propose(request) {
calls.push(request);
return {
proposalStatus: 'created',
approvalStatus: 'created',
proposal: {
actionRef: request.actionRef,
projectId: 'default',
actionInput: request.actionInput,
actionDigest: 'a'.repeat(64),
previewDigest: 'b'.repeat(64),
proposalDigest: 'c'.repeat(64),
createdAtMs: request.requestedAtMs,
},
approvalRequest: {
id: request.approvalRequestId,
projectId: 'default',
version: 1,
state: 'pending',
action: {
actionDigest: 'a'.repeat(64),
previewDigest: 'b'.repeat(64),
},
risk: 'high',
decisionMode: 'separation_of_duty',
requestedAtMs: request.requestedAtMs,
expiresAtMs: request.requestedAtMs + 60_000,
decision: null,
decisionReasonCode: null,
decidedAtMs: null,
dispatchId: null,
consumedAtMs: null,
},
};
},
async decide() {
throw new Error('not used');
},
async inspect() {
return { proposal: null, approvalRequest: null };
},
async inspectAuthorized() {
return { proposal: null, approvalRequest: null };
},
async inspectInstallationAuthorized() {
return null;
},
async listInstallationsAuthorized() {
return { items: [], truncated: false };
},
},
};
}
function proposeCommand() {
return {
schemaVersion: 1,
operation: 'plugin-package.propose',
request: {
actionRef: 'package:cluster-monitor:1',
approvalRequestId: 'approval-cluster-monitor-1',
proposalAuditEventId: 'proposal-audit-1',
approvalAuditEventId: 'approval-audit-1',
actionInput: {
manifest: {
metadata: { name: 'cluster-monitor', version: '1.0.0' },
},
plan: { operation: 'install' },
source: { kind: 'registry' },
architecture: 'arm64',
deploymentProfile: 'cluster',
targetGeneration: 1,
},
},
};
}
test('verifies a dedicated assertion and injects no raw token identity into management', async () => {
const key = reviewedKey();
const token = assertion(key);
const identity = verifier(key);
const principal = identity.verify(token);
const expectedAuthenticationId = `ql3oidc.${createHash('sha256')
.update(ISSUER)
.update('\0')
.update('assertion-session-1')
.digest('base64url')}`;
assert.deepEqual(principal, {
subject: { type: 'user', id: 'cluster-reviewer' },
authenticationId: expectedAuthenticationId,
authenticatedAtMs: NOW - 20_000,
expiresAtMs: NOW + 120_000,
assurance: 'multi_factor',
});
const fixture = fakeManagementService();
const transport = createClusterPluginPackageManagementTransport({
service: fixture.service,
now: () => NOW,
});
const result = await transport.execute(
proposeCommand(),
identity.bind(token),
);
assert.equal(fixture.calls.length, 1);
assert.deepEqual(fixture.calls[0].principal, principal);
const serialized = JSON.stringify({ result, request: fixture.calls[0] });
assert.equal(serialized.includes(token), false);
assert.equal(serialized.includes('assertion-session-1'), false);
});
test('keeps Plugin Package and Worker credential assertion purposes disjoint', () => {
const key = reviewedKey();
const workerAudience = 'qinglong3-worker-credential-management';
const workerProfile = {
type: 'ql3-worker-credential-management+jwt',
purpose: 'worker-credential-management',
};
const workerToken = assertion(
key,
{
aud: workerAudience,
ql3_purpose: workerProfile.purpose,
},
{ typ: workerProfile.type },
);
const workerVerifier = verifier(key, {
audience: workerAudience,
assertionProfile: workerProfile,
});
assert.equal(
workerVerifier.verify(workerToken).subject.id,
'cluster-reviewer',
);
assert.throws(
() => verifier(key).verify(workerToken),
ClusterPluginPackageIdentityAssertionAuthenticationError,
);
assert.throws(
() => workerVerifier.verify(assertion(key)),
ClusterPluginPackageIdentityAssertionAuthenticationError,
);
});
test('accepts reviewed EdDSA, ES256 and RS256 key algorithms', () => {
for (const algorithm of ['EdDSA', 'ES256', 'RS256']) {
const key = reviewedKey(algorithm, `key-${algorithm}`);
assert.equal(
verifier(key).verify(assertion(key)).subject.id,
'cluster-reviewer',
);
}
});
test('maps hardware only through an exact ACR and complete AMR rule', () => {
const key = reviewedKey();
const identity = verifier(key);
const hardware = identity.verify(
assertion(key, {
acr: 'urn:example:assurance:hardware',
amr: ['fido', 'hwk'],
}),
);
assert.equal(hardware.assurance, 'hardware');
assert.throws(
() =>
identity.verify(
assertion(key, {
acr: 'urn:example:assurance:hardware',
amr: ['fido'],
}),
),
ClusterPluginPackageIdentityAssertionAuthenticationError,
);
});
test('rejects signature, algorithm, key and dedicated-token confusion', () => {
const key = reviewedKey();
const other = reviewedKey('EdDSA', 'identity-key-2');
const identity = verifier(key);
const valid = assertion(key);
const tampered = `${valid.slice(0, -1)}${valid.endsWith('A') ? 'B' : 'A'}`;
for (const candidate of [
tampered,
assertion(other),
assertion(key, {}, { typ: 'JWT' }),
assertion(key, {}, { alg: 'RS256' }),
assertion(key, { unexpected_claim: 'must-fail-closed' }),
`${valid}.extra`,
'a'.repeat(9 * 1_024),
]) {
assert.throws(
() => identity.verify(candidate),
ClusterPluginPackageIdentityAssertionAuthenticationError,
);
}
});
test('rejects wrong trust domain, inactive lifetime and weak assurance facts', () => {
const key = reviewedKey();
const identity = verifier(key);
for (const overrides of [
{ iss: 'https://other.example.test/ql3' },
{ aud: 'another-service' },
{ ql3_purpose: 'cluster-control' },
{ exp: NOW / 1_000 },
{ iat: NOW / 1_000 + 10 },
{ auth_time: NOW / 1_000 - 301 },
{ exp: NOW / 1_000 + 301 },
{ acr: 'urn:example:assurance:single-factor', amr: ['pwd'] },
{ acr: 'urn:example:assurance:mfa', amr: ['pwd'] },
]) {
assert.throws(
() => identity.verify(assertion(key, overrides)),
ClusterPluginPackageIdentityAssertionAuthenticationError,
);
}
});
test('rejects unreviewed configuration before accepting any assertion', () => {
const key = reviewedKey();
const weakRsa = reviewedKey('RS256', 'weak-rsa');
const weakRsaPair = generateKeyPairSync('rsa', {
modulusLength: 1024,
publicExponent: 0x10001,
});
weakRsa.publicJwk = {
...weakRsaPair.publicKey.export({ format: 'jwk' }),
kid: weakRsa.kid,
use: 'sig',
alg: weakRsa.algorithm,
};
const privateJwk = {
...key.privateKey.export({ format: 'jwk' }),
kid: key.kid,
use: 'sig',
alg: key.algorithm,
};
for (const overrides of [
{ issuer: 'http://identity.example.test/ql3' },
{ keys: [] },
{ keys: [key.publicJwk, key.publicJwk] },
{ keys: [privateJwk] },
{ keys: [weakRsa.publicJwk] },
{
assuranceMappings: [
{
acr: 'urn:example:assurance:mfa',
assurance: 'single_factor',
requiredAmr: ['pwd'],
},
],
},
{ maxAssertionBytes: 32 * 1024 },
{ maxLifetimeMs: 60 * 60_000 },
{ clockSkewMs: 120_000 },
{
assertionProfile: {
type: 'JWT',
purpose: 'worker-credential-management',
},
},
{
assertionProfile: {
type: 'ql3-worker-credential-management+jwt',
purpose: 'Worker Credential Management',
},
},
]) {
assert.throws(
() => verifier(key, overrides),
ClusterPluginPackageIdentityAssertionConfigurationError,
);
}
});
@@ -0,0 +1,548 @@
const assert = require('node:assert/strict');
const { chmod, mkdtemp, rename, rm, writeFile } = require('node:fs/promises');
const { tmpdir } = require('node:os');
const { join } = require('node:path');
const { generateKeyPairSync, sign } = require('node:crypto');
const { test } = require('node:test');
const {
ClusterPluginPackageIdentityKeysetUnavailableError,
createClusterPluginPackageIdentityKeysetFile,
createClusterWorkerCredentialIdentityKeysetFile,
createClusterAutomationIdentityKeysetFile,
createClusterApprovalIdentityKeysetFile,
createClusterModelProviderCredentialIdentityKeysetFile,
} = require('@qinglong/cluster-admin/plugin-package-identity-keyset');
const NOW_MS = 1_700_000_000_000;
const ISSUER = 'https://identity.example.test/';
const AUDIENCE = 'qinglong3-package-management';
function reviewedKey(kid) {
const { privateKey, publicKey } = generateKeyPairSync('ed25519');
return {
kid,
privateKey,
publicJwk: {
...publicKey.export({ format: 'jwk' }),
alg: 'EdDSA',
kid,
use: 'sig',
},
};
}
function keyset(generation, keys, revokedKids = []) {
return {
schemaVersion: 1,
generation,
issuer: ISSUER,
audience: AUDIENCE,
keys: keys.map((key) => key.publicJwk),
revokedKids,
assuranceMappings: [
{
acr: 'urn:ql3:mfa',
assurance: 'multi_factor',
requiredAmr: ['pwd', 'otp'],
},
],
constraints: {
maxAssertionBytes: 8 * 1024,
maxLifetimeMs: 5 * 60 * 1000,
maxAuthenticationAgeMs: 5 * 60 * 1000,
clockSkewMs: 5 * 1000,
},
};
}
function assertion(key, overrides = {}) {
const header = Buffer.from(
JSON.stringify({
alg: 'EdDSA',
kid: key.kid,
typ: 'ql3-plugin-package-management+jwt',
}),
).toString('base64url');
const now = Math.floor(NOW_MS / 1000);
const payload = Buffer.from(
JSON.stringify({
acr: 'urn:ql3:mfa',
amr: ['pwd', 'otp'],
aud: AUDIENCE,
auth_time: now - 10,
exp: now + 120,
iat: now,
iss: ISSUER,
jti: `assertion-${key.kid}`,
ql3_purpose: 'plugin-package-management',
sub: 'user-1',
...overrides,
}),
).toString('base64url');
const signed = `${header}.${payload}`;
return `${signed}.${sign(
null,
Buffer.from(signed, 'ascii'),
key.privateKey,
).toString('base64url')}`;
}
function workerAssertion(key, overrides = {}) {
const header = Buffer.from(
JSON.stringify({
alg: 'EdDSA',
kid: key.kid,
typ: 'ql3-worker-credential-management+jwt',
}),
).toString('base64url');
const now = Math.floor(NOW_MS / 1000);
const payload = Buffer.from(
JSON.stringify({
acr: 'urn:ql3:mfa',
amr: ['pwd', 'otp'],
aud: 'qinglong3-worker-credential-management',
auth_time: now - 10,
exp: now + 120,
iat: now,
iss: ISSUER,
jti: `worker-assertion-${key.kid}`,
ql3_purpose: 'worker-credential-management',
sub: 'worker-operator-1',
...overrides,
}),
).toString('base64url');
const signed = `${header}.${payload}`;
return `${signed}.${sign(
null,
Buffer.from(signed, 'ascii'),
key.privateKey,
).toString('base64url')}`;
}
function automationAssertion(key, overrides = {}) {
const header = Buffer.from(
JSON.stringify({
alg: 'EdDSA',
kid: key.kid,
typ: 'ql3-automation-management+jwt',
}),
).toString('base64url');
const now = Math.floor(NOW_MS / 1000);
const payload = Buffer.from(
JSON.stringify({
acr: 'urn:ql3:mfa',
amr: ['pwd', 'otp'],
aud: 'qinglong3-automation-management',
auth_time: now - 10,
exp: now + 120,
iat: now,
iss: ISSUER,
jti: `automation-assertion-${key.kid}`,
ql3_purpose: 'automation-management',
sub: 'automation-operator-1',
...overrides,
}),
).toString('base64url');
const signed = `${header}.${payload}`;
return `${signed}.${sign(
null,
Buffer.from(signed, 'ascii'),
key.privateKey,
).toString('base64url')}`;
}
function approvalAssertion(key, overrides = {}) {
const header = Buffer.from(
JSON.stringify({
alg: 'EdDSA',
kid: key.kid,
typ: 'ql3-approval-management+jwt',
}),
).toString('base64url');
const now = Math.floor(NOW_MS / 1000);
const payload = Buffer.from(
JSON.stringify({
acr: 'urn:ql3:mfa',
amr: ['pwd', 'otp'],
aud: 'qinglong3-approval-management',
auth_time: now - 10,
exp: now + 120,
iat: now,
iss: ISSUER,
jti: `approval-assertion-${key.kid}`,
ql3_purpose: 'approval-management',
sub: 'approval-owner-1',
...overrides,
}),
).toString('base64url');
const signed = `${header}.${payload}`;
return `${signed}.${sign(
null,
Buffer.from(signed, 'ascii'),
key.privateKey,
).toString('base64url')}`;
}
function providerCredentialAssertion(key, overrides = {}) {
const header = Buffer.from(
JSON.stringify({
alg: 'EdDSA',
kid: key.kid,
typ: 'ql3-model-provider-credential-management+jwt',
}),
).toString('base64url');
const now = Math.floor(NOW_MS / 1000);
const payload = Buffer.from(
JSON.stringify({
acr: 'urn:ql3:mfa',
amr: ['pwd', 'otp'],
aud: 'qinglong3-model-provider-credential-management',
auth_time: now - 10,
exp: now + 120,
iat: now,
iss: ISSUER,
jti: `provider-credential-assertion-${key.kid}`,
ql3_purpose: 'model-provider-credential-management',
sub: 'provider-credential-operator-1',
...overrides,
}),
).toString('base64url');
const signed = `${header}.${payload}`;
return `${signed}.${sign(
null,
Buffer.from(signed, 'ascii'),
key.privateKey,
).toString('base64url')}`;
}
async function atomicWrite(filePath, document) {
const nextPath = `${filePath}.next`;
await writeFile(nextPath, `${JSON.stringify(document)}\n`, { mode: 0o644 });
await rename(nextPath, filePath);
}
async function fixture(run) {
const directory = await mkdtemp(join(tmpdir(), 'ql3-identity-keyset-'));
const filePath = join(directory, 'keyset.json');
try {
return await run({ directory, filePath });
} finally {
await rm(directory, { recursive: true, force: true });
}
}
test('loads one bounded keyset and authenticates through the current generation', async () => {
await fixture(async ({ filePath }) => {
const first = reviewedKey('issuer-key-1');
await atomicWrite(filePath, keyset(1, [first]));
const provider = createClusterPluginPackageIdentityKeysetFile({
filePath,
now: () => NOW_MS,
});
assert.deepEqual(await provider.reload(), {
schemaVersion: 1,
generation: 1,
digest: (await provider.reload()).digest,
issuer: ISSUER,
audience: AUDIENCE,
activeKeyIds: ['issuer-key-1'],
revokedKeyIds: [],
});
const principal = await provider.bind(assertion(first)).authenticate();
assert.deepEqual(principal.subject, { type: 'user', id: 'user-1' });
assert.equal(principal.assurance, 'multi_factor');
});
});
test('loads a Worker credential keyset with a distinct assertion purpose', async () => {
await fixture(async ({ filePath }) => {
const key = reviewedKey('worker-identity-key-1');
await atomicWrite(filePath, {
...keyset(1, [key]),
audience: 'qinglong3-worker-credential-management',
});
const provider = createClusterWorkerCredentialIdentityKeysetFile({
filePath,
now: () => NOW_MS,
});
const principal = await provider.bind(workerAssertion(key)).authenticate();
assert.deepEqual(principal.subject, {
type: 'user',
id: 'worker-operator-1',
});
await assert.rejects(
provider
.bind(
assertion(key, {
aud: 'qinglong3-worker-credential-management',
}),
)
.authenticate(),
{ code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
);
});
});
test('loads an automation keyset with a purpose isolated from other management planes', async () => {
await fixture(async ({ filePath }) => {
const key = reviewedKey('automation-identity-key-1');
await atomicWrite(filePath, {
...keyset(1, [key]),
audience: 'qinglong3-automation-management',
});
const provider = createClusterAutomationIdentityKeysetFile({
filePath,
now: () => NOW_MS,
});
const principal = await provider
.bind(automationAssertion(key))
.authenticate();
assert.deepEqual(principal.subject, {
type: 'user',
id: 'automation-operator-1',
});
await assert.rejects(
provider.bind(workerAssertion(key)).authenticate(),
{ code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
);
await assert.rejects(provider.bind(assertion(key)).authenticate(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID',
});
});
});
test('loads an Approval keyset isolated by type, purpose and audience', async () => {
await fixture(async ({ filePath }) => {
const key = reviewedKey('approval-identity-key-1');
await atomicWrite(filePath, {
...keyset(1, [key]),
audience: 'qinglong3-approval-management',
});
const provider = createClusterApprovalIdentityKeysetFile({
filePath,
now: () => NOW_MS,
});
const principal = await provider.bind(approvalAssertion(key)).authenticate();
assert.deepEqual(principal.subject, {
type: 'user',
id: 'approval-owner-1',
});
await assert.rejects(provider.bind(automationAssertion(key)).authenticate(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID',
});
await assert.rejects(provider.bind(assertion(key)).authenticate(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID',
});
});
});
test('loads a provider credential keyset isolated by type, purpose and audience', async () => {
await fixture(async ({ filePath }) => {
const key = reviewedKey('provider-credential-identity-key-1');
await atomicWrite(filePath, {
...keyset(1, [key]),
audience: 'qinglong3-model-provider-credential-management',
});
const provider = createClusterModelProviderCredentialIdentityKeysetFile({
filePath,
now: () => NOW_MS,
});
const principal = await provider
.bind(providerCredentialAssertion(key))
.authenticate();
assert.deepEqual(principal.subject, {
type: 'user',
id: 'provider-credential-operator-1',
});
await assert.rejects(
provider.bind(automationAssertion(key)).authenticate(),
{ code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
);
await assert.rejects(
provider
.bind(
providerCredentialAssertion(key, {
ql3_purpose: 'automation-management',
}),
)
.authenticate(),
{ code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
);
});
});
test('supports overlap rotation then immediately revokes the previous key', async () => {
await fixture(async ({ filePath }) => {
const first = reviewedKey('issuer-key-1');
const second = reviewedKey('issuer-key-2');
await atomicWrite(filePath, keyset(1, [first]));
const provider = createClusterPluginPackageIdentityKeysetFile({
filePath,
now: () => NOW_MS,
});
await provider.reload();
await atomicWrite(filePath, keyset(2, [first, second]));
assert.deepEqual((await provider.reload()).activeKeyIds, [
'issuer-key-1',
'issuer-key-2',
]);
assert.equal(
(await provider.bind(assertion(second)).authenticate()).subject.id,
'user-1',
);
await atomicWrite(filePath, keyset(3, [first, second], ['issuer-key-1']));
assert.deepEqual(await provider.reload(), {
schemaVersion: 1,
generation: 3,
digest: (await provider.reload()).digest,
issuer: ISSUER,
audience: AUDIENCE,
activeKeyIds: ['issuer-key-2'],
revokedKeyIds: ['issuer-key-1'],
});
await assert.rejects(provider.bind(assertion(first)).authenticate(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID',
});
assert.equal(
(await provider.bind(assertion(second)).authenticate()).subject.id,
'user-1',
);
});
});
test('rejects generation rollback, same-generation rewrite and implicit removal', async () => {
await fixture(async ({ filePath }) => {
const first = reviewedKey('issuer-key-1');
const second = reviewedKey('issuer-key-2');
await atomicWrite(filePath, keyset(1, [first]));
const provider = createClusterPluginPackageIdentityKeysetFile({
filePath,
now: () => NOW_MS,
});
await provider.reload();
await atomicWrite(filePath, keyset(1, [first, second]));
await assert.rejects(provider.reload(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_KEYSET_UNAVAILABLE',
});
await atomicWrite(filePath, keyset(2, [first, second]));
await provider.reload();
await atomicWrite(filePath, keyset(3, [second]));
await assert.rejects(provider.reload(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_KEYSET_UNAVAILABLE',
});
await atomicWrite(filePath, keyset(1, [first]));
await assert.rejects(provider.reload(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_KEYSET_UNAVAILABLE',
});
});
});
test('keeps revocation append-only and rejects stale fallback on file failure', async () => {
await fixture(async ({ filePath }) => {
const first = reviewedKey('issuer-key-1');
const second = reviewedKey('issuer-key-2');
await atomicWrite(filePath, keyset(1, [first, second]));
const provider = createClusterPluginPackageIdentityKeysetFile({
filePath,
now: () => NOW_MS,
});
await provider.reload();
await atomicWrite(filePath, keyset(2, [first, second], ['issuer-key-1']));
await provider.reload();
await atomicWrite(filePath, keyset(3, [first, second]));
await assert.rejects(provider.reload(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_KEYSET_UNAVAILABLE',
});
await atomicWrite(filePath, keyset(3, [first, second], ['issuer-key-1']));
await chmod(filePath, 0o666);
await assert.rejects(
provider.bind(assertion(second)).authenticate(),
ClusterPluginPackageIdentityKeysetUnavailableError,
);
});
});
test('rechecks one durable ledger across unchanged files and fresh replicas', async () => {
await fixture(async ({ filePath }) => {
const first = reviewedKey('issuer-key-1');
const second = reviewedKey('issuer-key-2');
let minimumGeneration = 1;
const observations = [];
const ledger = {
async observe(snapshot) {
observations.push(snapshot);
if (snapshot.generation < minimumGeneration) {
throw new Error('durable generation rollback');
}
minimumGeneration = snapshot.generation;
},
};
await atomicWrite(filePath, keyset(1, [first]));
const firstReplica = createClusterPluginPackageIdentityKeysetFile({
filePath,
now: () => NOW_MS,
ledger,
});
await firstReplica.reload();
await firstReplica.reload();
assert.equal(observations.length, 2);
await atomicWrite(filePath, keyset(2, [first, second], ['issuer-key-1']));
await firstReplica.reload();
assert.equal(minimumGeneration, 2);
await atomicWrite(filePath, keyset(1, [first]));
const restartedReplica = createClusterPluginPackageIdentityKeysetFile({
filePath,
now: () => NOW_MS,
ledger,
});
await assert.rejects(
restartedReplica.reload(),
ClusterPluginPackageIdentityKeysetUnavailableError,
);
await atomicWrite(filePath, keyset(2, [first, second], ['issuer-key-1']));
minimumGeneration = 3;
await assert.rejects(
firstReplica.bind(assertion(second)).authenticate(),
ClusterPluginPackageIdentityKeysetUnavailableError,
);
});
});
test('rejects malformed, private and oversized trust documents', async () => {
await fixture(async ({ filePath }) => {
const first = reviewedKey('issuer-key-1');
const provider = createClusterPluginPackageIdentityKeysetFile({
filePath,
maxFileBytes: 4 * 1024,
now: () => NOW_MS,
});
await atomicWrite(filePath, {
...keyset(1, [first]),
keys: [{ ...first.publicJwk, d: 'private' }],
});
await assert.rejects(provider.reload(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_KEYSET_UNAVAILABLE',
});
await writeFile(filePath, Buffer.alloc(4 * 1024 + 1, 0x20), {
mode: 0o644,
});
await assert.rejects(provider.reload(), {
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_KEYSET_UNAVAILABLE',
});
});
});
@@ -0,0 +1,577 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
PLUGIN_PACKAGE_ACTIVATION_INTENT_SCHEMA,
PluginPackageActivationConflictError,
PluginPackageActivationUnavailableError,
} = require('@qinglong/runtime-core/plugin-package-activation');
const {
createPluginPackageResourceGeneration,
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
const {
PLUGIN_PACKAGE_API_VERSION,
PLUGIN_PACKAGE_KIND,
planPluginPackageInstall,
} = require('@qinglong/runtime-core/plugin-package');
const {
PluginPackageInstallTransitionConflictError,
createPluginPackageInstall,
createPluginPackageLock,
pluginPackageInstallActionDigest,
pluginPackageInstallCommit,
pluginPackageInstallPlanDigest,
transitionPluginPackageInstall,
} = require('@qinglong/runtime-core/plugin-package-install');
const {
PluginPackageRecoveryCoordinator,
} = require('@qinglong/runtime-core/plugin-package-recovery');
const {
PluginPackageKubernetesActivationPublisher,
} = require('../dist/plugin-package/recovery/pluginPackageKubernetesActivation');
function apiError(code) {
return Object.assign(new Error(`Kubernetes API ${code}`), { code });
}
class FakeConfigMapApi {
constructor() {
this.items = new Map();
this.revision = 0;
this.createCalls = 0;
this.replaceCalls = 0;
this.readCalls = 0;
this.beforeRead = null;
this.loseCreateResponse = false;
this.loseReplaceResponse = false;
}
clone(value) {
return structuredClone(value);
}
serverValue(body, current) {
this.revision += 1;
return this.clone({
...body,
metadata: {
...body.metadata,
uid: current?.metadata.uid ?? `uid-${this.revision}`,
resourceVersion: String(this.revision),
},
});
}
async readNamespacedConfigMap({ namespace, name }) {
this.readCalls += 1;
await this.beforeRead?.(this.readCalls);
const current = this.items.get(`${namespace}/${name}`);
if (!current) throw apiError(404);
return this.clone(current);
}
async createNamespacedConfigMap({ namespace, body }) {
this.createCalls += 1;
const key = `${namespace}/${body.metadata.name}`;
if (this.items.has(key)) throw apiError(409);
const created = this.serverValue(body, null);
this.items.set(key, created);
if (this.loseCreateResponse) {
this.loseCreateResponse = false;
throw new Error('create response lost');
}
return this.clone(created);
}
async replaceNamespacedConfigMap({ namespace, name, body }) {
this.replaceCalls += 1;
const key = `${namespace}/${name}`;
const current = this.items.get(key);
if (
!current ||
body.metadata.resourceVersion !== current.metadata.resourceVersion
) {
throw apiError(409);
}
const replaced = this.serverValue(body, current);
this.items.set(key, replaced);
if (this.loseReplaceResponse) {
this.loseReplaceResponse = false;
throw new Error('replace response lost');
}
return this.clone(replaced);
}
}
function intent(overrides = {}) {
const lockDigest = overrides.lockDigest ?? 'a'.repeat(64);
const installationId = overrides.installationId ?? 'install-001';
const targetGeneration = overrides.targetGeneration ?? 1;
const previousActiveLockDigest = overrides.previousActiveLockDigest ?? null;
const contentDigest = overrides.contentDigest ?? 'd'.repeat(64);
const resourceGeneration = createPluginPackageResourceGeneration({
installationId,
projectId: 'default',
packageName: 'example-monitor',
lockDigest,
generation: targetGeneration,
previousActiveLockDigest,
contentDigest,
contents: {
tasks: ['tasks/example.yaml'],
workflows: [],
prompts: [],
tools: [],
},
});
return Object.freeze({
schema: PLUGIN_PACKAGE_ACTIVATION_INTENT_SCHEMA,
installationId,
projectId: 'default',
packageName: 'example-monitor',
lockDigest,
targetGeneration,
previousActiveLockDigest,
stageRef: `cluster-stage:${lockDigest}`,
stageReceiptDigest: overrides.stageReceiptDigest ?? 'b'.repeat(64),
stageEvidenceDigest: overrides.stageEvidenceDigest ?? 'c'.repeat(64),
contentDigest,
resourceGeneration: overrides.resourceGeneration ?? resourceGeneration,
intentDigest: overrides.intentDigest ?? 'e'.repeat(64),
});
}
function exactEvidence(value) {
return Object.freeze({
lockDigest: value.lockDigest,
stageRef: value.stageRef,
stageReceiptDigest: value.stageReceiptDigest,
stageEvidenceDigest: value.stageEvidenceDigest,
contentDigest: value.contentDigest,
});
}
function stagedInstallFixture() {
const manifest = {
apiVersion: PLUGIN_PACKAGE_API_VERSION,
kind: PLUGIN_PACKAGE_KIND,
metadata: {
name: 'example-monitor',
displayName: 'Example Monitor',
version: '1.2.0',
description: 'One bounded package',
license: 'Apache-2.0',
},
spec: {
compatibility: {
qinglong: '>=3.0.0-0 <4.0.0',
architectures: ['arm64'],
deploymentProfiles: ['cluster-control'],
},
runtimes: [],
resources: {
memory: { recommended: '16Mi' },
disk: { install: '4Mi', working: '16Mi' },
},
permissions: {
network: { allowedHosts: [] },
secrets: [],
tools: [],
},
contents: { tasks: [], workflows: [], prompts: [], tools: [] },
},
};
const environment = {
qinglongVersion: '3.0.0-alpha.0',
architecture: 'arm64',
deploymentProfile: 'cluster-control',
runtimes: [],
availableMemoryBytes: 128 * 1024 * 1024,
availableDiskBytes: 256 * 1024 * 1024,
};
const plan = planPluginPackageInstall(manifest, environment);
const action = {
lockId: 'lock-cluster-install',
projectId: 'default',
manifest,
plan,
environment,
source: {
kind: 'offline',
locator: `offline:sha256:${'a'.repeat(64)}`,
artifactDigest: 'a'.repeat(64),
artifactBytes: 2048,
contentDigest: 'd'.repeat(64),
},
architecture: 'arm64',
deploymentProfile: 'cluster-control',
targetGeneration: 1,
};
const lock = createPluginPackageLock({
...action,
approval: {
requestId: 'approval-cluster-install',
requestVersion: 1,
dispatchId: 'dispatch-cluster-install',
actionDigest: pluginPackageInstallActionDigest(action),
previewDigest: pluginPackageInstallPlanDigest(plan),
approvedBy: { type: 'user', id: 'owner-001' },
approvedAtMs: 100,
expiresAtMs: 10_000,
fence: { projectVersion: 1, bindingVersion: 1 },
},
createdAtMs: 200,
});
const queued = createPluginPackageInstall(lock, {
installationId: 'install-cluster-recovery',
mutationId: 'mutation-cluster-create',
occurredAtMs: 201,
});
const staged = transitionPluginPackageInstall(lock, queued, {
type: 'stage_completed',
mutationId: 'mutation-cluster-stage',
occurredAtMs: 202,
stageRef: `cluster-stage:${lock.lockDigest}`,
artifactDigest: lock.source.artifactDigest,
manifestDigest: lock.manifestDigest,
contentDigest: lock.source.contentDigest,
evidenceDigest: 'c'.repeat(64),
});
return { lock, staged };
}
class MemoryInstallRepository {
constructor(lock, record) {
this.lock = lock;
this.record = record;
}
async find(projectId, packageName) {
return this.record.projectId === projectId &&
this.record.packageName === packageName
? this.record
: null;
}
async findLock(lockDigest) {
return lockDigest === this.lock.lockDigest ? this.lock : null;
}
async commit(command) {
const canonical = pluginPackageInstallCommit(this.record, command.record);
assert.deepEqual(command, canonical);
if (
command.expectedVersion !== this.record.version ||
command.expectedRecordDigest !== this.record.recordDigest
) {
throw new PluginPackageInstallTransitionConflictError();
}
this.record = command.record;
return { status: 'committed', record: this.record };
}
async listRecoveryPage({ limit }) {
const records = ['queued', 'staged', 'activating'].includes(
this.record.state,
)
? [this.record].slice(0, limit)
: [];
return { records, truncated: false };
}
}
function publisher(api = new FakeConfigMapApi(), overrides = {}) {
let nowCalls = 0;
const value = new PluginPackageKubernetesActivationPublisher(
api,
{
async verify(activationIntent) {
return overrides.verify
? overrides.verify(activationIntent)
: exactEvidence(activationIntent);
},
},
{
clusterIdentity: 'cluster-primary',
namespace: 'qinglong-system',
now() {
nowCalls += 1;
return overrides.now?.() ?? 500 + nowCalls;
},
},
);
return { api, publisher: value, nowCalls: () => nowCalls };
}
test('publishes one resourceVersion-fenced ConfigMap and exact replays it', async () => {
const fixture = publisher();
const value = intent();
assert.equal(
await fixture.publisher.findActiveResourceGeneration(
'default',
'example-monitor',
),
null,
);
assert.deepEqual(await fixture.publisher.inspect(value), {
status: 'not_published',
});
const receipt = await fixture.publisher.publish(value);
assert.equal(receipt.intentDigest, value.intentDigest);
assert.match(receipt.activationRef, /^k8s-configmap:[0-9a-f]{64}$/);
assert.deepEqual(await fixture.publisher.inspect(value), {
status: 'published',
receipt,
});
assert.deepEqual(await fixture.publisher.publish(value), receipt);
assert.deepEqual(
await fixture.publisher.findActiveResourceGeneration(
'default',
'example-monitor',
),
value.resourceGeneration,
);
await assert.rejects(
fixture.publisher.findActiveResourceGeneration(
'default',
'Example_Monitor',
),
TypeError,
);
assert.equal(fixture.api.createCalls, 1);
assert.equal(fixture.api.replaceCalls, 0);
assert.equal(fixture.nowCalls(), 1);
const [stored] = fixture.api.items.values();
assert.match(stored.metadata.name, /^ql3p-[0-9a-f]{52}$/);
assert.equal(Object.keys(stored.data).join(','), 'active.json');
assert.equal(
stored.metadata.labels['app.kubernetes.io/managed-by'],
'qinglong3',
);
});
test('replaces only the exact previous lock and rejects a stale writer', async () => {
const fixture = publisher();
const first = intent();
await fixture.publisher.publish(first);
const second = intent({
installationId: 'install-002',
lockDigest: '1'.repeat(64),
targetGeneration: 2,
previousActiveLockDigest: first.lockDigest,
stageReceiptDigest: '2'.repeat(64),
stageEvidenceDigest: '3'.repeat(64),
contentDigest: '4'.repeat(64),
intentDigest: '5'.repeat(64),
});
const receipt = await fixture.publisher.publish(second);
assert.equal(receipt.generation, 2);
assert.equal(fixture.api.replaceCalls, 1);
assert.deepEqual(
await fixture.publisher.findActiveResourceGeneration(
'default',
'example-monitor',
),
second.resourceGeneration,
);
const stale = intent({
installationId: 'install-003',
lockDigest: '6'.repeat(64),
targetGeneration: 3,
previousActiveLockDigest: first.lockDigest,
stageReceiptDigest: '7'.repeat(64),
stageEvidenceDigest: '8'.repeat(64),
contentDigest: '9'.repeat(64),
intentDigest: '0'.repeat(64),
});
await assert.rejects(
fixture.publisher.publish(stale),
PluginPackageActivationConflictError,
);
});
test('leaves response loss for recovery inspection without republishing', async () => {
const api = new FakeConfigMapApi();
api.loseCreateResponse = true;
const fixture = publisher(api);
const value = intent();
await assert.rejects(
fixture.publisher.publish(value),
PluginPackageActivationUnavailableError,
);
assert.equal(api.createCalls, 1);
const observation = await fixture.publisher.inspect(value);
assert.equal(observation.status, 'published');
assert.deepEqual(await fixture.publisher.publish(value), observation.receipt);
assert.equal(api.createCalls, 1);
assert.equal(fixture.nowCalls(), 1);
});
test('converges a lost Kubernetes publish response through durable startup recovery', async () => {
const value = stagedInstallFixture();
const repository = new MemoryInstallRepository(value.lock, value.staged);
const api = new FakeConfigMapApi();
api.loseCreateResponse = true;
const fixture = publisher(api);
const coordinator = new PluginPackageRecoveryCoordinator({
repository,
stageProvider: {
async stage() {
throw new Error('a staged recovery must not restage');
},
},
publisher: fixture.publisher,
now: () => 250,
});
const first = await coordinator.recover();
assert.equal(first.retry, 1);
assert.equal(first.safeToAdmit, false);
assert.equal(repository.record.state, 'activating');
assert.equal(api.createCalls, 1);
const second = await coordinator.recover();
assert.equal(second.settled, 1);
assert.equal(second.safeToAdmit, true);
assert.equal(repository.record.state, 'active');
assert.equal(api.createCalls, 1);
});
test('gives two concurrent replacements from one resourceVersion one winner', async () => {
const fixture = publisher();
const first = intent();
await fixture.publisher.publish(first);
const left = intent({
installationId: 'install-left',
lockDigest: '1'.repeat(64),
targetGeneration: 2,
previousActiveLockDigest: first.lockDigest,
stageReceiptDigest: '2'.repeat(64),
stageEvidenceDigest: '3'.repeat(64),
contentDigest: '4'.repeat(64),
intentDigest: '5'.repeat(64),
});
const right = intent({
installationId: 'install-right',
lockDigest: '6'.repeat(64),
targetGeneration: 2,
previousActiveLockDigest: first.lockDigest,
stageReceiptDigest: '7'.repeat(64),
stageEvidenceDigest: '8'.repeat(64),
contentDigest: '9'.repeat(64),
intentDigest: '0'.repeat(64),
});
const results = await Promise.allSettled([
fixture.publisher.publish(left),
fixture.publisher.publish(right),
]);
assert.equal(
results.filter((result) => result.status === 'fulfilled').length,
1,
);
assert.equal(
results.filter(
(result) =>
result.status === 'rejected' &&
result.reason instanceof PluginPackageActivationConflictError,
).length,
1,
);
const winner = results[0].status === 'fulfilled' ? left : right;
assert.equal((await fixture.publisher.inspect(winner)).status, 'published');
});
test('does not overwrite a winner published between its two reads', async () => {
const fixture = publisher();
const first = intent();
await fixture.publisher.publish(first);
const winner = intent({
installationId: 'install-winner',
lockDigest: '1'.repeat(64),
targetGeneration: 2,
previousActiveLockDigest: first.lockDigest,
stageReceiptDigest: '2'.repeat(64),
stageEvidenceDigest: '3'.repeat(64),
contentDigest: '4'.repeat(64),
intentDigest: '5'.repeat(64),
});
const stale = intent({
installationId: 'install-stale',
lockDigest: '6'.repeat(64),
targetGeneration: 2,
previousActiveLockDigest: first.lockDigest,
stageReceiptDigest: '7'.repeat(64),
stageEvidenceDigest: '8'.repeat(64),
contentDigest: '9'.repeat(64),
intentDigest: '0'.repeat(64),
});
fixture.api.readCalls = 0;
fixture.api.beforeRead = async (readCalls) => {
if (readCalls !== 2) return;
fixture.api.beforeRead = null;
await fixture.publisher.publish(winner);
};
await assert.rejects(
fixture.publisher.publish(stale),
PluginPackageActivationConflictError,
);
assert.equal((await fixture.publisher.inspect(winner)).status, 'published');
assert.equal(fixture.api.replaceCalls, 1);
});
test('fails closed on stage evidence or ConfigMap pointer drift', async () => {
const drifted = publisher(new FakeConfigMapApi(), {
verify(value) {
return { ...exactEvidence(value), stageEvidenceDigest: 'f'.repeat(64) };
},
});
await assert.rejects(
drifted.publisher.publish(intent()),
PluginPackageActivationConflictError,
);
assert.equal(drifted.api.createCalls, 0);
const fixture = publisher();
const value = intent();
await fixture.publisher.publish(value);
const [key, stored] = fixture.api.items.entries().next().value;
fixture.api.items.set(key, {
...stored,
metadata: {
...stored.metadata,
labels: {
...stored.metadata.labels,
'qinglong.io/plugin-package-active': 'v1',
},
},
});
await assert.rejects(
fixture.publisher.findActiveResourceGeneration(
'default',
'example-monitor',
),
PluginPackageActivationConflictError,
);
fixture.api.items.set(key, {
...stored,
data: { 'active.json': `${stored.data['active.json']} ` },
});
await assert.rejects(
fixture.publisher.inspect(value),
PluginPackageActivationConflictError,
);
});
test('keeps the Kubernetes publisher behind one explicit cluster-admin subpath', () => {
assert.equal(
require('@qinglong/cluster-admin')
.PluginPackageKubernetesActivationPublisher,
undefined,
);
assert.equal(
require('@qinglong/cluster-admin/plugin-package-kubernetes-activation')
.PluginPackageKubernetesActivationPublisher,
PluginPackageKubernetesActivationPublisher,
);
});
@@ -0,0 +1,45 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_DECISION_MODE,
createClusterPluginPackageManagementService,
} = require('@qinglong/cluster-admin/plugin-package-management');
test('composes cluster Package management as short-lived separation-of-duty authority', () => {
const pool = {
async query() {
throw new Error('construction must not touch PostgreSQL');
},
async connect() {
throw new Error('construction must not open PostgreSQL');
},
};
const service = createClusterPluginPackageManagementService({
pool,
now: () => 100,
});
assert.equal(
CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_DECISION_MODE,
'separation_of_duty',
);
assert.equal(typeof service.propose, 'function');
assert.equal(typeof service.decide, 'function');
assert.equal(typeof service.inspect, 'function');
assert.equal(typeof service.inspectAuthorized, 'function');
assert.equal(typeof service.inspectInstallationAuthorized, 'function');
assert.equal(typeof service.listInstallationsAuthorized, 'function');
assert.deepEqual(Object.keys(service).sort(), [
'decide',
'inspect',
'inspectAuthorized',
'inspectInstallationAuthorized',
'listInstallationsAuthorized',
'propose',
]);
assert.equal(
require('@qinglong/cluster-admin')
.createClusterPluginPackageManagementService,
undefined,
);
});
@@ -0,0 +1,773 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const {
chmodSync,
mkdtempSync,
readFileSync,
realpathSync,
rmSync,
symlinkSync,
writeFileSync,
} = require('node:fs');
const { createServer } = require('node:https');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterPluginPackageManagementClientConfigurationError,
ClusterPluginPackageManagementClientRemoteError,
ClusterPluginPackageManagementClientRequestError,
executeClusterPluginPackageManagementClient,
} = require('@qinglong/cluster-admin/plugin-package-management-client');
const CA_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/ca-cert.pem',
);
const SERVER_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-key.pem',
);
const SERVER_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem',
);
const CLIENT_CLI = resolve(
__dirname,
'../dist/plugin-package/management/pluginPackageManagementClientCli.js',
);
const ASSERTION = 'eyJhbGciOiJFUzI1NiJ9.eyJzdWIiOiJvcGVyYXRvciJ9.c2ln';
function inspectCommand(operation = 'plugin-package.inspect') {
return {
schemaVersion: 1,
operation,
request: {
actionRef: 'package:cluster-monitor:1',
approvalRequestId: 'approval-cluster-monitor-1',
inspectionId: 'inspection-cluster-monitor-1',
},
};
}
function commands() {
const decision = {
actionRef: 'package:cluster-monitor:1',
approvalRequestId: 'approval-cluster-monitor-1',
expectedVersion: 1,
decisionId: 'decision-cluster-monitor-1',
auditEventId: 'audit-cluster-monitor-decision-1',
decision: 'approved',
reasonCode: 'reviewed',
};
const inspection = inspectCommand().request;
return [
{
schemaVersion: 1,
operation: 'plugin-package.propose',
request: {
actionRef: 'package:cluster-monitor:1',
approvalRequestId: 'approval-cluster-monitor-1',
proposalAuditEventId: 'audit-cluster-monitor-proposal-1',
approvalAuditEventId: 'audit-cluster-monitor-approval-1',
actionInput: {},
},
},
{
schemaVersion: 1,
operation: 'plugin-package.decide',
request: decision,
},
inspectCommand(),
{
schemaVersion: 1,
operation: 'plugin-package.lifecycle.propose',
request: {
actionRef: 'lifecycle:cluster-monitor:disable:1',
approvalRequestId: 'approval-lifecycle-cluster-monitor-1',
approvalAuditEventId: 'audit-lifecycle-approval-1',
},
},
{
schemaVersion: 1,
operation: 'plugin-package.lifecycle.decide',
request: {
...decision,
actionRef: 'lifecycle:cluster-monitor:disable:1',
approvalRequestId: 'approval-lifecycle-cluster-monitor-1',
},
},
{
schemaVersion: 1,
operation: 'plugin-package.lifecycle.inspect',
request: {
actionRef: 'lifecycle:cluster-monitor:disable:1',
approvalRequestId: 'approval-lifecycle-cluster-monitor-1',
inspectionId: 'inspection-lifecycle-cluster-monitor-1',
},
},
{
schemaVersion: 1,
operation: 'plugin-package.installation.inspect',
request: {
projectId: 'project-1',
packageName: 'cluster-monitor',
inspectionId: 'inspection-installation-1',
},
},
{
schemaVersion: 1,
operation: 'plugin-package.installation.list',
request: {
projectId: 'project-1',
limit: 8,
inspectionId: 'inspection-installation-list-1',
},
},
{
schemaVersion: 1,
operation: 'plugin-package.publisher-revocation.propose',
request: {
actionRef: 'publisher:example:revocation:1',
approvalRequestId: 'approval-publisher-revocation-1',
proposalAuditEventId: 'audit-publisher-revocation-proposal-1',
approvalAuditEventId: 'audit-publisher-revocation-approval-1',
publisher: 'example',
keyId: 'publisher-key-1',
authorizationMode: 'dual_control',
reasonCode: 'suspected_key_compromise',
},
},
{
schemaVersion: 1,
operation: 'plugin-package.publisher-revocation.decide',
request: decision,
},
{
schemaVersion: 1,
operation: 'plugin-package.publisher-revocation.inspect',
request: inspection,
},
{
schemaVersion: 1,
operation: 'plugin-package.publisher-trust-transition.propose',
request: {
actionRef: 'publisher:example:transition:1',
approvalRequestId: 'approval-publisher-transition-1',
proposalAuditEventId: 'audit-publisher-transition-proposal-1',
approvalAuditEventId: 'audit-publisher-transition-approval-1',
mode: 'overlap_add',
publisher: 'example',
keyId: 'publisher-key-2',
},
},
{
schemaVersion: 1,
operation: 'plugin-package.publisher-trust-transition.decide',
request: decision,
},
{
schemaVersion: 1,
operation: 'plugin-package.publisher-trust-transition.inspect',
request: inspection,
},
];
}
function approvalSummary() {
return {
id: 'approval-cluster-monitor-1',
projectId: 'project-1',
version: 2,
state: 'approved',
risk: 'high',
decisionMode: 'separation_of_duty',
requestedAtMs: 1_000,
expiresAtMs: 10_000,
decision: 'approved',
decisionReasonCode: 'reviewed',
decidedAtMs: 2_000,
dispatchId: null,
consumedAtMs: null,
actionDigest: 'action-digest-1',
previewDigest: 'preview-digest-1',
};
}
function proposalSummary(operation) {
const common = {
actionRef: 'package:cluster-monitor:1',
projectId: 'project-1',
actionDigest: 'action-digest-1',
previewDigest: 'preview-digest-1',
proposalDigest: 'proposal-digest-1',
createdAtMs: 1_000,
};
if (operation.startsWith('plugin-package.publisher-trust-transition.')) {
return {
...common,
trustAuthorityId: 'cluster',
trustGeneration: 2,
mode: 'overlap_add',
publisher: 'example',
keyId: 'publisher-key-2',
previousTrustDigest: 'trust-digest-1',
currentTrustDigest: 'trust-digest-2',
};
}
if (operation.startsWith('plugin-package.publisher-revocation.')) {
return {
...common,
trustAuthorityId: 'cluster',
trustGeneration: 2,
publisher: 'example',
keyId: 'publisher-key-1',
previousTrustDigest: 'trust-digest-1',
currentTrustDigest: 'trust-digest-2',
authorizationMode: 'dual_control',
reasonCode: 'suspected_key_compromise',
};
}
return {
...common,
packageName: '@example/cluster-monitor',
packageVersion: '1.0.0',
operation: 'install',
sourceKind: 'oci',
architecture: 'arm64',
deploymentProfile: 'cluster',
targetGeneration: 1,
};
}
function installationSummary() {
return {
installationId: 'install-cluster-monitor-1',
projectId: 'project-1',
packageName: 'cluster-monitor',
packageVersion: '1.0.0',
operation: 'install',
state: 'active',
targetGeneration: 1,
activeLockDigest: 'a'.repeat(64),
previousActiveLockDigest: null,
recoveryAction: 'none',
availability: 'active',
quarantineReason: null,
quarantineAuthorizationMode: null,
quarantineEventDigest: null,
quarantinedAtMs: null,
withdrawalStatus: null,
withdrawalReceiptDigest: null,
withdrawalCommittedAtMs: null,
failureReason: null,
failedFrom: null,
failedAtMs: null,
version: 4,
createdAtMs: 1_000,
updatedAtMs: 2_000,
recordDigest: 'b'.repeat(64),
};
}
function lifecyclePlanSummary() {
return {
actionRef: 'lifecycle:cluster-monitor:disable:1',
planDigest: '1'.repeat(64),
plannedAtMs: 1_000,
expiresAtMs: 10_000,
action: 'disable',
projectId: 'project-1',
packageName: 'cluster-monitor',
installationId: 'install-cluster-monitor-1',
lockDigest: '2'.repeat(64),
installVersion: 4,
installRecordDigest: '3'.repeat(64),
expected: {
version: 0,
disposition: 'active',
eventDigest: null,
},
generationDigest: '4'.repeat(64),
materializedRevisionDigest: '5'.repeat(64),
currentToolSnapshotDigest: '6'.repeat(64),
taskIds: ['collect'],
resourceCounts: { tasks: 1, tools: 0, workflows: 0, prompts: 0 },
referenceGraphDigest: '7'.repeat(64),
blockingReferences: [],
impactDigest: '8'.repeat(64),
};
}
function successfulResult(operation) {
if (operation === 'plugin-package.installation.inspect') {
return {
schemaVersion: 1,
operation,
installation: installationSummary(),
};
}
if (operation === 'plugin-package.installation.list') {
return {
schemaVersion: 1,
operation,
installations: [installationSummary()],
truncated: false,
next: null,
};
}
if (operation === 'plugin-package.lifecycle.propose') {
return {
schemaVersion: 1,
operation,
approvalStatus: 'created',
plan: lifecyclePlanSummary(),
approval: approvalSummary(),
};
}
if (operation === 'plugin-package.lifecycle.inspect') {
return {
schemaVersion: 1,
operation,
plan: lifecyclePlanSummary(),
approval: approvalSummary(),
stale: false,
};
}
if (operation.endsWith('.propose')) {
return {
schemaVersion: 1,
operation,
proposalStatus: 'created',
approvalStatus: 'created',
proposal: proposalSummary(operation),
approval: approvalSummary(),
};
}
if (operation.endsWith('.decide')) {
return {
schemaVersion: 1,
operation,
status: 'decided',
approval: approvalSummary(),
};
}
return {
schemaVersion: 1,
operation,
proposal: null,
approval: null,
};
}
function privateWrite(path, value) {
writeFileSync(
path,
typeof value === 'string' ? value : JSON.stringify(value),
{ mode: 0o600 },
);
chmodSync(path, 0o600);
}
function createClientFiles(port, command = inspectCommand()) {
const directory = realpathSync(
mkdtempSync(join(tmpdir(), 'ql3-management-client-')),
);
const configFile = join(directory, 'client.json');
const commandFile = join(directory, 'command.json');
const assertionFile = join(directory, 'assertion.jwt');
privateWrite(configFile, {
schemaVersion: 1,
endpoint: `https://localhost:${port}/api/v3/plugin-packages/management`,
servername: 'localhost',
caFile: CA_CERT,
requestTimeoutMs: 1_000,
});
privateWrite(commandFile, command);
privateWrite(assertionFile, ASSERTION);
return {
directory,
paths: { configFile, commandFile, assertionFile },
};
}
async function startServer(handler) {
const server = createServer(
{
key: readFileSync(SERVER_KEY),
cert: readFileSync(SERVER_CERT),
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
},
handler,
);
await new Promise((resolvePromise, reject) => {
server.once('error', reject);
server.listen(0, '127.0.0.1', resolvePromise);
});
return {
server,
port: server.address().port,
close: () =>
new Promise((resolvePromise, reject) => {
server.close((error) => (error ? reject(error) : resolvePromise()));
}),
};
}
function sendJson(response, statusCode, value, headers = {}) {
const body = Buffer.from(JSON.stringify(value));
response.writeHead(statusCode, {
'content-type': 'application/json; charset=utf-8',
'content-length': String(body.length),
...headers,
});
response.end(body);
}
test('sends one TLS 1.3 management command and validates the low-sensitive result', async () => {
const received = [];
const fixture = await startServer((request, response) => {
const chunks = [];
request.on('data', (chunk) => chunks.push(chunk));
request.once('end', () => {
received.push({
method: request.method,
path: request.url,
authorization: request.headers.authorization,
acceptEncoding: request.headers['accept-encoding'],
protocol: request.socket.getProtocol(),
command: JSON.parse(Buffer.concat(chunks).toString('utf8')),
});
sendJson(response, 200, {
schemaVersion: 1,
requestId: 'request-client-1',
result: {
schemaVersion: 1,
operation: 'plugin-package.inspect',
proposal: null,
approval: null,
},
});
});
});
const files = createClientFiles(fixture.port);
try {
const result = await executeClusterPluginPackageManagementClient(
files.paths,
);
assert.equal(result.requestId, 'request-client-1');
assert.equal(result.result.operation, 'plugin-package.inspect');
assert.deepEqual(received, [
{
method: 'POST',
path: '/api/v3/plugin-packages/management',
authorization: `Bearer ${ASSERTION}`,
acceptEncoding: 'identity',
protocol: 'TLSv1.3',
command: inspectCommand(),
},
]);
} finally {
await fixture.close();
rmSync(files.directory, { recursive: true, force: true });
}
});
test('permits and validates exactly the fourteen public management operations', async () => {
const received = [];
const fixture = await startServer((request, response) => {
const chunks = [];
request.on('data', (chunk) => chunks.push(chunk));
request.once('end', () => {
const operation = JSON.parse(
Buffer.concat(chunks).toString('utf8'),
).operation;
received.push(operation);
sendJson(response, 200, {
schemaVersion: 1,
requestId: `request-operation-${received.length}`,
result: successfulResult(operation),
});
});
});
const files = createClientFiles(fixture.port);
try {
for (const command of commands()) {
privateWrite(files.paths.commandFile, command);
const result = await executeClusterPluginPackageManagementClient(
files.paths,
);
assert.equal(result.result.operation, command.operation);
}
assert.deepEqual(
received,
commands().map((command) => command.operation),
);
privateWrite(files.paths.commandFile, {
schemaVersion: 1,
operation: 'plugin-package.execute',
request: {},
});
await assert.rejects(
executeClusterPluginPackageManagementClient(files.paths),
(error) => {
assert.equal(
error.code,
'CLUSTER_PLUGIN_PACKAGE_TRANSPORT_REQUEST_INVALID',
);
return true;
},
);
assert.equal(received.length, 14);
} finally {
await fixture.close();
rmSync(files.directory, { recursive: true, force: true });
}
});
test('rejects installation inventory responses outside the requested project and keyset', async () => {
const responses = [
{
installations: [
{ ...installationSummary(), projectId: 'another-project' },
],
truncated: false,
next: null,
},
{
installations: [installationSummary(), installationSummary()],
truncated: false,
next: null,
},
{
installations: [installationSummary()],
truncated: true,
next: { packageName: 'another-package' },
},
{
installations: [
{
...installationSummary(),
availability: 'quarantined',
quarantineReason: 'confirmed_key_compromise',
},
],
truncated: false,
next: null,
},
];
const fixture = await startServer((_request, response) => {
sendJson(response, 200, {
schemaVersion: 1,
requestId: `request-invalid-inventory-${responses.length}`,
result: {
schemaVersion: 1,
operation: 'plugin-package.installation.list',
...responses.shift(),
},
});
});
const command = {
schemaVersion: 1,
operation: 'plugin-package.installation.list',
request: {
projectId: 'project-1',
limit: 2,
inspectionId: 'inspection-invalid-inventory-1',
},
};
const files = createClientFiles(fixture.port, command);
try {
for (let index = 0; index < 4; index += 1) {
await assert.rejects(
executeClusterPluginPackageManagementClient(files.paths),
ClusterPluginPackageManagementClientRequestError,
);
}
} finally {
await fixture.close();
rmSync(files.directory, { recursive: true, force: true });
}
});
test('rejects non-private, symlinked, and non-exact input files before transport', async () => {
const files = createClientFiles(443);
try {
chmodSync(files.paths.assertionFile, 0o644);
await assert.rejects(
executeClusterPluginPackageManagementClient(files.paths),
ClusterPluginPackageManagementClientConfigurationError,
);
chmodSync(files.paths.assertionFile, 0o600);
const commandLink = join(files.directory, 'command-link.json');
symlinkSync(files.paths.commandFile, commandLink);
await assert.rejects(
executeClusterPluginPackageManagementClient({
...files.paths,
commandFile: commandLink,
}),
ClusterPluginPackageManagementClientConfigurationError,
);
privateWrite(files.paths.configFile, {
schemaVersion: 1,
endpoint: 'https://localhost/api/v3/plugin-packages/management',
servername: 'localhost',
caFile: CA_CERT,
requestTimeoutMs: 1_000,
proxy: 'https://proxy.invalid',
});
await assert.rejects(
executeClusterPluginPackageManagementClient(files.paths),
ClusterPluginPackageManagementClientConfigurationError,
);
} finally {
rmSync(files.directory, { recursive: true, force: true });
}
});
test('does not follow redirects and rejects malformed, oversized, and timed-out responses', async () => {
let behavior = 'remote-error';
let hits = 0;
const fixture = await startServer((_request, response) => {
hits += 1;
if (behavior === 'remote-error') {
sendJson(
response,
409,
{
schemaVersion: 1,
requestId: 'request-rejected-1',
error: { code: 'test_rejection' },
},
{ 'retry-after': '7' },
);
return;
}
if (behavior === 'redirect') {
sendJson(
response,
302,
{
schemaVersion: 1,
requestId: 'request-redirect-1',
error: { code: 'redirected' },
},
{ location: 'https://example.invalid/' },
);
return;
}
if (behavior === 'wrong-content-type') {
response.writeHead(200, { 'content-type': 'text/plain' });
response.end('no');
return;
}
if (behavior === 'oversized') {
response.writeHead(200, {
'content-type': 'application/json; charset=utf-8',
});
response.end(Buffer.alloc(128 * 1024 + 1, 0x61));
return;
}
if (behavior === 'truncated') {
response.writeHead(200, {
'content-type': 'application/json; charset=utf-8',
'content-length': '100',
});
response.write('{"');
response.destroy();
}
});
const files = createClientFiles(fixture.port);
try {
await assert.rejects(
executeClusterPluginPackageManagementClient(files.paths),
(error) => {
assert.equal(
error instanceof ClusterPluginPackageManagementClientRemoteError,
true,
);
assert.equal(error.statusCode, 409);
assert.equal(error.responseCode, 'test_rejection');
assert.equal(error.requestId, 'request-rejected-1');
assert.equal(error.retryAfterSeconds, 7);
return true;
},
);
behavior = 'redirect';
await assert.rejects(
executeClusterPluginPackageManagementClient(files.paths),
ClusterPluginPackageManagementClientRequestError,
);
assert.equal(hits, 2);
behavior = 'wrong-content-type';
await assert.rejects(
executeClusterPluginPackageManagementClient(files.paths),
ClusterPluginPackageManagementClientRequestError,
);
behavior = 'oversized';
await assert.rejects(
executeClusterPluginPackageManagementClient(files.paths),
ClusterPluginPackageManagementClientRequestError,
);
behavior = 'truncated';
await assert.rejects(
executeClusterPluginPackageManagementClient(files.paths),
ClusterPluginPackageManagementClientRequestError,
);
behavior = 'timeout';
await assert.rejects(
executeClusterPluginPackageManagementClient(files.paths),
ClusterPluginPackageManagementClientRequestError,
);
assert.equal(hits, 6);
} finally {
await fixture.close();
rmSync(files.directory, { recursive: true, force: true });
}
});
test('CLI accepts path-only arguments and never reports secret content or paths', () => {
const files = createClientFiles(443);
try {
privateWrite(files.paths.assertionFile, 'top-secret-assertion');
const result = spawnSync(
process.execPath,
[
CLIENT_CLI,
`--config=${files.paths.configFile}`,
`--command=${files.paths.commandFile}`,
`--assertion=${files.paths.assertionFile}`,
],
{ encoding: 'utf8' },
);
assert.equal(result.status, 1);
assert.equal(result.stdout, '');
const fact = JSON.parse(result.stderr);
assert.deepEqual(fact, {
schemaVersion: 1,
component: 'qinglong3-plugin-package-management-client',
event: 'command_failed',
code: 'QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_CONFIG_INVALID',
});
assert.equal(result.stderr.includes('top-secret-assertion'), false);
assert.equal(result.stderr.includes(files.directory), false);
const help = spawnSync(process.execPath, [CLIENT_CLI, '--help'], {
encoding: 'utf8',
});
assert.equal(help.status, 0);
assert.match(help.stdout, /^Usage: ql3-plugin-package-client /);
assert.equal(help.stderr, '');
} finally {
rmSync(files.directory, { recursive: true, force: true });
}
});
@@ -0,0 +1,409 @@
const assert = require('node:assert/strict');
const { readFileSync } = require('node:fs');
const { request: httpsRequest } = require('node:https');
const { resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterPluginPackageIdentityKeysetUnavailableError,
} = require('@qinglong/cluster-admin/plugin-package-identity-keyset');
const {
ClusterPluginPackageManagementHttpConfigurationError,
startClusterPluginPackageManagementHttp,
} = require('@qinglong/cluster-admin/plugin-package-management-http');
const {
PluginPackageManagementQuotaExceededError,
} = require('@qinglong/runtime-core/plugin-package-management');
const SERVER_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-key.pem',
);
const SERVER_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem',
);
function principal() {
return {
subject: { type: 'user', id: 'cluster-reviewer' },
authenticationId: 'ql3oidc.authentication-id',
authenticatedAtMs: 900,
expiresAtMs: 10_000,
assurance: 'multi_factor',
};
}
function command() {
return {
schemaVersion: 1,
operation: 'plugin-package.inspect',
request: {
actionRef: 'package:cluster-monitor:1',
approvalRequestId: 'approval-cluster-monitor-1',
inspectionId: 'inspection-cluster-monitor-1',
},
};
}
function identityProvider(overrides = {}) {
const calls = {
reload: 0,
bind: [],
authenticate: 0,
};
return {
calls,
provider: {
async reload() {
calls.reload += 1;
return {
schemaVersion: 1,
generation: 1,
digest: 'digest',
issuer: 'https://identity.example.test/',
audience: 'qinglong3-package-management',
activeKeyIds: ['key-1'],
revokedKeyIds: [],
};
},
bind(assertion) {
calls.bind.push(assertion);
return {
async authenticate() {
calls.authenticate += 1;
if (overrides.authenticate) {
return overrides.authenticate();
}
return principal();
},
};
},
},
};
}
function transportFixture(overrides = {}) {
const calls = [];
return {
calls,
transport: {
async execute(value, authentication) {
calls.push({
command: value,
principal: await authentication.authenticate(),
});
if (overrides.execute) return overrides.execute(value);
return {
schemaVersion: 1,
operation: 'plugin-package.inspect',
proposal: null,
approval: null,
};
},
},
};
}
async function startFixture(overrides = {}) {
const identities = overrides.identities ?? identityProvider();
const transport = overrides.transport ?? transportFixture();
const privateKey = Buffer.from(readFileSync(SERVER_KEY));
const application = await startClusterPluginPackageManagementHttp({
host: '127.0.0.1',
port: 0,
tls: {
privateKey,
certificate: Buffer.from(readFileSync(SERVER_CERT)),
},
identities: identities.provider,
transport: transport.transport,
limits: {
requestTimeoutMs: 2_000,
drainTimeoutMs: 500,
...(overrides.limits ?? {}),
},
now: overrides.now ?? (() => 1_000),
createRequestId: overrides.createRequestId ?? (() => 'request-1'),
onError: overrides.onError,
});
assert.equal(
privateKey.every((value) => value === 0),
true,
);
return { application, identities, transport };
}
async function request(application, options = {}) {
const body =
options.body === undefined
? Buffer.from(JSON.stringify(command()))
: Buffer.isBuffer(options.body)
? options.body
: Buffer.from(options.body);
return new Promise((resolvePromise, reject) => {
const outgoing = httpsRequest(
{
host: '127.0.0.1',
port: application.address.port,
path: options.path ?? '/api/v3/plugin-packages/management',
method: options.method ?? 'POST',
rejectUnauthorized: false,
agent: false,
headers: {
...(options.authorization === false
? {}
: { authorization: 'Bearer assertion-value' }),
...(options.contentType === false
? {}
: { 'content-type': 'application/json' }),
...(options.omitLength
? {}
: { 'content-length': String(body.length) }),
...(options.headers ?? {}),
},
},
(incoming) => {
const chunks = [];
incoming.on('data', (chunk) => chunks.push(chunk));
incoming.once('end', () => {
const bytes = Buffer.concat(chunks);
resolvePromise({
statusCode: incoming.statusCode,
headers: incoming.headers,
body:
bytes.length === 0 ? null : JSON.parse(bytes.toString('utf8')),
});
});
},
);
outgoing.once('error', reject);
if (body.length > 0) outgoing.write(body);
outgoing.end();
});
}
test('rejects request and connection ceilings above hard bounds', async () => {
for (const limits of [
{ maxBodyBytes: 256 * 1024 + 1 },
{ maxConnections: 513 },
]) {
const privateKey = Buffer.from(readFileSync(SERVER_KEY));
try {
await assert.rejects(
startClusterPluginPackageManagementHttp({
host: '127.0.0.1',
port: 0,
tls: {
privateKey,
certificate: Buffer.from(readFileSync(SERVER_CERT)),
},
identities: identityProvider().provider,
transport: transportFixture().transport,
limits,
}),
ClusterPluginPackageManagementHttpConfigurationError,
);
} finally {
privateKey.fill(0);
}
}
});
test('serves health and authenticates before one exact management command', async () => {
const fixture = await startFixture();
try {
const live = await request(fixture.application, {
method: 'GET',
path: '/livez',
body: Buffer.alloc(0),
authorization: false,
contentType: false,
});
assert.equal(live.statusCode, 200);
assert.equal(live.headers['x-request-id'], 'request-1');
assert.equal(live.headers['cache-control'], 'no-store');
assert.deepEqual(live.body, { schemaVersion: 1, status: 'live' });
const response = await request(fixture.application);
assert.equal(response.statusCode, 200);
assert.equal(response.body.result.operation, 'plugin-package.inspect');
assert.deepEqual(fixture.identities.calls.bind, ['assertion-value']);
assert.equal(fixture.identities.calls.authenticate, 1);
assert.equal(fixture.transport.calls.length, 1);
assert.deepEqual(fixture.transport.calls[0].command, command());
assert.deepEqual(fixture.transport.calls[0].principal.subject, {
type: 'user',
id: 'cluster-reviewer',
});
} finally {
await fixture.application.close();
}
});
test('maps durable quota exhaustion to 429 with a bounded Retry-After', async () => {
const fixture = await startFixture({
transport: transportFixture({
async execute() {
throw new PluginPackageManagementQuotaExceededError(1_250);
},
}),
});
try {
const response = await request(fixture.application);
assert.equal(response.statusCode, 429);
assert.equal(response.headers['retry-after'], '2');
assert.equal(response.body.error.code, 'quota_exceeded');
} finally {
await fixture.application.close();
}
});
test('rejects routes, missing authentication and media type before transport', async () => {
const fixture = await startFixture();
try {
assert.equal(
(
await request(fixture.application, {
path: '/api/v3/plugin-packages/unknown',
})
).statusCode,
404,
);
assert.equal(
(await request(fixture.application, { authorization: false })).statusCode,
401,
);
assert.equal(
(await request(fixture.application, { contentType: false })).statusCode,
415,
);
assert.equal(
(
await request(fixture.application, {
headers: { expect: '100-continue' },
})
).statusCode,
417,
);
assert.equal(fixture.transport.calls.length, 0);
assert.equal(fixture.identities.calls.authenticate, 1);
} finally {
await fixture.application.close();
}
});
test('fails closed when the live keyset is unavailable', async () => {
const identities = identityProvider({
authenticate() {
throw new ClusterPluginPackageIdentityKeysetUnavailableError();
},
});
const fixture = await startFixture({ identities });
try {
const response = await request(fixture.application);
assert.equal(response.statusCode, 503);
assert.equal(response.body.error.code, 'unavailable');
assert.equal(fixture.transport.calls.length, 0);
} finally {
await fixture.application.close();
}
});
test('enforces peer limits before authentication with bounded retry evidence', async () => {
const fixture = await startFixture({
limits: {
rateWindowMs: 60_000,
peerRequestLimit: 1,
globalRequestLimit: 10,
maxRateLimitPeers: 2,
},
});
try {
assert.equal((await request(fixture.application)).statusCode, 200);
const limited = await request(fixture.application);
assert.equal(limited.statusCode, 429);
assert.equal(limited.body.error.code, 'rate_limited');
assert.equal(limited.headers['retry-after'], '60');
assert.equal(fixture.identities.calls.authenticate, 1);
assert.equal(fixture.transport.calls.length, 1);
} finally {
await fixture.application.close();
}
});
test('bounds request bodies and concurrent management work', async () => {
let release;
const gate = new Promise((resolvePromise) => {
release = resolvePromise;
});
const fixture = await startFixture({
limits: {
maxBodyBytes: 1_024,
maxConcurrentRequests: 1,
},
transport: transportFixture({
async execute() {
await gate;
return {
schemaVersion: 1,
operation: 'plugin-package.inspect',
proposal: null,
approval: null,
};
},
}),
});
try {
assert.equal(
(
await request(fixture.application, {
body: Buffer.alloc(1_025, 0x20),
})
).statusCode,
413,
);
const first = request(fixture.application);
await new Promise((resolvePromise) => setImmediate(resolvePromise));
const overloaded = await request(fixture.application);
assert.equal(overloaded.statusCode, 503);
assert.equal(overloaded.body.error.code, 'overloaded');
release();
assert.equal((await first).statusCode, 200);
} finally {
release();
await fixture.application.close();
}
});
test('withdraws readiness without killing liveness and closes idempotently', async () => {
const fixture = await startFixture();
fixture.application.withdraw(new Error('database unavailable'));
assert.equal(fixture.application.availabilityStatus(), 'unavailable');
assert.equal(
(
await request(fixture.application, {
method: 'GET',
path: '/readyz',
body: Buffer.alloc(0),
authorization: false,
contentType: false,
})
).statusCode,
503,
);
assert.equal(
(
await request(fixture.application, {
method: 'GET',
path: '/livez',
body: Buffer.alloc(0),
authorization: false,
contentType: false,
})
).statusCode,
200,
);
assert.equal((await request(fixture.application)).statusCode, 503);
await Promise.all([fixture.application.close(), fixture.application.close()]);
assert.equal(fixture.application.availabilityStatus(), 'stopped');
});
@@ -0,0 +1,781 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const { once } = require('node:events');
const {
chmodSync,
mkdtempSync,
readFileSync,
realpathSync,
rmSync,
writeFileSync,
} = require('node:fs');
const { createServer } = require('node:https');
const { connect: connectTcp } = require('node:net');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterPluginPackageManagementKubernetesClientConfigurationError,
ClusterPluginPackageManagementKubernetesClientTunnelError,
executeClusterPluginPackageManagementKubernetesClient,
openClusterPluginPackageManagementPortForward,
} = require(
'@qinglong/cluster-admin/plugin-package-management-kubernetes-client'
);
const {
ClusterPluginPackageManagementClientRemoteError,
} = require('@qinglong/cluster-admin/plugin-package-management-client');
const SERVICE_HOST =
'ql3-plugin-package-management.qinglong3-system.svc';
const SERVICE_CERT = resolve(
__dirname,
'fixtures/management-service-cert.pem',
);
const SERVICE_KEY = resolve(
__dirname,
'fixtures/management-service-key.pem',
);
const KUBERNETES_CLIENT_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/client-cert.pem',
);
const KUBERNETES_CLIENT_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/client-key.pem',
);
const CLIENT_CLI = resolve(
__dirname,
'../dist/plugin-package/management/pluginPackageManagementKubernetesClientCli.js',
);
const ASSERTION = 'eyJhbGciOiJFUzI1NiJ9.eyJzdWIiOiJvcGVyYXRvciJ9.c2ln';
const KUBERNETES_TOKEN = 'kube-token-secret';
function privateWrite(path, value) {
writeFileSync(
path,
typeof value === 'string' ? value : JSON.stringify(value),
{ mode: 0o600 },
);
chmodSync(path, 0o600);
}
function inspectCommand() {
return {
schemaVersion: 1,
operation: 'plugin-package.inspect',
request: {
actionRef: 'package:cluster-monitor:1',
approvalRequestId: 'approval-cluster-monitor-1',
inspectionId: 'inspection-cluster-monitor-1',
},
};
}
function kubeconfig(overrides = {}) {
const certificateAuthorityData = readFileSync(SERVICE_CERT).toString(
'base64',
);
const cluster = {
server: 'https://kubernetes.example.test:6443',
'certificate-authority-data': certificateAuthorityData,
...(overrides.cluster ?? {}),
};
const user = {
token: KUBERNETES_TOKEN,
...(overrides.user ?? {}),
};
return {
apiVersion: 'v1',
kind: 'Config',
clusters: [{ name: 'production', cluster }],
users: [{ name: 'ql3-operator', user }],
contexts: [
{
name: 'production',
context: {
cluster: 'production',
user: 'ql3-operator',
namespace: 'qinglong3-system',
},
},
],
'current-context': overrides.currentContext ?? 'production',
};
}
function createClientFiles(kubeconfigValue = kubeconfig()) {
const directory = realpathSync(
mkdtempSync(join(tmpdir(), 'ql3-kubernetes-client-')),
);
const configFile = join(directory, 'client.json');
const commandFile = join(directory, 'command.json');
const assertionFile = join(directory, 'assertion.jwt');
const kubeconfigFile = join(directory, 'kubeconfig.json');
const kubernetesFile = join(directory, 'kubernetes.json');
privateWrite(configFile, {
schemaVersion: 1,
endpoint: `https://${SERVICE_HOST}:8443/api/v3/plugin-packages/management`,
servername: SERVICE_HOST,
caFile: SERVICE_CERT,
requestTimeoutMs: 2_000,
});
privateWrite(commandFile, inspectCommand());
privateWrite(assertionFile, ASSERTION);
privateWrite(kubeconfigFile, kubeconfigValue);
privateWrite(kubernetesFile, {
schemaVersion: 1,
kubeconfigFile,
context: 'production',
namespace: 'qinglong3-system',
apiTimeoutMs: 2_000,
});
return {
directory,
paths: {
configFile,
commandFile,
assertionFile,
kubernetesFile,
},
kubeconfigFile,
};
}
function readyPod(name, overrides = {}) {
return {
metadata: {
name,
namespace: 'qinglong3-system',
uid: `uid-${name}`,
labels: {
'app.kubernetes.io/name': 'ql3-plugin-package-management',
'app.kubernetes.io/component': 'plugin-package-management',
},
...(overrides.metadata ?? {}),
},
spec: {
serviceAccountName: 'ql3-plugin-package-management',
automountServiceAccountToken: false,
containers: [{ name: 'management' }],
...(overrides.spec ?? {}),
},
status: {
phase: 'Running',
conditions: [{ type: 'Ready', status: 'True' }],
containerStatuses: [{ name: 'management', ready: true }],
...(overrides.status ?? {}),
},
};
}
async function startServer(handler) {
const server = createServer(
{
key: readFileSync(SERVICE_KEY),
cert: readFileSync(SERVICE_CERT),
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
},
handler,
);
await new Promise((resolvePromise, reject) => {
server.once('error', reject);
server.listen(0, '127.0.0.1', resolvePromise);
});
return {
port: server.address().port,
close: () =>
new Promise((resolvePromise, reject) => {
server.close((error) => (error ? reject(error) : resolvePromise()));
}),
};
}
function sendJson(response, value) {
const body = Buffer.from(JSON.stringify(value));
response.writeHead(200, {
'content-type': 'application/json; charset=utf-8',
'content-length': String(body.length),
});
response.end(body);
}
test('uses one ready Pod tunnel and preserves end-to-end TLS 1.3 hostname verification', async () => {
const requests = [];
const server = await startServer((request, response) => {
const chunks = [];
request.on('data', (chunk) => chunks.push(chunk));
request.once('end', () => {
requests.push({
authorization: request.headers.authorization,
protocol: request.socket.getProtocol(),
servername: request.socket.servername,
body: JSON.parse(Buffer.concat(chunks).toString('utf8')),
});
sendJson(response, {
schemaVersion: 1,
requestId: 'request-kubernetes-client-1',
result: {
schemaVersion: 1,
operation: 'plugin-package.inspect',
proposal: null,
approval: null,
},
});
});
});
const files = createClientFiles();
const calls = { lists: [], tunnels: [], closes: 0 };
try {
const result =
await executeClusterPluginPackageManagementKubernetesClient(
files.paths,
{
createRuntime() {
return {
pods: {
async listNamespacedPod(request) {
calls.lists.push(request);
return {
items: [
readyPod(
'ql3-plugin-package-management-bbbbb-22222',
),
readyPod(
'ql3-plugin-package-management-aaaaa-11111',
),
],
};
},
},
async openPortForward(request) {
calls.tunnels.push(request);
const stream = connectTcp({
host: '127.0.0.1',
port: server.port,
});
return {
stream,
close() {
calls.closes += 1;
stream.destroy();
},
};
},
};
},
},
);
assert.equal(result.requestId, 'request-kubernetes-client-1');
assert.deepEqual(calls.lists, [
{
namespace: 'qinglong3-system',
labelSelector:
'app.kubernetes.io/name=ql3-plugin-package-management,' +
'app.kubernetes.io/component=plugin-package-management',
limit: 3,
timeoutSeconds: 2,
watch: false,
},
]);
assert.deepEqual(calls.tunnels, [
{
namespace: 'qinglong3-system',
podName: 'ql3-plugin-package-management-aaaaa-11111',
port: 8443,
},
]);
assert.equal(calls.closes, 1);
assert.deepEqual(requests, [
{
authorization: `Bearer ${ASSERTION}`,
protocol: 'TLSv1.3',
servername: SERVICE_HOST,
body: inspectCommand(),
},
]);
} finally {
await server.close();
rmSync(files.directory, { recursive: true, force: true });
}
});
test('rejects ambient, executable, proxied, insecure, and file-backed kubeconfig authority', async () => {
const cases = [
kubeconfig({ cluster: { 'insecure-skip-tls-verify': true } }),
kubeconfig({ cluster: { 'proxy-url': 'https://proxy.invalid' } }),
kubeconfig({
cluster: {
'certificate-authority': '/private/ca.pem',
'certificate-authority-data': undefined,
},
}),
kubeconfig({
user: {
token: undefined,
exec: {
apiVersion: 'client.authentication.k8s.io/v1',
command: '/usr/bin/cloud-login',
},
},
}),
kubeconfig({
user: {
token: undefined,
'auth-provider': { name: 'oidc' },
},
}),
kubeconfig({
user: { token: undefined, username: 'admin', password: 'secret' },
}),
kubeconfig({ user: { as: 'root' } }),
kubeconfig({
user: {
token: undefined,
'client-certificate-data': readFileSync(
KUBERNETES_CLIENT_CERT,
).toString('base64'),
'client-key-data': readFileSync(SERVICE_KEY).toString('base64'),
},
}),
];
for (const candidate of cases) {
const files = createClientFiles(candidate);
try {
await assert.rejects(
executeClusterPluginPackageManagementKubernetesClient(
files.paths,
{
createRuntime() {
assert.fail('invalid kubeconfig reached runtime creation');
},
},
),
ClusterPluginPackageManagementKubernetesClientConfigurationError,
);
} finally {
rmSync(files.directory, { recursive: true, force: true });
}
}
});
test('accepts one matching embedded Kubernetes client certificate and key', async () => {
const files = createClientFiles(
kubeconfig({
user: {
token: undefined,
'client-certificate-data': readFileSync(
KUBERNETES_CLIENT_CERT,
).toString('base64'),
'client-key-data': readFileSync(KUBERNETES_CLIENT_KEY).toString(
'base64',
),
},
}),
);
let runtimeCreated = false;
try {
await assert.rejects(
executeClusterPluginPackageManagementKubernetesClient(files.paths, {
createRuntime() {
runtimeCreated = true;
return {
pods: {
async listNamespacedPod() {
return { items: [] };
},
},
async openPortForward() {
assert.fail('empty Pod list opened a tunnel');
},
};
},
}),
ClusterPluginPackageManagementKubernetesClientTunnelError,
);
assert.equal(runtimeCreated, true);
} finally {
rmSync(files.directory, { recursive: true, force: true });
}
});
test('maps one PortForward WebSocket to one bounded raw duplex and closes once', async () => {
const calls = [];
const clientBytes = [];
const listeners = { close: [], error: [] };
let closeCalls = 0;
let errorStream;
let serverOutput;
const connection =
await openClusterPluginPackageManagementPortForward(
{
async portForward(
namespace,
podName,
ports,
output,
error,
input,
retryCount,
) {
calls.push({
namespace,
podName,
ports,
retryCount,
});
serverOutput = output;
errorStream = error;
input.on('data', (chunk) =>
clientBytes.push(Buffer.from(chunk)),
);
return {
addEventListener(type, listener) {
listeners[type].push(listener);
},
close() {
closeCalls += 1;
},
};
},
},
{
namespace: 'qinglong3-system',
podName: 'ql3-plugin-package-management-aaaaa-11111',
port: 8443,
},
);
assert.deepEqual(calls, [
{
namespace: 'qinglong3-system',
podName: 'ql3-plugin-package-management-aaaaa-11111',
ports: [8443],
retryCount: 0,
},
]);
connection.stream.write(Buffer.from('client-request'));
await new Promise((resolvePromise) => setImmediate(resolvePromise));
assert.equal(Buffer.concat(clientBytes).toString(), 'client-request');
const incoming = once(connection.stream, 'data');
serverOutput.write(Buffer.from('server-response'));
assert.equal((await incoming)[0].toString(), 'server-response');
errorStream.write(Buffer.alloc(0));
assert.equal(connection.stream.destroyed, false);
const failed = once(connection.stream, 'error');
errorStream.write(Buffer.from('redacted Kubernetes diagnostic'));
assert.equal(
(await failed)[0] instanceof
ClusterPluginPackageManagementKubernetesClientTunnelError,
true,
);
connection.close();
connection.close();
assert.equal(closeCalls, 1);
});
test('binds the upstream Kubernetes PortForward path and channel protocol exactly', async () => {
const kubernetes = await import('@kubernetes/client-node');
const config = new kubernetes.KubeConfig();
config.loadFromString(JSON.stringify(kubeconfig()));
config.setCurrentContext('production');
const sent = [];
const listeners = { close: [], error: [] };
let connectedPath;
let binaryHandler;
let closeCalls = 0;
const webSocket = {
protocol: 'v5.channel.k8s.io',
send(chunk) {
sent.push(Buffer.from(chunk));
},
close() {
closeCalls += 1;
},
addEventListener(type, listener) {
listeners[type].push(listener);
},
};
const forward = new kubernetes.PortForward(config, true, {
async connect(path, _textHandler, handler) {
connectedPath = path;
binaryHandler = handler;
return webSocket;
},
});
const connection =
await openClusterPluginPackageManagementPortForward(
forward,
{
namespace: 'qinglong3-system',
podName: 'ql3-plugin-package-management-aaaaa-11111',
port: 8443,
},
);
assert.equal(
connectedPath,
'/api/v1/namespaces/qinglong3-system/pods/' +
'ql3-plugin-package-management-aaaaa-11111/' +
'portforward?ports=8443',
);
connection.stream.write(Buffer.from('client-bytes'));
await new Promise((resolvePromise) => setImmediate(resolvePromise));
assert.deepEqual(
sent[0],
Buffer.concat([Buffer.from([0]), Buffer.from('client-bytes')]),
);
const portHeader = Buffer.alloc(2);
portHeader.writeUInt16BE(8443);
const incoming = once(connection.stream, 'data');
assert.equal(
binaryHandler(
0,
Buffer.concat([portHeader, Buffer.from('server-bytes')]),
),
true,
);
assert.equal((await incoming)[0].toString(), 'server-bytes');
connection.close();
assert.equal(closeCalls, 1);
});
test('rejects overflow, continue, unready, and token-mounted Pod targets without a tunnel', async () => {
const files = createClientFiles();
const lists = [
{
metadata: { continue: 'next-page' },
items: [readyPod('ql3-plugin-package-management-aaaaa-11111')],
},
{
items: [
readyPod('ql3-plugin-package-management-aaaaa-11111'),
readyPod('ql3-plugin-package-management-bbbbb-22222'),
readyPod('ql3-plugin-package-management-ccccc-33333'),
],
},
{
items: [
readyPod('ql3-plugin-package-management-aaaaa-11111', {
status: {
phase: 'Pending',
conditions: [],
containerStatuses: [],
},
}),
],
},
{
items: [
readyPod('ql3-plugin-package-management-aaaaa-11111', {
spec: { automountServiceAccountToken: true },
}),
],
},
];
try {
for (const list of lists) {
let tunnelCalls = 0;
await assert.rejects(
executeClusterPluginPackageManagementKubernetesClient(
files.paths,
{
createRuntime() {
return {
pods: { async listNamespacedPod() { return list; } },
async openPortForward() {
tunnelCalls += 1;
throw new Error('must not open');
},
};
},
},
),
ClusterPluginPackageManagementKubernetesClientTunnelError,
);
assert.equal(tunnelCalls, 0);
}
} finally {
rmSync(files.directory, { recursive: true, force: true });
}
});
test('does not retry or switch Pods when the tunneled response is lost', async () => {
const server = await startServer((request) => {
request.resume();
request.once('end', () => request.socket.destroy());
});
const files = createClientFiles();
let listCalls = 0;
let tunnelCalls = 0;
let closeCalls = 0;
try {
await assert.rejects(
executeClusterPluginPackageManagementKubernetesClient(files.paths, {
createRuntime() {
return {
pods: {
async listNamespacedPod() {
listCalls += 1;
return {
items: [
readyPod(
'ql3-plugin-package-management-aaaaa-11111',
),
readyPod(
'ql3-plugin-package-management-bbbbb-22222',
),
],
};
},
},
async openPortForward() {
tunnelCalls += 1;
const stream = connectTcp({
host: '127.0.0.1',
port: server.port,
});
return {
stream,
close() {
closeCalls += 1;
stream.destroy();
},
};
},
};
},
}),
);
assert.equal(listCalls, 1);
assert.equal(tunnelCalls, 1);
assert.equal(closeCalls, 1);
} finally {
await server.close();
rmSync(files.directory, { recursive: true, force: true });
}
});
test('preserves bounded management rejection facts across the tunnel', async () => {
const server = await startServer((request, response) => {
request.resume();
request.once('end', () => {
const body = Buffer.from(
JSON.stringify({
schemaVersion: 1,
requestId: 'request-kubernetes-rejected-1',
error: { code: 'forbidden' },
}),
);
response.writeHead(403, {
'content-type': 'application/json; charset=utf-8',
'content-length': String(body.length),
});
response.end(body);
});
});
const files = createClientFiles();
let closes = 0;
try {
await assert.rejects(
executeClusterPluginPackageManagementKubernetesClient(files.paths, {
createRuntime() {
return {
pods: {
async listNamespacedPod() {
return {
items: [
readyPod(
'ql3-plugin-package-management-aaaaa-11111',
),
],
};
},
},
async openPortForward() {
const stream = connectTcp({
host: '127.0.0.1',
port: server.port,
});
return {
stream,
close() {
closes += 1;
stream.destroy();
},
};
},
};
},
}),
(error) => {
assert.equal(
error instanceof ClusterPluginPackageManagementClientRemoteError,
true,
);
assert.equal(error.statusCode, 403);
assert.equal(error.responseCode, 'forbidden');
assert.equal(error.requestId, 'request-kubernetes-rejected-1');
return true;
},
);
assert.equal(closes, 1);
} finally {
await server.close();
rmSync(files.directory, { recursive: true, force: true });
}
});
test('rejects non-private kubeconfig and keeps CLI failure output secret-free', () => {
const files = createClientFiles(
kubeconfig({
user: {
token: KUBERNETES_TOKEN,
exec: {
apiVersion: 'client.authentication.k8s.io/v1',
command: '/usr/bin/cloud-login-secret',
},
},
}),
);
try {
chmodSync(files.kubeconfigFile, 0o644);
const result = spawnSync(
process.execPath,
[
CLIENT_CLI,
`--config=${files.paths.configFile}`,
`--command=${files.paths.commandFile}`,
`--assertion=${files.paths.assertionFile}`,
`--kubernetes=${files.paths.kubernetesFile}`,
],
{ encoding: 'utf8' },
);
assert.equal(result.status, 1);
assert.equal(result.stdout, '');
const fact = JSON.parse(result.stderr);
assert.deepEqual(fact, {
schemaVersion: 1,
component:
'qinglong3-plugin-package-management-kubernetes-client',
event: 'command_failed',
code:
'QL3_PLUGIN_PACKAGE_MANAGEMENT_KUBERNETES_CLIENT_CONFIG_INVALID',
});
for (const secret of [
KUBERNETES_TOKEN,
ASSERTION,
'cloud-login-secret',
files.directory,
]) {
assert.equal(result.stderr.includes(secret), false);
}
} finally {
rmSync(files.directory, { recursive: true, force: true });
}
});
@@ -0,0 +1,461 @@
const assert = require('node:assert/strict');
const { readFile, writeFile, chmod, mkdtemp, rm } = require('node:fs/promises');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterPluginPackageManagementProcessConfigError,
loadClusterPluginPackageManagementProcessConfig,
startClusterPluginPackageManagementProcess,
} = require('@qinglong/cluster-admin/plugin-package-management-process');
const SERVER_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-key.pem',
);
const SERVER_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem',
);
function enabledEnvironment(paths, overrides = {}) {
return {
QL3_PLUGIN_PACKAGE_MANAGEMENT_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
QL3_PLUGIN_PACKAGE_MANAGEMENT_HOST: '127.0.0.1',
QL3_PLUGIN_PACKAGE_MANAGEMENT_PORT: '8443',
QL3_PLUGIN_PACKAGE_MANAGEMENT_TLS_CERT_FILE: paths.certificateFile,
QL3_PLUGIN_PACKAGE_MANAGEMENT_TLS_KEY_FILE: paths.privateKeyFile,
QL3_PLUGIN_PACKAGE_MANAGEMENT_IDENTITY_KEYSET_FILE:
paths.identityKeysetFile,
QL3_PLUGIN_PACKAGE_PUBLISHER_TRUST_FILE: paths.publisherTrustFile,
QL3_PLUGIN_PACKAGE_TRUST_AUTHORITY_PROJECT_ID:
'cluster-trust-authority',
QL3_POSTGRES_PACKAGE_MANAGER_URL:
'postgresql://ql3_package_manager:secret@postgres.example.test/ql3',
QL3_POSTGRES_PACKAGE_MANAGER_TLS_MODE: 'disable',
QL3_POSTGRES_PACKAGE_MANAGER_ALLOW_INSECURE: 'true',
...overrides,
};
}
function publisherAuthority() {
const snapshot = {
schema: 'qinglong/plugin-package-publisher-trust-snapshot@v1',
keys: [
{
publisher: 'publisher-a.example',
keyId: 'key-a',
publicKeyDigest: 'a'.repeat(64),
notBeforeMs: 1,
notAfterMs: 10_000,
},
],
snapshotDigest: 'b'.repeat(64),
};
const head = {
schema: 'qinglong/plugin-package-publisher-trust-head@v1',
authorityId: 'cluster',
generation: 1,
baseSnapshotDigest: snapshot.snapshotDigest,
effectiveTrustDigest: snapshot.snapshotDigest,
updatedAtMs: 1_000,
headDigest: 'c'.repeat(64),
};
return {
publisherTrustEvidence: { registry: {}, snapshot },
async observePublisherTrust(_pool, input) {
assert.equal(input.authorityId, 'cluster');
assert.deepEqual(input.snapshot, snapshot);
return {
status: 'created',
head,
effectiveSnapshot: snapshot,
};
},
};
}
function readiness() {
return {
ready: true,
writablePrimary: true,
serverVersionNum: 180004,
serverMajor: 18,
currentUser: 'ql3_package_manager',
contractName: 'control-core',
contractVersion: 24,
migrationIds: ['pg-0025-plugin-package-materialized-revisions'],
};
}
function identities(overrides = {}) {
let reloads = 0;
return {
get reloads() {
return reloads;
},
provider: {
async reload() {
reloads += 1;
if (overrides.reload) return overrides.reload();
return {
schemaVersion: 1,
generation: 4,
digest: 'keyset-digest',
issuer: 'https://identity.example.test/',
audience: 'qinglong3-package-management',
activeKeyIds: ['identity-key-2'],
revokedKeyIds: ['identity-key-1'],
};
},
bind() {
throw new Error('HTTP stub must not authenticate');
},
},
};
}
async function tlsFixture(run) {
const directory = await mkdtemp(join(tmpdir(), 'ql3-management-process-'));
const paths = {
certificateFile: join(directory, 'tls.crt'),
privateKeyFile: join(directory, 'tls.key'),
identityKeysetFile: join(directory, 'keyset.json'),
publisherTrustFile: join(directory, 'publisher-trust.json'),
};
try {
await writeFile(paths.certificateFile, await readFile(SERVER_CERT), {
mode: 0o644,
});
await writeFile(paths.privateKeyFile, await readFile(SERVER_KEY), {
mode: 0o640,
});
await writeFile(paths.identityKeysetFile, '{}\n', { mode: 0o644 });
await writeFile(paths.publisherTrustFile, '{}\n', { mode: 0o644 });
return await run(paths);
} finally {
await rm(directory, { recursive: true, force: true });
}
}
test('disabled gate reads no profile, trust, TLS or PostgreSQL authority', async () => {
const reads = [];
const environment = new Proxy(
{ QL3_PLUGIN_PACKAGE_MANAGEMENT_ENABLED: 'false' },
{
get(target, property) {
reads.push(property);
if (property === 'QL3_PLUGIN_PACKAGE_MANAGEMENT_ENABLED') {
return target[property];
}
throw new Error(`disabled config read ${String(property)}`);
},
},
);
let opened = 0;
const runtime = await startClusterPluginPackageManagementProcess({
environment,
async openDatabase() {
opened += 1;
throw new Error('must not open');
},
});
assert.equal(runtime.status, 'disabled');
assert.equal(opened, 0);
assert.deepEqual(reads, ['QL3_PLUGIN_PACKAGE_MANAGEMENT_ENABLED']);
await runtime.close();
});
test('loads one explicit manager-only HTTPS and database configuration', () => {
const paths = {
certificateFile: '/run/ql3-management/tls.crt',
privateKeyFile: '/run/ql3-management/tls.key',
identityKeysetFile: '/run/ql3-management/keyset.json',
publisherTrustFile: '/run/ql3-management/publisher-trust.json',
};
const config = loadClusterPluginPackageManagementProcessConfig(
enabledEnvironment(paths),
);
assert.equal(config.enabled, true);
assert.equal(config.profile, 'cluster-admin');
assert.equal(config.port, 8443);
assert.equal(config.database.connection.tls.mode, 'disable');
assert.equal(config.database.pool.maxConnections, 2);
assert.equal(
config.database.pool.applicationName,
'qinglong3-plugin-package-manager',
);
assert.equal(config.http.maxConnections, 64);
assert.equal(config.http.maxConcurrentRequests, 32);
assert.equal(config.http.maxRateLimitPeers, 1024);
assert.deepEqual(config.quota, {
windowMs: 60_000,
proposeLimit: 30,
decideLimit: 60,
inspectLimit: 600,
});
assert.deepEqual(config.publisherTrust, {
file: '/run/ql3-management/publisher-trust.json',
authorityProjectId: 'cluster-trust-authority',
authorityId: 'cluster',
observerId: 'cluster-package-manager',
});
});
test('rejects profile drift, implicit insecure PostgreSQL and unsafe files', () => {
const paths = {
certificateFile: '/run/ql3-management/tls.crt',
privateKeyFile: '/run/ql3-management/tls.key',
identityKeysetFile: '/run/ql3-management/keyset.json',
publisherTrustFile: '/run/ql3-management/publisher-trust.json',
};
for (const environment of [
enabledEnvironment(paths, { QL3_PROFILE: 'cluster-control' }),
enabledEnvironment(paths, {
QL3_POSTGRES_PACKAGE_MANAGER_ALLOW_INSECURE: 'false',
}),
enabledEnvironment(paths, {
QL3_PLUGIN_PACKAGE_MANAGEMENT_TLS_KEY_FILE: 'relative.key',
}),
]) {
assert.throws(
() => loadClusterPluginPackageManagementProcessConfig(environment),
ClusterPluginPackageManagementProcessConfigError,
);
}
});
test('rejects configured request and connection ceilings above hard bounds', () => {
for (const overrides of [
{
QL3_PLUGIN_PACKAGE_MANAGEMENT_MAX_BODY_BYTES: String(256 * 1024 + 1),
},
{ QL3_PLUGIN_PACKAGE_MANAGEMENT_MAX_CONNECTIONS: '513' },
{ QL3_PLUGIN_PACKAGE_MANAGEMENT_INSPECT_QUOTA: '1001' },
]) {
assert.throws(
() =>
loadClusterPluginPackageManagementProcessConfig(
enabledEnvironment(
{
certificateFile: '/run/ql3-management/tls.crt',
privateKeyFile: '/run/ql3-management/tls.key',
identityKeysetFile: '/run/ql3-management/keyset.json',
publisherTrustFile:
'/run/ql3-management/publisher-trust.json',
},
overrides,
),
),
ClusterPluginPackageManagementProcessConfigError,
);
}
});
test('starts only after Package manager readiness and keyset validation then closes in order', async () => {
await tlsFixture(async (paths) => {
const order = [];
const identity = identities();
const pool = {
async query() {
throw new Error('construction must not query PostgreSQL');
},
async connect() {
throw new Error('construction must not acquire PostgreSQL');
},
};
let privateKey;
let httpOptions;
const runtime = await startClusterPluginPackageManagementProcess({
environment: enabledEnvironment(paths),
identities: identity.provider,
...publisherAuthority(),
async openDatabase() {
order.push('database.open');
return {
pool,
async close() {
order.push('database.close');
},
};
},
async assertReady(observedPool) {
order.push('database.ready');
assert.equal(observedPool, pool);
return readiness();
},
async startHttp(options) {
order.push('http.start');
httpOptions = options;
privateKey = options.tls.privateKey;
assert.equal(
privateKey.some((value) => value !== 0),
true,
);
return {
status: 'active',
address: { host: '127.0.0.1', port: 9443 },
availabilityStatus: () => 'ready',
withdraw() {},
async close() {
order.push('http.close');
},
};
},
now: () => 1_000,
});
assert.equal(runtime.status, 'active');
assert.deepEqual(order, ['database.open', 'database.ready', 'http.start']);
assert.equal(identity.reloads, 1);
assert.equal(
privateKey.every((value) => value === 0),
true,
);
assert.equal(typeof httpOptions.transport.execute, 'function');
assert.equal(httpOptions.identities, identity.provider);
assert.deepEqual(runtime.identity.activeKeyIds, ['identity-key-2']);
assert.equal(runtime.database.contractVersion, 24);
await Promise.all([runtime.close(), runtime.close()]);
assert.deepEqual(order, [
'database.open',
'database.ready',
'http.start',
'http.close',
'database.close',
]);
});
});
test('closes the Package manager database when readiness or HTTP startup fails', async () => {
await tlsFixture(async (paths) => {
for (const failureAt of ['readiness', 'http']) {
let closes = 0;
const identity = identities();
await assert.rejects(
startClusterPluginPackageManagementProcess({
environment: enabledEnvironment(paths),
identities: identity.provider,
...publisherAuthority(),
async openDatabase() {
return {
pool: {
async query() {
throw new Error('must not query');
},
async connect() {
throw new Error('must not connect');
},
},
async close() {
closes += 1;
},
};
},
async assertReady() {
if (failureAt === 'readiness') {
throw new Error('readiness failed');
}
return readiness();
},
async startHttp() {
throw new Error('HTTP failed');
},
}),
new RegExp(`${failureAt} failed`, 'i'),
);
assert.equal(closes, 1);
}
});
});
test('rejects publicly readable private TLS authority before opening a listener', async () => {
await tlsFixture(async (paths) => {
await chmod(paths.privateKeyFile, 0o644);
let starts = 0;
await assert.rejects(
startClusterPluginPackageManagementProcess({
environment: enabledEnvironment(paths),
identities: identities().provider,
...publisherAuthority(),
async openDatabase() {
return {
pool: {
async query() {
throw new Error('must not query');
},
async connect() {
throw new Error('must not connect');
},
},
async close() {},
};
},
async assertReady() {
return readiness();
},
async startHttp() {
starts += 1;
throw new Error('must not start');
},
}),
ClusterPluginPackageManagementProcessConfigError,
);
assert.equal(starts, 0);
});
});
test('clears loaded private key bytes when certificate loading fails', async () => {
await tlsFixture(async (paths) => {
const privateKeyBytes = await readFile(paths.privateKeyFile);
await chmod(paths.certificateFile, 0o666);
const originalAllocate = Buffer.alloc;
let privateKeyStorage;
Buffer.alloc = function allocate(size, ...rest) {
const bytes = originalAllocate.call(Buffer, size, ...rest);
if (size === privateKeyBytes.length + 1 && !privateKeyStorage) {
privateKeyStorage = bytes;
}
return bytes;
};
try {
await assert.rejects(
startClusterPluginPackageManagementProcess({
environment: enabledEnvironment(paths),
identities: identities().provider,
...publisherAuthority(),
async openDatabase() {
return {
pool: {
async query() {
throw new Error('must not query');
},
async connect() {
throw new Error('must not connect');
},
},
async close() {},
};
},
async assertReady() {
return readiness();
},
async startHttp() {
throw new Error('must not start');
},
}),
ClusterPluginPackageManagementProcessConfigError,
);
} finally {
Buffer.alloc = originalAllocate;
}
assert.ok(privateKeyStorage);
assert.equal(
privateKeyStorage.every((value) => value === 0),
true,
);
});
});
@@ -0,0 +1,911 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
fixture: installFixture,
} = require('../../../test/contracts/pluginPackageInstallRepositoryContract.cjs');
const {
ClusterPluginPackageManagementTransportAuthenticationError,
ClusterPluginPackageManagementTransportRequestError,
createClusterPluginPackageManagementTransport,
} = require('@qinglong/cluster-admin/plugin-package-management-transport');
const NOW = 1_000;
const PRIVATE_LOCATOR = `registry:private.example/qinglong/monitor@sha256:${'a'.repeat(
64,
)}`;
function principal(overrides = {}) {
return {
subject: { type: 'user', id: 'cluster-reviewer' },
authenticationId: 'oidc-session-secret',
authenticatedAtMs: NOW - 100,
expiresAtMs: NOW + 1_000,
assurance: 'multi_factor',
...overrides,
};
}
function actionInput() {
return {
lockId: 'cluster-monitor-v1',
projectId: 'default',
manifest: {
apiVersion: 'qinglong.io/v1alpha1',
kind: 'PluginPackage',
metadata: {
name: 'cluster-monitor',
displayName: 'Cluster Monitor',
version: '1.0.0',
description: 'must not cross the low-sensitive response boundary',
license: 'Apache-2.0',
},
spec: {
compatibility: {
qinglong: '>=3.0.0-0 <4.0.0',
architectures: ['arm64'],
deploymentProfiles: ['cluster'],
},
runtimes: [],
resources: {
memory: { recommended: '32Mi' },
disk: { install: '4Mi', working: '16Mi' },
},
permissions: {
network: { allowedHosts: [] },
secrets: ['private-token'],
tools: [],
},
contents: { tasks: [], workflows: [], prompts: [], tools: [] },
},
},
plan: {
schema: 'qinglong/plugin-package-install-plan@v1',
operation: 'install',
},
environment: {
qinglongVersion: '3.0.0-alpha.0',
architecture: 'arm64',
deploymentProfile: 'cluster',
runtimes: [],
availableMemoryBytes: 128 * 1024 * 1024,
availableDiskBytes: 512 * 1024 * 1024,
},
source: {
kind: 'registry',
locator: PRIVATE_LOCATOR,
artifactDigest: 'a'.repeat(64),
artifactBytes: 4_096,
contentDigest: 'b'.repeat(64),
},
architecture: 'arm64',
deploymentProfile: 'cluster',
targetGeneration: 1,
};
}
function proposal(createdAtMs = NOW) {
return {
schema: 'qinglong/plugin-package-install-proposal@v1',
actionRef: 'package:cluster-monitor:1',
projectId: 'default',
actionType: 'plugin_package.install',
permission: 'package.manage',
actionInput: actionInput(),
actionDigest: 'c'.repeat(64),
previewDigest: 'd'.repeat(64),
proposedBy: { type: 'user', id: 'cluster-reviewer' },
proposalFence: { projectVersion: 1, bindingVersion: 1 },
createdAtMs,
proposalDigest: 'e'.repeat(64),
};
}
function approval(overrides = {}) {
return {
schema: 'qinglong/approval-request@v1',
id: 'approval-cluster-monitor-1',
projectId: 'default',
version: 1,
state: 'pending',
action: {
permission: 'package.manage',
actionType: 'plugin_package.install',
actionRef: 'package:cluster-monitor:1',
actionDigest: 'c'.repeat(64),
previewDigest: 'd'.repeat(64),
},
risk: 'high',
decisionMode: 'separation_of_duty',
requestedBy: { type: 'user', id: 'cluster-requester' },
requestedAtMs: NOW,
expiresAtMs: NOW + 15 * 60 * 1_000,
requestFence: { projectVersion: 1, bindingVersion: 1 },
decisionId: null,
decision: null,
decisionReasonCode: null,
decidedBy: null,
decisionAuthenticationId: null,
decisionAssurance: null,
decidedAtMs: null,
decisionFence: null,
consumptionId: null,
dispatchId: null,
consumedBy: null,
consumedAtMs: null,
consumptionFence: null,
...overrides,
};
}
function proposeCommand() {
return {
schemaVersion: 1,
operation: 'plugin-package.propose',
request: {
actionRef: 'package:cluster-monitor:1',
approvalRequestId: 'approval-cluster-monitor-1',
proposalAuditEventId: 'proposal-audit-1',
approvalAuditEventId: 'approval-audit-1',
actionInput: actionInput(),
},
};
}
function decideCommand(overrides = {}) {
return {
schemaVersion: 1,
operation: 'plugin-package.decide',
request: {
actionRef: 'package:cluster-monitor:1',
approvalRequestId: 'approval-cluster-monitor-1',
expectedVersion: 1,
decisionId: 'decision-cluster-monitor-1',
auditEventId: 'decision-audit-1',
decision: 'approved',
reasonCode: 'reviewed',
...overrides,
},
};
}
function inspectCommand(overrides = {}) {
return {
schemaVersion: 1,
operation: 'plugin-package.inspect',
request: {
actionRef: 'package:cluster-monitor:1',
approvalRequestId: 'approval-cluster-monitor-1',
inspectionId: 'inspection-cluster-monitor-1',
...overrides,
},
};
}
function installationInspectCommand(overrides = {}) {
return {
schemaVersion: 1,
operation: 'plugin-package.installation.inspect',
request: {
projectId: 'default',
packageName: 'cluster-monitor',
inspectionId: 'installation-inspection-cluster-monitor-1',
...overrides,
},
};
}
function installationListCommand(overrides = {}) {
return {
schemaVersion: 1,
operation: 'plugin-package.installation.list',
request: {
projectId: 'default',
limit: 8,
inspectionId: 'installation-list-cluster-monitor-1',
...overrides,
},
};
}
function currentInstallationItem(quarantined = false) {
const record = installFixture('transport-installation', {
packageName: 'cluster-monitor',
installationId: 'cluster-monitor-installation',
}).install;
return {
record,
quarantine: quarantined
? {
eventDigest: '1'.repeat(64),
reasonCode: 'confirmed_key_compromise',
authorizationMode: 'break_glass',
occurredAtMs: NOW - 10,
capabilityStatus: 'not_active',
receiptDigest: '2'.repeat(64),
committedAtMs: NOW - 9,
}
: null,
};
}
function fakeService(
inspectResult = { proposal: null, approvalRequest: null },
installationItem = currentInstallationItem(),
) {
const calls = {
propose: [],
decide: [],
consume: [],
inspect: [],
inspectAuthorized: [],
inspectInstallationAuthorized: [],
listInstallationsAuthorized: [],
dispatch: [],
};
return {
calls,
service: {
async propose(request) {
calls.propose.push(request);
return {
proposalStatus: 'created',
approvalStatus: 'created',
proposal: proposal(request.requestedAtMs),
approvalRequest: approval({
requestedBy: request.principal.subject,
requestedAtMs: request.requestedAtMs,
}),
};
},
async decide(request) {
calls.decide.push(request);
return {
status: 'decided',
request: approval({
version: 2,
state: request.decision,
decisionId: request.decisionId,
decision: request.decision,
decisionReasonCode: request.reasonCode,
decidedBy: request.principal.subject,
decisionAuthenticationId: request.principal.authenticationId,
decisionAssurance: request.principal.assurance,
decidedAtMs: request.decidedAtMs,
decisionFence: { projectVersion: 1, bindingVersion: 1 },
}),
};
},
async consume(request) {
calls.consume.push(request);
throw new Error('consume must not be public');
},
async inspect(actionRef, approvalRequestId) {
calls.inspect.push({ actionRef, approvalRequestId });
return inspectResult;
},
async inspectAuthorized(request) {
calls.inspectAuthorized.push(request);
return inspectResult;
},
async inspectInstallationAuthorized(request) {
calls.inspectInstallationAuthorized.push(request);
return installationItem;
},
async listInstallationsAuthorized(request) {
calls.listInstallationsAuthorized.push(request);
return {
items: [installationItem],
truncated: false,
};
},
async dispatch(limit) {
calls.dispatch.push(limit);
throw new Error('dispatch must not be public');
},
},
};
}
function authentication(value = principal()) {
let calls = 0;
return {
get calls() {
return calls;
},
authority: {
async authenticate() {
calls += 1;
return value;
},
},
};
}
function lifecyclePlan() {
return {
schema: 'qinglong/plugin-package-lifecycle-plan@v1',
actionRef: 'lifecycle:cluster-monitor:disable:1',
impact: {
schema: 'qinglong/plugin-package-lifecycle-impact@v1',
action: 'disable',
target: {
projectId: 'default',
packageName: 'cluster-monitor',
installationId: 'cluster-monitor-installation',
lockDigest: '1'.repeat(64),
installVersion: 4,
installRecordDigest: '2'.repeat(64),
},
expected: {
version: 0,
disposition: 'active',
eventDigest: null,
},
generationDigest: '3'.repeat(64),
materializedRevisionDigest: '4'.repeat(64),
currentToolSnapshotDigest: '5'.repeat(64),
taskIds: ['collect'],
resourceCounts: { tasks: 1, tools: 0, workflows: 0, prompts: 0 },
referenceGraphDigest: '6'.repeat(64),
blockingReferences: [],
impactDigest: '7'.repeat(64),
},
requestedBy: { type: 'user', id: 'cluster-reviewer' },
plannedAtMs: NOW - 10,
expiresAtMs: NOW + 10_000,
planDigest: '8'.repeat(64),
};
}
function fakeLifecycle() {
const calls = { propose: [], decide: [], inspectAuthorized: [] };
return {
calls,
service: {
async propose(request) {
calls.propose.push(request);
return {
plan: lifecyclePlan(),
approvalStatus: 'created',
approvalRequest: approval({
id: 'approval-lifecycle-cluster-monitor-1',
action: {
permission: 'package.manage',
actionType: 'plugin_package.lifecycle.disable',
actionRef: lifecyclePlan().actionRef,
actionDigest: '9'.repeat(64),
previewDigest: lifecyclePlan().impact.impactDigest,
},
requestedBy: request.principal.subject,
}),
};
},
async decide(request) {
calls.decide.push(request);
return {
status: 'decided',
request: approval({
id: request.approvalRequestId,
version: 2,
state: request.decision,
decisionId: request.decisionId,
decision: request.decision,
decisionReasonCode: request.reasonCode,
decidedBy: request.principal.subject,
decisionAuthenticationId: request.principal.authenticationId,
decisionAssurance: request.principal.assurance,
decidedAtMs: NOW,
decisionFence: { projectVersion: 1, bindingVersion: 1 },
}),
};
},
async inspectAuthorized(request) {
calls.inspectAuthorized.push(request);
return {
plan: lifecyclePlan(),
approvalRequest: null,
stale: true,
};
},
},
};
}
function publisherProposal(overrides = {}) {
return {
schema: 'qinglong/plugin-package-publisher-key-revocation-proposal@v1',
actionRef: 'publisher-revoke:publisher-a.example:key-a',
projectId: 'cluster-trust-authority',
actionType: 'plugin_package.publisher_key.revoke',
permission: 'package.manage',
actionInput: {
authorityProjectId: 'cluster-trust-authority',
trustAuthorityId: 'cluster',
trustGeneration: 4,
publisher: 'publisher-a.example',
keyId: 'key-a',
previousTrustDigest: '1'.repeat(64),
currentTrustDigest: '2'.repeat(64),
authorizationMode: 'dual_control',
reasonCode: 'suspected_key_compromise',
},
actionDigest: '3'.repeat(64),
previewDigest: '4'.repeat(64),
proposedBy: { type: 'user', id: 'cluster-reviewer' },
proposerAssurance: 'multi_factor',
proposalFence: { projectVersion: 1, bindingVersion: 1 },
createdAtMs: NOW,
proposalDigest: '5'.repeat(64),
...overrides,
};
}
function publisherApproval(candidate = publisherProposal(), overrides = {}) {
return approval({
id: 'approval-publisher-revoke-1',
projectId: candidate.projectId,
action: {
permission: candidate.permission,
actionType: candidate.actionType,
actionRef: candidate.actionRef,
actionDigest: candidate.actionDigest,
previewDigest: candidate.previewDigest,
},
risk: 'critical',
...overrides,
});
}
function publisherCommand(operation, request = {}) {
const common = {
actionRef: 'publisher-revoke:publisher-a.example:key-a',
approvalRequestId: 'approval-publisher-revoke-1',
};
if (operation.endsWith('.propose')) {
return {
schemaVersion: 1,
operation,
request: {
...common,
proposalAuditEventId: 'proposal-publisher-audit-1',
approvalAuditEventId: 'approval-publisher-audit-1',
publisher: 'publisher-a.example',
keyId: 'key-a',
authorizationMode: 'dual_control',
reasonCode: 'suspected_key_compromise',
...request,
},
};
}
if (operation.endsWith('.decide')) {
return {
schemaVersion: 1,
operation,
request: {
...common,
expectedVersion: 1,
decisionId: 'decision-publisher-revoke-1',
auditEventId: 'decision-publisher-audit-1',
decision: 'approved',
reasonCode: 'reviewed',
...request,
},
};
}
return {
schemaVersion: 1,
operation,
request: {
...common,
inspectionId: 'inspection-publisher-revoke-1',
...request,
},
};
}
function fakePublisherTrust(
inspectResult = { proposal: null, approvalRequest: null },
) {
const calls = { propose: [], inspect: [], inspectAuthorized: [] };
return {
calls,
service: {
async propose(request) {
calls.propose.push(request);
const candidate = publisherProposal({
createdAtMs: request.requestedAtMs,
proposedBy: request.principal.subject,
proposerAssurance: request.principal.assurance,
actionInput: {
...publisherProposal().actionInput,
authorizationMode: request.authorizationMode,
reasonCode: request.reasonCode,
},
});
return {
proposalStatus: 'created',
approvalStatus: 'created',
proposal: candidate,
approvalRequest: publisherApproval(candidate, {
requestedBy: request.principal.subject,
requestedAtMs: request.requestedAtMs,
}),
};
},
async inspect(actionRef, approvalRequestId) {
calls.inspect.push({ actionRef, approvalRequestId });
return inspectResult;
},
async inspectAuthorized(request) {
calls.inspectAuthorized.push(request);
return inspectResult;
},
},
};
}
test('rejects weak or non-user principals before Package management state access', async () => {
for (const candidate of [
principal({ assurance: 'single_factor' }),
principal({ subject: { type: 'api_app', id: 'cluster-api' } }),
null,
]) {
const fixture = fakeService();
const auth = authentication(candidate);
const transport = createClusterPluginPackageManagementTransport({
service: fixture.service,
now: () => NOW,
});
await assert.rejects(
transport.execute(proposeCommand(), auth.authority),
ClusterPluginPackageManagementTransportAuthenticationError,
);
assert.equal(auth.calls, 1);
assert.deepEqual(fixture.calls.inspect, []);
assert.deepEqual(fixture.calls.propose, []);
}
});
test('rejects internal consume and dispatch operations before authentication', async () => {
const fixture = fakeService();
const auth = authentication();
const transport = createClusterPluginPackageManagementTransport({
service: fixture.service,
now: () => NOW,
});
for (const operation of [
'plugin-package.consume',
'plugin-package.dispatch',
]) {
await assert.rejects(
transport.execute(
{ schemaVersion: 1, operation, request: {} },
auth.authority,
),
ClusterPluginPackageManagementTransportRequestError,
);
}
assert.equal(auth.calls, 0);
assert.deepEqual(fixture.calls.consume, []);
assert.deepEqual(fixture.calls.dispatch, []);
});
test('injects strong transport authority and emits only low-sensitive proposal data', async () => {
const fixture = fakeService();
const auth = authentication();
const transport = createClusterPluginPackageManagementTransport({
service: fixture.service,
now: () => NOW,
});
const result = await transport.execute(proposeCommand(), auth.authority);
assert.equal(auth.calls, 1);
assert.equal(fixture.calls.inspect.length, 1);
assert.equal(fixture.calls.propose.length, 1);
assert.equal(fixture.calls.propose[0].requestedAtMs, NOW);
assert.deepEqual(fixture.calls.propose[0].principal, principal());
assert.equal(Object.hasOwn(proposeCommand().request, 'principal'), false);
assert.equal(Object.hasOwn(proposeCommand().request, 'requestedAtMs'), false);
const serialized = JSON.stringify(result);
assert.equal(serialized.includes(PRIVATE_LOCATOR), false);
assert.equal(serialized.includes('oidc-session-secret'), false);
assert.equal(serialized.includes('private-token'), false);
assert.equal(
serialized.includes('must not cross the low-sensitive response boundary'),
false,
);
assert.equal(result.proposal.packageName, 'cluster-monitor');
assert.equal(result.proposal.sourceKind, 'registry');
});
test('uses the durable proposal time when recovering a partial proposal', async () => {
const durableProposal = proposal(NOW - 250);
const fixture = fakeService({
proposal: durableProposal,
approvalRequest: null,
});
const transport = createClusterPluginPackageManagementTransport({
service: fixture.service,
now: () => NOW,
});
await transport.execute(proposeCommand(), authentication().authority);
assert.equal(
fixture.calls.propose[0].requestedAtMs,
durableProposal.createdAtMs,
);
});
test('replays an exact decision without mutating and timestamps a new decision', async () => {
const decided = approval({
version: 2,
state: 'approved',
decisionId: 'decision-cluster-monitor-1',
decision: 'approved',
decisionReasonCode: 'reviewed',
decidedBy: { type: 'user', id: 'cluster-reviewer' },
decisionAuthenticationId: 'previous-oidc-session',
decisionAssurance: 'multi_factor',
decidedAtMs: NOW - 10,
decisionFence: { projectVersion: 1, bindingVersion: 1 },
});
const replayFixture = fakeService({
proposal: proposal(),
approvalRequest: decided,
});
const replayTransport = createClusterPluginPackageManagementTransport({
service: replayFixture.service,
now: () => NOW,
});
const replay = await replayTransport.execute(
decideCommand(),
authentication().authority,
);
assert.equal(replay.status, 'existing');
assert.deepEqual(replayFixture.calls.decide, []);
const newFixture = fakeService({
proposal: proposal(),
approvalRequest: approval(),
});
const newTransport = createClusterPluginPackageManagementTransport({
service: newFixture.service,
now: () => NOW,
});
const decidedResult = await newTransport.execute(
decideCommand({ decisionId: 'decision-cluster-monitor-2' }),
authentication().authority,
);
assert.equal(decidedResult.status, 'decided');
assert.equal(newFixture.calls.decide.length, 1);
assert.equal(newFixture.calls.decide[0].decidedAtMs, NOW);
assert.deepEqual(newFixture.calls.decide[0].principal, principal());
});
test('routes public inspection through the authorized, quota-aware service path', async () => {
const fixture = fakeService({
proposal: proposal(),
approvalRequest: approval(),
});
const transport = createClusterPluginPackageManagementTransport({
service: fixture.service,
now: () => NOW,
});
const result = await transport.execute(
inspectCommand(),
authentication().authority,
);
assert.equal(result.operation, 'plugin-package.inspect');
assert.equal(fixture.calls.inspect.length, 0);
assert.deepEqual(fixture.calls.inspectAuthorized, [
{
actionRef: 'package:cluster-monitor:1',
approvalRequestId: 'approval-cluster-monitor-1',
inspectionId: 'inspection-cluster-monitor-1',
principal: principal(),
},
]);
});
test('routes bounded installation inventory and reports quarantine availability', async () => {
const fixture = fakeService(
{ proposal: null, approvalRequest: null },
currentInstallationItem(true),
);
const transport = createClusterPluginPackageManagementTransport({
service: fixture.service,
now: () => NOW,
});
const inspected = await transport.execute(
installationInspectCommand(),
authentication().authority,
);
assert.equal(inspected.installation.availability, 'quarantined');
assert.equal(
inspected.installation.quarantineReason,
'confirmed_key_compromise',
);
assert.equal(inspected.installation.withdrawalStatus, 'not_active');
assert.deepEqual(fixture.calls.inspectInstallationAuthorized, [
{
...installationInspectCommand().request,
principal: principal(),
},
]);
const listed = await transport.execute(
installationListCommand(),
authentication().authority,
);
assert.equal(listed.installations.length, 1);
assert.equal(listed.truncated, false);
assert.equal(listed.next, null);
assert.deepEqual(fixture.calls.listInstallationsAuthorized, [
{
...installationListCommand().request,
principal: principal(),
},
]);
});
test('routes lifecycle review without exposing executor mutation authority', async () => {
const management = fakeService();
const lifecycle = fakeLifecycle();
const transport = createClusterPluginPackageManagementTransport({
service: management.service,
lifecycle: lifecycle.service,
now: () => NOW,
});
const proposed = await transport.execute(
{
schemaVersion: 1,
operation: 'plugin-package.lifecycle.propose',
request: {
actionRef: lifecyclePlan().actionRef,
approvalRequestId: 'approval-lifecycle-cluster-monitor-1',
approvalAuditEventId: 'audit-lifecycle-approval-1',
},
},
authentication().authority,
);
assert.equal(proposed.plan.action, 'disable');
assert.equal(proposed.plan.impactDigest, '7'.repeat(64));
assert.equal(lifecycle.calls.propose.length, 1);
assert.deepEqual(lifecycle.calls.propose[0].principal, principal());
const decided = await transport.execute(
{
schemaVersion: 1,
operation: 'plugin-package.lifecycle.decide',
request: {
actionRef: lifecyclePlan().actionRef,
approvalRequestId: 'approval-lifecycle-cluster-monitor-1',
expectedVersion: 1,
decisionId: 'decision-lifecycle-cluster-monitor-1',
auditEventId: 'audit-lifecycle-decision-1',
decision: 'approved',
reasonCode: 'reviewed',
},
},
authentication().authority,
);
assert.equal(decided.status, 'decided');
assert.equal(lifecycle.calls.decide.length, 1);
const inspected = await transport.execute(
{
schemaVersion: 1,
operation: 'plugin-package.lifecycle.inspect',
request: {
actionRef: lifecyclePlan().actionRef,
approvalRequestId: 'approval-lifecycle-cluster-monitor-1',
inspectionId: 'inspection-lifecycle-cluster-monitor-1',
},
},
authentication().authority,
);
assert.equal(inspected.stale, true);
assert.equal(inspected.approval, null);
assert.equal(lifecycle.calls.inspectAuthorized.length, 1);
assert.deepEqual(management.calls.consume, []);
assert.deepEqual(management.calls.dispatch, []);
});
test('routes publisher revocation proposal with derived-only low-sensitive output', async () => {
const management = fakeService();
const publisherTrust = fakePublisherTrust();
const transport = createClusterPluginPackageManagementTransport({
service: management.service,
publisherTrust: publisherTrust.service,
now: () => NOW,
});
const command = publisherCommand(
'plugin-package.publisher-revocation.propose',
);
const result = await transport.execute(command, authentication().authority);
assert.equal(result.operation, command.operation);
assert.equal(publisherTrust.calls.propose.length, 1);
assert.equal(Object.hasOwn(command.request, 'previousTrustDigest'), false);
assert.equal(Object.hasOwn(command.request, 'currentTrustDigest'), false);
assert.deepEqual(Object.keys(result.proposal).sort(), [
'actionDigest',
'actionRef',
'authorizationMode',
'createdAtMs',
'currentTrustDigest',
'keyId',
'previewDigest',
'previousTrustDigest',
'projectId',
'proposalDigest',
'publisher',
'reasonCode',
'trustAuthorityId',
'trustGeneration',
]);
});
test('requires hardware assurance for break-glass confirmation', async () => {
const candidate = publisherProposal({
actionInput: {
...publisherProposal().actionInput,
authorizationMode: 'break_glass',
},
proposerAssurance: 'hardware',
});
const state = {
proposal: candidate,
approvalRequest: publisherApproval(candidate, {
decisionMode: 'human_confirmation',
}),
};
const management = fakeService();
const publisherTrust = fakePublisherTrust(state);
const transport = createClusterPluginPackageManagementTransport({
service: management.service,
publisherTrust: publisherTrust.service,
now: () => NOW,
});
await assert.rejects(
transport.execute(
publisherCommand('plugin-package.publisher-revocation.decide'),
authentication().authority,
),
ClusterPluginPackageManagementTransportAuthenticationError,
);
const result = await transport.execute(
publisherCommand('plugin-package.publisher-revocation.decide'),
authentication(principal({ assurance: 'hardware' })).authority,
);
assert.equal(result.operation, 'plugin-package.publisher-revocation.decide');
assert.equal(management.calls.decide.length, 1);
});
test('authorizes publisher revocation inspection through its scoped service', async () => {
const candidate = publisherProposal();
const state = {
proposal: candidate,
approvalRequest: publisherApproval(candidate),
};
const management = fakeService();
const publisherTrust = fakePublisherTrust(state);
const transport = createClusterPluginPackageManagementTransport({
service: management.service,
publisherTrust: publisherTrust.service,
now: () => NOW,
});
const result = await transport.execute(
publisherCommand('plugin-package.publisher-revocation.inspect'),
authentication().authority,
);
assert.equal(result.operation, 'plugin-package.publisher-revocation.inspect');
assert.equal(publisherTrust.calls.inspect.length, 0);
assert.equal(publisherTrust.calls.inspectAuthorized.length, 1);
});
@@ -0,0 +1,511 @@
const assert = require('node:assert/strict');
const {
createHash,
generateKeyPairSync,
sign,
} = require('node:crypto');
const { test } = require('node:test');
const {
PluginPackagePublisherTrustRegistry,
PLUGIN_PACKAGE_SIGNATURE_SCHEMA,
pluginPackageContentTreeDigest,
pluginPackagePublisherSignaturePayload,
} = require('@qinglong/runtime-core/plugin-package-bundle');
const {
PLUGIN_PACKAGE_API_VERSION,
PLUGIN_PACKAGE_KIND,
planPluginPackageInstall,
} = require('@qinglong/runtime-core/plugin-package');
const {
InvalidPluginPackageInstallError,
PluginPackageInstallUnavailableError,
createPluginPackageLock,
pluginPackageInstallActionDigest,
pluginPackageInstallPlanDigest,
serializePluginPackageManifest,
} = require('@qinglong/runtime-core/plugin-package-install');
const {
createPluginPackageResourceGenerationFromReferences,
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
const {
ClusterPluginPackageOciResourceByteSource,
ClusterPluginPackageOciStageAuthority,
PLUGIN_PACKAGE_OCI_ARTIFACT_TYPE,
PLUGIN_PACKAGE_OCI_CONFIG_MEDIA_TYPE,
PLUGIN_PACKAGE_OCI_SIGNATURE_ARTIFACT_TYPE,
PLUGIN_PACKAGE_OCI_SIGNATURE_CONFIG_MEDIA_TYPE,
} = require('@qinglong/cluster-admin/plugin-package-oci-stage');
const OCI_MANIFEST = 'application/vnd.oci.image.manifest.v1+json';
const OCI_INDEX = 'application/vnd.oci.image.index.v1+json';
const BUNDLE = 'application/vnd.qinglong.package.v1+tar';
const REGISTRY = 'registry.example.com';
const REPOSITORY = 'qinglong/example-monitor';
const PUBLISHER = 'packages.example.com';
const KEY_ID = 'release-2026';
function sha256(value) {
return createHash('sha256').update(value).digest('hex');
}
function bytes(value) {
return Buffer.from(JSON.stringify(value));
}
function octal(value, width) {
return Buffer.from(
`${value.toString(8).padStart(width - 1, '0')}\0`,
'ascii',
);
}
function tarHeader(path, size) {
const header = Buffer.alloc(512);
Buffer.from(path).copy(header, 0);
Buffer.from('0000644\0').copy(header, 100);
Buffer.from('0000000\0').copy(header, 108);
Buffer.from('0000000\0').copy(header, 116);
octal(size, 12).copy(header, 124);
Buffer.from('00000000000\0').copy(header, 136);
header.fill(0x20, 148, 156);
Buffer.from('0').copy(header, 156);
Buffer.from('ustar\0').copy(header, 257);
Buffer.from('00').copy(header, 263);
const checksum = header.reduce((total, byte) => total + byte, 0);
Buffer.from(`${checksum.toString(8).padStart(6, '0')}\0 `).copy(header, 148);
return header;
}
function tar(entries) {
const parts = [];
for (const entry of entries) {
parts.push(tarHeader(entry.path, entry.body.byteLength), entry.body);
const padding = (512 - (entry.body.byteLength % 512)) % 512;
if (padding > 0) parts.push(Buffer.alloc(padding));
}
parts.push(Buffer.alloc(1024));
return Buffer.concat(parts);
}
function manifest() {
return {
apiVersion: PLUGIN_PACKAGE_API_VERSION,
kind: PLUGIN_PACKAGE_KIND,
metadata: {
name: 'example-monitor',
displayName: 'Example Monitor',
version: '1.2.0',
description: 'One cluster package',
license: 'Apache-2.0',
},
spec: {
compatibility: {
qinglong: '>=3.0.0-0 <4.0.0',
architectures: ['arm64'],
deploymentProfiles: ['cluster-control'],
},
runtimes: [],
resources: {
memory: { recommended: '16Mi' },
disk: { install: '4Mi', working: '16Mi' },
},
permissions: {
network: { allowedHosts: [] },
secrets: [],
tools: [],
},
contents: {
tasks: ['tasks/collect.json'],
workflows: [],
prompts: [],
tools: [],
},
},
};
}
function fixture() {
const packageManifest = manifest();
const resourceMaterial = Buffer.from(
JSON.stringify({
schema: 'qinglong/plugin-package-task-resource@v1',
id: 'collect',
}),
);
const artifact = tar([
{
path: 'package.json',
body: Buffer.from(serializePluginPackageManifest(packageManifest)),
},
{
path: 'tasks/collect.json',
body: resourceMaterial,
},
]);
const packageConfig = bytes({
schema: 'qinglong/plugin-package-oci-config@v1',
manifest: packageManifest,
});
const packageConfigDigest = sha256(packageConfig);
const packageManifestValue = {
schemaVersion: 2,
mediaType: OCI_MANIFEST,
artifactType: PLUGIN_PACKAGE_OCI_ARTIFACT_TYPE,
config: {
mediaType: PLUGIN_PACKAGE_OCI_CONFIG_MEDIA_TYPE,
digest: `sha256:${packageConfigDigest}`,
size: packageConfig.byteLength,
},
layers: [
{
mediaType: BUNDLE,
digest: `sha256:${sha256(artifact)}`,
size: artifact.byteLength,
},
],
};
const packageManifestBytes = bytes(packageManifestValue);
const packageManifestDigest = sha256(packageManifestBytes);
const environment = {
qinglongVersion: '3.0.0-alpha.0',
architecture: 'arm64',
deploymentProfile: 'cluster-control',
runtimes: [],
availableMemoryBytes: 128 * 1024 * 1024,
availableDiskBytes: 256 * 1024 * 1024,
};
const plan = planPluginPackageInstall(packageManifest, environment);
const source = {
kind: 'oci',
locator: `oci://${REGISTRY}/${REPOSITORY}@sha256:${packageManifestDigest}`,
artifactDigest: sha256(artifact),
artifactBytes: artifact.byteLength,
contentDigest: pluginPackageContentTreeDigest([
{
path: 'tasks/collect.json',
bytes: resourceMaterial.byteLength,
digest: sha256(resourceMaterial),
},
]),
};
const action = {
lockId: 'lock-cluster-oci',
projectId: 'default',
manifest: packageManifest,
plan,
environment,
source,
architecture: 'arm64',
deploymentProfile: 'cluster-control',
targetGeneration: 1,
};
const lock = createPluginPackageLock({
...action,
approval: {
requestId: 'approval-cluster-oci',
requestVersion: 1,
dispatchId: 'dispatch-cluster-oci',
actionDigest: pluginPackageInstallActionDigest(action),
previewDigest: pluginPackageInstallPlanDigest(plan),
approvedBy: { type: 'user', id: 'owner-001' },
approvedAtMs: 100,
expiresAtMs: 10_000,
fence: { projectVersion: 1, bindingVersion: 1 },
},
createdAtMs: 200,
});
const generation = createPluginPackageResourceGenerationFromReferences({
installationId: 'install-cluster-oci',
projectId: lock.projectId,
packageName: lock.packageName,
lockDigest: lock.lockDigest,
generation: lock.targetGeneration,
previousActiveLockDigest: null,
contentDigest: lock.source.contentDigest,
resources: lock.resources,
});
const { publicKey, privateKey } = generateKeyPairSync('ed25519');
const trust = new PluginPackagePublisherTrustRegistry([
{
publisher: PUBLISHER,
keyId: KEY_ID,
publicKeyPem: publicKey.export({ format: 'pem', type: 'spki' }),
notBeforeMs: 100,
notAfterMs: 10_000,
},
]);
const signature = {
schema: PLUGIN_PACKAGE_SIGNATURE_SCHEMA,
publisher: PUBLISHER,
keyId: KEY_ID,
signature: sign(
null,
pluginPackagePublisherSignaturePayload(lock, PUBLISHER, KEY_ID),
privateKey,
).toString('base64url'),
};
const signatureConfig = bytes(signature);
const signatureConfigDigest = sha256(signatureConfig);
const signatureManifestValue = {
schemaVersion: 2,
mediaType: OCI_MANIFEST,
artifactType: PLUGIN_PACKAGE_OCI_SIGNATURE_ARTIFACT_TYPE,
config: {
mediaType: PLUGIN_PACKAGE_OCI_SIGNATURE_CONFIG_MEDIA_TYPE,
digest: `sha256:${signatureConfigDigest}`,
size: signatureConfig.byteLength,
},
layers: [],
subject: {
mediaType: OCI_MANIFEST,
digest: `sha256:${packageManifestDigest}`,
size: packageManifestBytes.byteLength,
},
};
const signatureManifestBytes = bytes(signatureManifestValue);
const signatureManifestDigest = sha256(signatureManifestBytes);
const referrers = bytes({
schemaVersion: 2,
mediaType: OCI_INDEX,
manifests: [
{
mediaType: OCI_MANIFEST,
digest: `sha256:${signatureManifestDigest}`,
size: signatureManifestBytes.byteLength,
artifactType: PLUGIN_PACKAGE_OCI_SIGNATURE_ARTIFACT_TYPE,
annotations: {
'qinglong.io/plugin-package-lock-digest': lock.lockDigest,
},
},
],
});
const prefix = `https://${REGISTRY}/v2/${REPOSITORY}`;
const routes = new Map([
[
`${prefix}/manifests/sha256:${packageManifestDigest}`,
packageManifestBytes,
],
[`${prefix}/blobs/sha256:${packageConfigDigest}`, packageConfig],
[
`${prefix}/referrers/sha256:${packageManifestDigest}?artifactType=${encodeURIComponent(
PLUGIN_PACKAGE_OCI_SIGNATURE_ARTIFACT_TYPE,
)}`,
referrers,
],
[
`${prefix}/manifests/sha256:${signatureManifestDigest}`,
signatureManifestBytes,
],
[`${prefix}/blobs/sha256:${signatureConfigDigest}`, signatureConfig],
[`${prefix}/blobs/sha256:${sha256(artifact)}`, artifact],
]);
let calls = 0;
const fetch = async (url, init) => {
calls += 1;
assert.equal(init.method, 'GET');
assert.equal(init.redirect, 'error');
assert.equal(init.signal.aborted, false);
const body = routes.get(url);
if (!body) return { status: 404, headers: { get: () => null }, body: null };
return {
status: 200,
headers: {
get(name) {
return name.toLowerCase() === 'content-length'
? String(body.byteLength)
: null;
},
},
body: (async function* () {
for (let offset = 0; offset < body.byteLength; offset += 37) {
yield body.subarray(offset, Math.min(offset + 37, body.byteLength));
}
})(),
};
};
return {
artifact,
fetch,
generation,
lock,
packageManifest,
resourceMaterial,
routes,
trust,
calls: () => calls,
};
}
test('streams one allowlisted OCI artifact and reuses bounded evidence for activation', async () => {
const value = fixture();
const authority = new ClusterPluginPackageOciStageAuthority({
allowedRegistries: [REGISTRY],
trust: value.trust,
fetch: value.fetch,
requestTimeoutMs: 1_000,
});
const stage = await authority.stage(value.lock);
assert.equal(stage.stageRef, `cluster-oci:${value.lock.lockDigest}`);
assert.equal(stage.artifactDigest, value.lock.source.artifactDigest);
assert.equal(stage.manifestDigest, value.lock.manifestDigest);
assert.equal(stage.contentDigest, value.lock.source.contentDigest);
assert.match(stage.evidenceDigest, /^[0-9a-f]{64}$/);
assert.equal(value.calls(), 6);
await authority.verify(value.lock, {
...stage,
stagedAtMs: 201,
receiptDigest: 'f'.repeat(64),
});
assert.equal(value.calls(), 6);
});
test('re-resolves durable stage evidence after a process restart', async () => {
const value = fixture();
const first = new ClusterPluginPackageOciStageAuthority({
allowedRegistries: [REGISTRY],
trust: value.trust,
fetch: value.fetch,
});
const stage = await first.stage(value.lock);
const restarted = new ClusterPluginPackageOciStageAuthority({
allowedRegistries: [REGISTRY],
trust: value.trust,
fetch: value.fetch,
});
await restarted.verify(value.lock, {
...stage,
stagedAtMs: 201,
receiptDigest: 'f'.repeat(64),
});
assert.equal(value.calls(), 12);
});
test('captures one bounded verified OCI layer as a caller-owned resource session', async () => {
const value = fixture();
const authority = new ClusterPluginPackageOciStageAuthority({
allowedRegistries: [REGISTRY],
trust: value.trust,
fetch: value.fetch,
});
const source = new ClusterPluginPackageOciResourceByteSource({
authority,
lockSource: {
async findLock(lockDigest) {
assert.equal(lockDigest, value.lock.lockDigest);
return value.lock;
},
},
});
const reader = await source.open(value.generation);
assert.deepEqual(
await reader.read('package.json', 64 * 1024),
Buffer.from(serializePluginPackageManifest(value.packageManifest)),
);
assert.deepEqual(
await reader.read('tasks/collect.json', 1024 * 1024),
value.resourceMaterial,
);
await assert.rejects(
reader.read('tasks/collect.json', 1024 * 1024),
/unknown or exceeds/,
);
await reader.close();
assert.equal(value.calls(), 6);
});
test('fails before network access for a non-allowlisted registry', async () => {
const value = fixture();
const authority = new ClusterPluginPackageOciStageAuthority({
allowedRegistries: ['other.example.com'],
trust: value.trust,
fetch: value.fetch,
});
await assert.rejects(
authority.stage(value.lock),
/registry is not explicitly allowed/,
);
assert.equal(value.calls(), 0);
});
test('rejects content that no longer matches the immutable OCI manifest digest', async () => {
const value = fixture();
const packageUrl = `https://${REGISTRY}/v2/${REPOSITORY}/manifests/sha256:${
value.lock.source.locator.split('@sha256:')[1]
}`;
value.routes.set(packageUrl, Buffer.from('{}'));
const authority = new ClusterPluginPackageOciStageAuthority({
allowedRegistries: [REGISTRY],
trust: value.trust,
fetch: value.fetch,
});
await assert.rejects(
authority.stage(value.lock),
InvalidPluginPackageInstallError,
);
});
test('injects one exact-registry credential without changing redirect policy', async () => {
const value = fixture();
const registries = [];
const authority = new ClusterPluginPackageOciStageAuthority({
allowedRegistries: [REGISTRY],
trust: value.trust,
credentialProvider: {
authorizationFor(registry) {
registries.push(registry);
return 'Bearer exact-registry-token';
},
},
fetch(url, init) {
assert.equal(init.headers.authorization, 'Bearer exact-registry-token');
assert.equal(init.redirect, 'error');
return value.fetch(url, init);
},
});
await authority.stage(value.lock);
assert.deepEqual(registries, Array(6).fill(REGISTRY));
assert.equal(value.calls(), 6);
});
test('never queries credentials before the source registry passes its allowlist', async () => {
const value = fixture();
let credentialCalls = 0;
const authority = new ClusterPluginPackageOciStageAuthority({
allowedRegistries: ['other.example.com'],
trust: value.trust,
credentialProvider: {
authorizationFor() {
credentialCalls += 1;
return 'Bearer must-not-be-used';
},
},
fetch: value.fetch,
});
await assert.rejects(
authority.stage(value.lock),
InvalidPluginPackageInstallError,
);
assert.equal(credentialCalls, 0);
assert.equal(value.calls(), 0);
});
test('maps malformed credential-provider output to unavailable before fetch', async () => {
const value = fixture();
const authority = new ClusterPluginPackageOciStageAuthority({
allowedRegistries: [REGISTRY],
trust: value.trust,
credentialProvider: {
authorizationFor() {
return 'Bearer secret\r\nx-overreach: true';
},
},
fetch: value.fetch,
});
await assert.rejects(
authority.stage(value.lock),
PluginPackageInstallUnavailableError,
);
assert.equal(value.calls(), 0);
});
@@ -0,0 +1,233 @@
const assert = require('node:assert/strict');
const { generateKeyPairSync } = require('node:crypto');
const { test } = require('node:test');
const {
consumeApprovalRequest,
createApprovalRequest,
decideApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
createApprovedActionExecution,
claimApprovedActionExecution,
startApprovedActionExecution,
} = require('@qinglong/runtime-core/approved-action-execution');
const {
createPluginPackagePublisherRevocationProposal,
} = require('@qinglong/runtime-core/plugin-package-publisher-revocation-proposal');
const {
createPluginPackagePublisherTrustSnapshot,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
const {
ClusterPluginPackagePublisherRevocationApprovedActionHandler,
} = require('../dist/plugin-package/publisher/pluginPackagePublisherRevocationApprovedAction');
const REQUESTER = Object.freeze({ type: 'user', id: 'usr_owner' });
const REVIEWER = Object.freeze({ type: 'user', id: 'usr_security' });
const SYSTEM = Object.freeze({ type: 'system', id: 'cluster_package_executor' });
const FENCE = Object.freeze({ projectVersion: 4, bindingVersion: 7 });
function proposal() {
const { publicKey } = generateKeyPairSync('ed25519');
return createPluginPackagePublisherRevocationProposal({
actionRef: 'publisher-revoke:publisher-a.example:key-a',
authorityProjectId: 'cluster-trust-authority',
trustAuthorityId: 'cluster',
trustGeneration: 1,
trustSnapshot: createPluginPackagePublisherTrustSnapshot([
{
publisher: 'publisher-a.example',
keyId: 'key-a',
publicKeyPem: publicKey.export({ type: 'spki', format: 'pem' }),
notBeforeMs: 1,
notAfterMs: 10_000,
},
]),
publisher: 'publisher-a.example',
keyId: 'key-a',
authorizationMode: 'dual_control',
reasonCode: 'suspected_key_compromise',
proposedBy: REQUESTER,
proposerAssurance: 'multi_factor',
proposalFence: FENCE,
createdAtMs: 5,
});
}
function dispatch(candidate) {
const action = {
permission: candidate.permission,
actionType: candidate.actionType,
actionRef: candidate.actionRef,
actionDigest: candidate.actionDigest,
previewDigest: candidate.previewDigest,
};
const pending = createApprovalRequest({
id: 'approval-publisher-revoke',
projectId: candidate.projectId,
action,
risk: 'critical',
decisionMode: 'separation_of_duty',
requestedBy: REQUESTER,
requestedAtMs: 10,
expiresAtMs: 1_000,
requestFence: FENCE,
});
const approved = decideApprovalRequest(pending, {
expectedVersion: 1,
decisionId: 'decision-publisher-revoke',
decision: 'approved',
reasonCode: 'reviewed',
principal: {
subject: REVIEWER,
authenticationId: 'auth-reviewer',
authenticatedAtMs: 15,
expiresAtMs: 500,
assurance: 'multi_factor',
},
decidedAtMs: 20,
authorizationFence: FENCE,
});
return consumeApprovalRequest(approved, {
expectedVersion: 2,
consumptionId: 'consume-publisher-revoke',
dispatchId: 'dispatch-publisher-revoke',
action,
requestedBy: REQUESTER,
consumedBy: SYSTEM,
consumedAtMs: 30,
authorizationFence: FENCE,
}).dispatch;
}
function execution(approvedDispatch) {
const baseline = createApprovedActionExecution(approvedDispatch, 5);
const claimed = claimApprovedActionExecution(baseline, {
owner: 'publisher-executor',
leaseToken: 'lease-publisher-revoke',
nowMs: 31,
leaseDurationMs: 1_000,
});
assert.equal(claimed.status, 'leased');
return startApprovedActionExecution(
{ dispatch: approvedDispatch, execution: claimed },
{
dispatchId: approvedDispatch.id,
approvalRequestId: approvedDispatch.approvalRequestId,
actionDigest: approvedDispatch.action.actionDigest,
owner: 'publisher-executor',
leaseToken: 'lease-publisher-revoke',
expectedVersion: claimed.version,
startedAtMs: 40,
},
);
}
test('inspects authority and executes the exact approved revocation receipt', async () => {
const candidate = proposal();
const approvedDispatch = dispatch(candidate);
const started = execution(approvedDispatch);
const receipts = [];
const handler =
new ClusterPluginPackagePublisherRevocationApprovedActionHandler(
{
async findProposalByActionRef() {
return candidate;
},
async createProposal() {
throw new Error('must not create');
},
},
{
async run(receipt) {
receipts.push(receipt);
return {
safeToAdmit: true,
receiptDigest: receipt.receiptDigest,
impactDigest: 'a'.repeat(64),
};
},
},
);
assert.deepEqual(await handler.inspect(approvedDispatch), {
status: 'ready',
actionDigest: candidate.actionDigest,
});
const result = await handler.execute({
dispatch: approvedDispatch,
execution: started,
idempotencyKey: 'publisher-revoke-attempt',
fence: {
owner: started.leaseOwner,
leaseToken: started.leaseToken,
version: started.version,
},
});
assert.deepEqual(result, {
outcome: 'succeeded',
resultCode: 'publisher_revocation_converged',
resultDigest: 'a'.repeat(64),
});
assert.equal(receipts.length, 1);
assert.equal(receipts[0].mutationId, approvedDispatch.id);
assert.equal(receipts[0].revokedAtMs, started.startedAtMs);
});
test('blocks missing proposals and incomplete quarantine convergence', async () => {
const candidate = proposal();
const approvedDispatch = dispatch(candidate);
const missing =
new ClusterPluginPackagePublisherRevocationApprovedActionHandler(
{
async findProposalByActionRef() {
return null;
},
async createProposal() {
throw new Error('must not create');
},
},
{ async run() { throw new Error('must not run'); } },
);
assert.deepEqual(await missing.inspect(approvedDispatch), {
status: 'blocked',
resultCode: 'publisher_revocation_proposal_missing',
});
const started = execution(approvedDispatch);
const incomplete =
new ClusterPluginPackagePublisherRevocationApprovedActionHandler(
{
async findProposalByActionRef() {
return candidate;
},
async createProposal() {
throw new Error('must not create');
},
},
{
async run(receipt) {
return {
safeToAdmit: false,
receiptDigest: receipt.receiptDigest,
impactDigest: 'b'.repeat(64),
};
},
},
);
assert.deepEqual(
await incomplete.execute({
dispatch: approvedDispatch,
execution: started,
idempotencyKey: 'publisher-revoke-attempt',
fence: {
owner: started.leaseOwner,
leaseToken: started.leaseToken,
version: started.version,
},
}),
{
outcome: 'indeterminate',
resultCode: 'publisher_revocation_convergence_incomplete',
},
);
});
@@ -0,0 +1,302 @@
const assert = require('node:assert/strict');
const { generateKeyPairSync } = require('node:crypto');
const { test } = require('node:test');
const {
consumeApprovalRequest,
createApprovalRequest,
decideApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
claimApprovedActionExecution,
createApprovedActionExecution,
startApprovedActionExecution,
} = require('@qinglong/runtime-core/approved-action-execution');
const {
createPluginPackagePublisherTrustSnapshot,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
const {
PluginPackagePublisherTrustTransitionConflictError,
createPluginPackagePublisherTrustTransitionProposal,
resolvePluginPackagePublisherTrustTransitionProposal,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust-transition-proposal');
const {
ClusterPluginPackagePublisherTrustTransitionApprovedActionHandler,
} = require('../dist/plugin-package/publisher/pluginPackagePublisherTrustTransitionApprovedAction');
const REQUESTER = Object.freeze({ type: 'user', id: 'usr_owner' });
const REVIEWER = Object.freeze({ type: 'user', id: 'usr_security' });
const SYSTEM = Object.freeze({
type: 'system',
id: 'cluster_package_executor',
});
const FENCE = Object.freeze({ projectVersion: 4, bindingVersion: 7 });
function definition(keyId, notAfterMs = 20_000) {
const { publicKey } = generateKeyPairSync('ed25519');
return {
publisher: 'publisher-a.example',
keyId,
publicKeyPem: publicKey.export({ type: 'spki', format: 'pem' }),
notBeforeMs: 1,
notAfterMs,
};
}
function authority(mode) {
const oldDefinition = definition('key-old');
const newDefinition = definition('key-new', 30_000);
const currentSnapshot =
createPluginPackagePublisherTrustSnapshot(
mode === 'overlap_add'
? [oldDefinition]
: [oldDefinition, newDefinition],
);
return createPluginPackagePublisherTrustTransitionProposal({
actionRef:
mode === 'overlap_add'
? 'publisher-overlap:publisher-a.example:key-new'
: 'publisher-retire:publisher-a.example:key-old',
authorityProjectId: 'cluster-trust-authority',
trustAuthorityId: 'cluster',
trustGeneration: 3,
mode,
trustSnapshot: currentSnapshot,
...(mode === 'overlap_add'
? {
materialSnapshot:
createPluginPackagePublisherTrustSnapshot([
oldDefinition,
newDefinition,
]),
}
: {}),
publisher: 'publisher-a.example',
keyId: mode === 'overlap_add' ? 'key-new' : 'key-old',
proposedBy: REQUESTER,
proposerAssurance: 'multi_factor',
proposalFence: FENCE,
createdAtMs: 100,
});
}
function approvedDispatch(candidate) {
const action = {
permission: candidate.permission,
actionType: candidate.actionType,
actionRef: candidate.actionRef,
actionDigest: candidate.actionDigest,
previewDigest: candidate.previewDigest,
};
const pending = createApprovalRequest({
id: `approval-${candidate.actionInput.mode}`,
projectId: candidate.projectId,
action,
risk: 'critical',
decisionMode: 'separation_of_duty',
requestedBy: REQUESTER,
requestedAtMs: 100,
expiresAtMs: 1_000,
requestFence: FENCE,
});
const approved = decideApprovalRequest(pending, {
expectedVersion: 1,
decisionId: `decision-${candidate.actionInput.mode}`,
decision: 'approved',
reasonCode: 'reviewed',
principal: {
subject: REVIEWER,
authenticationId: 'auth-reviewer',
authenticatedAtMs: 101,
expiresAtMs: 900,
assurance: 'multi_factor',
},
decidedAtMs: 110,
authorizationFence: FENCE,
});
return consumeApprovalRequest(approved, {
expectedVersion: 2,
consumptionId: `consume-${candidate.actionInput.mode}`,
dispatchId: `dispatch-${candidate.actionInput.mode}`,
action,
requestedBy: REQUESTER,
consumedBy: SYSTEM,
consumedAtMs: 120,
authorizationFence: FENCE,
}).dispatch;
}
function startedExecution(dispatch) {
const baseline = createApprovedActionExecution(dispatch, 5);
const claimed = claimApprovedActionExecution(baseline, {
owner: 'publisher-trust-executor',
leaseToken: `lease-${dispatch.id}`,
nowMs: 121,
leaseDurationMs: 1_000,
});
assert.equal(claimed.status, 'leased');
return startApprovedActionExecution(
{ dispatch, execution: claimed },
{
dispatchId: dispatch.id,
approvalRequestId: dispatch.approvalRequestId,
actionDigest: dispatch.action.actionDigest,
owner: 'publisher-trust-executor',
leaseToken: `lease-${dispatch.id}`,
expectedVersion: claimed.version,
startedAtMs: 130,
},
);
}
function proposalRepository(candidate) {
return {
async findProposalByActionRef() {
return candidate;
},
async createProposal() {
throw new Error('must not create');
},
};
}
test('executes exact overlap-add and safe-retire Approved Actions', async () => {
for (const mode of ['overlap_add', 'safe_retire']) {
const created = authority(mode);
const dispatch = approvedDispatch(created.proposal);
const execution = startedExecution(dispatch);
const calls = [];
const handler =
new ClusterPluginPackagePublisherTrustTransitionApprovedActionHandler(
mode,
proposalRepository(created.proposal),
{
async applyApprovedTransition(input) {
calls.push(input);
const receipt =
resolvePluginPackagePublisherTrustTransitionProposal(
created.proposal,
input.dispatch,
input.executedAtMs,
mode === 'safe_retire' ? 0 : null,
);
return {
status: 'created',
receipt,
head: {
generation: receipt.currentGeneration,
effectiveTrustDigest: receipt.currentTrustDigest,
},
};
},
},
);
assert.deepEqual(await handler.inspect(dispatch), {
status: 'ready',
actionDigest: created.proposal.actionDigest,
});
assert.deepEqual(
await handler.execute({
dispatch,
execution,
idempotencyKey: dispatch.id,
fence: {
owner: execution.leaseOwner,
leaseToken: execution.leaseToken,
version: execution.version,
},
}),
{
outcome: 'succeeded',
resultCode:
mode === 'overlap_add'
? 'publisher_trust_overlap_added'
: 'publisher_trust_key_retired',
resultDigest:
resolvePluginPackagePublisherTrustTransitionProposal(
created.proposal,
dispatch,
execution.startedAtMs,
mode === 'safe_retire' ? 0 : null,
).receiptDigest,
},
);
assert.deepEqual(calls, [
{
dispatch,
executedAtMs: execution.startedAtMs,
},
]);
}
});
test('blocks missing or mismatched proposals and rejects stale execution fences', async () => {
const created = authority('overlap_add');
const dispatch = approvedDispatch(created.proposal);
const missing =
new ClusterPluginPackagePublisherTrustTransitionApprovedActionHandler(
'overlap_add',
proposalRepository(null),
{ async applyApprovedTransition() { throw new Error('must not run'); } },
);
assert.deepEqual(await missing.inspect(dispatch), {
status: 'blocked',
resultCode: 'publisher_trust_transition_proposal_missing',
});
const wrongMode =
new ClusterPluginPackagePublisherTrustTransitionApprovedActionHandler(
'safe_retire',
proposalRepository(created.proposal),
{ async applyApprovedTransition() { throw new Error('must not run'); } },
);
assert.deepEqual(await wrongMode.inspect(dispatch), {
status: 'blocked',
resultCode: 'publisher_trust_transition_proposal_rejected',
});
const execution = startedExecution(dispatch);
const conflict =
new ClusterPluginPackagePublisherTrustTransitionApprovedActionHandler(
'overlap_add',
proposalRepository(created.proposal),
{
async applyApprovedTransition() {
throw new PluginPackagePublisherTrustTransitionConflictError();
},
},
);
assert.deepEqual(
await conflict.execute({
dispatch,
execution,
idempotencyKey: dispatch.id,
fence: {
owner: execution.leaseOwner,
leaseToken: execution.leaseToken,
version: execution.version + 1,
},
}),
{
outcome: 'failed',
resultCode: 'publisher_trust_transition_execution_rejected',
},
);
assert.deepEqual(
await conflict.execute({
dispatch,
execution,
idempotencyKey: dispatch.id,
fence: {
owner: execution.leaseOwner,
leaseToken: execution.leaseToken,
version: execution.version,
},
}),
{
outcome: 'failed',
resultCode: 'publisher_trust_transition_conflict',
},
);
});
@@ -0,0 +1,340 @@
const assert = require('node:assert/strict');
const { generateKeyPairSync } = require('node:crypto');
const { test } = require('node:test');
const {
createApprovalRequest,
decideApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
createPluginPackagePublisherTrustSnapshot,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust');
const {
createPluginPackagePublisherTrustTransitionProposal,
} = require('@qinglong/runtime-core/plugin-package-publisher-trust-transition-proposal');
const {
ClusterPluginPackageManagementTransportRequestError,
createClusterPluginPackageManagementTransport,
} = require('@qinglong/cluster-admin/plugin-package-management-transport');
const NOW = 1_000;
const OWNER = Object.freeze({ type: 'user', id: 'cluster-owner' });
const REVIEWER = Object.freeze({ type: 'user', id: 'cluster-reviewer' });
const FENCE = Object.freeze({ projectVersion: 4, bindingVersion: 7 });
function principal() {
return {
subject: REVIEWER,
authenticationId: 'reviewer-session',
authenticatedAtMs: NOW - 100,
expiresAtMs: NOW + 1_000,
assurance: 'multi_factor',
};
}
function authentication() {
return {
async authenticate() {
return principal();
},
};
}
function definition(keyId) {
const { publicKey } = generateKeyPairSync('ed25519');
return {
publisher: 'publisher-a.example',
keyId,
publicKeyPem: publicKey.export({ type: 'spki', format: 'pem' }),
notBeforeMs: 1,
notAfterMs: 20_000,
};
}
const oldDefinition = definition('key-old');
const newDefinition = definition('key-new');
const currentSnapshot = createPluginPackagePublisherTrustSnapshot([
oldDefinition,
]);
const materialSnapshot = createPluginPackagePublisherTrustSnapshot([
oldDefinition,
newDefinition,
]);
function transition(createdAtMs = NOW) {
return createPluginPackagePublisherTrustTransitionProposal({
actionRef: 'publisher-overlap:publisher-a.example:key-new',
authorityProjectId: 'cluster-trust-authority',
trustAuthorityId: 'cluster',
trustGeneration: 4,
mode: 'overlap_add',
trustSnapshot: currentSnapshot,
materialSnapshot,
publisher: 'publisher-a.example',
keyId: 'key-new',
proposedBy: OWNER,
proposerAssurance: 'multi_factor',
proposalFence: FENCE,
createdAtMs,
}).proposal;
}
function approval(candidate = transition(), overrides = {}) {
return createApprovalRequest({
id: 'approval-publisher-overlap',
projectId: candidate.projectId,
action: {
permission: candidate.permission,
actionType: candidate.actionType,
actionRef: candidate.actionRef,
actionDigest: candidate.actionDigest,
previewDigest: candidate.previewDigest,
},
risk: 'critical',
decisionMode: 'separation_of_duty',
requestedBy: OWNER,
requestedAtMs: candidate.createdAtMs,
expiresAtMs: candidate.createdAtMs + 10_000,
requestFence: FENCE,
...overrides,
});
}
function command(operation, request = {}) {
const common = {
actionRef: 'publisher-overlap:publisher-a.example:key-new',
approvalRequestId: 'approval-publisher-overlap',
};
if (operation.endsWith('.propose')) {
return {
schemaVersion: 1,
operation,
request: {
...common,
proposalAuditEventId: 'proposal-overlap-audit',
approvalAuditEventId: 'approval-overlap-audit',
mode: 'overlap_add',
publisher: 'publisher-a.example',
keyId: 'key-new',
...request,
},
};
}
if (operation.endsWith('.decide')) {
return {
schemaVersion: 1,
operation,
request: {
...common,
expectedVersion: 1,
decisionId: 'decision-publisher-overlap',
auditEventId: 'decision-overlap-audit',
decision: 'approved',
reasonCode: 'reviewed',
...request,
},
};
}
return {
schemaVersion: 1,
operation,
request: {
...common,
inspectionId: 'inspection-publisher-overlap',
...request,
},
};
}
function services(state = { proposal: null, approvalRequest: null }) {
const calls = {
decide: [],
proposeTransition: [],
inspectTransition: [],
inspectTransitionAuthorized: [],
};
const installService = {
async propose() {
throw new Error('install proposal must not run');
},
async decide(request) {
calls.decide.push(request);
return {
status: 'decided',
request: decideApprovalRequest(state.approvalRequest, {
expectedVersion: request.expectedVersion,
decisionId: request.decisionId,
decision: request.decision,
reasonCode: request.reasonCode,
principal: request.principal,
decidedAtMs: request.decidedAtMs,
authorizationFence: FENCE,
}),
};
},
async inspect() {
return { proposal: null, approvalRequest: null };
},
async inspectAuthorized() {
return { proposal: null, approvalRequest: null };
},
async inspectInstallationAuthorized() {
return null;
},
async listInstallationsAuthorized() {
return { items: [], truncated: false };
},
};
const publisherTrust = {
async propose() {
throw new Error('revocation proposal must not run');
},
async inspect() {
return { proposal: null, approvalRequest: null };
},
async inspectAuthorized() {
return { proposal: null, approvalRequest: null };
},
async proposeTransition(request) {
calls.proposeTransition.push(request);
const candidate = transition(request.requestedAtMs);
return {
proposalStatus: 'created',
approvalStatus: 'created',
proposal: candidate,
approvalRequest: approval(candidate),
};
},
async inspectTransition(actionRef, approvalRequestId) {
calls.inspectTransition.push({ actionRef, approvalRequestId });
return state;
},
async inspectTransitionAuthorized(request) {
calls.inspectTransitionAuthorized.push(request);
return state;
},
};
return { calls, installService, publisherTrust };
}
test('routes derived-only trust overlap proposal without key material', async () => {
const fixture = services();
const transport = createClusterPluginPackageManagementTransport({
service: fixture.installService,
publisherTrust: fixture.publisherTrust,
now: () => NOW,
});
const proposed = command('plugin-package.publisher-trust-transition.propose');
const result = await transport.execute(proposed, authentication());
assert.equal(result.operation, proposed.operation);
assert.equal(fixture.calls.proposeTransition.length, 1);
assert.deepEqual(Object.keys(proposed.request).sort(), [
'actionRef',
'approvalAuditEventId',
'approvalRequestId',
'keyId',
'mode',
'proposalAuditEventId',
'publisher',
]);
assert.equal(JSON.stringify(proposed).includes('PUBLIC KEY'), false);
assert.equal(JSON.stringify(result).includes('PUBLIC KEY'), false);
assert.deepEqual(Object.keys(result.proposal).sort(), [
'actionDigest',
'actionRef',
'createdAtMs',
'currentTrustDigest',
'keyId',
'mode',
'previewDigest',
'previousTrustDigest',
'projectId',
'proposalDigest',
'publisher',
'trustAuthorityId',
'trustGeneration',
]);
assert.deepEqual(fixture.calls.proposeTransition[0].principal, principal());
assert.equal(fixture.calls.proposeTransition[0].requestedAtMs, NOW);
});
test('requires exact separation-of-duty authority before transition decision', async () => {
const candidate = transition();
const pending = approval(candidate);
const fixture = services({
proposal: candidate,
approvalRequest: pending,
});
const transport = createClusterPluginPackageManagementTransport({
service: fixture.installService,
publisherTrust: fixture.publisherTrust,
now: () => NOW,
});
const result = await transport.execute(
command('plugin-package.publisher-trust-transition.decide'),
authentication(),
);
assert.equal(result.status, 'decided');
assert.equal(result.approval.decisionMode, 'separation_of_duty');
assert.equal(fixture.calls.decide.length, 1);
assert.deepEqual(fixture.calls.decide[0].principal, principal());
const invalid = services({
proposal: candidate,
approvalRequest: approval(candidate, {
decisionMode: 'human_confirmation',
}),
});
const invalidTransport = createClusterPluginPackageManagementTransport({
service: invalid.installService,
publisherTrust: invalid.publisherTrust,
now: () => NOW,
});
await assert.rejects(
invalidTransport.execute(
command('plugin-package.publisher-trust-transition.decide'),
authentication(),
),
ClusterPluginPackageManagementTransportRequestError,
);
assert.deepEqual(invalid.calls.decide, []);
});
test('uses scoped authorized inspection and rejects client key material', async () => {
const candidate = transition();
const state = {
proposal: candidate,
approvalRequest: approval(candidate),
};
const fixture = services(state);
const transport = createClusterPluginPackageManagementTransport({
service: fixture.installService,
publisherTrust: fixture.publisherTrust,
now: () => NOW,
});
const result = await transport.execute(
command('plugin-package.publisher-trust-transition.inspect'),
authentication(),
);
assert.equal(
result.operation,
'plugin-package.publisher-trust-transition.inspect',
);
assert.deepEqual(fixture.calls.inspectTransition, []);
assert.equal(fixture.calls.inspectTransitionAuthorized.length, 1);
assert.deepEqual(
fixture.calls.inspectTransitionAuthorized[0].principal,
principal(),
);
await assert.rejects(
transport.execute(
command('plugin-package.publisher-trust-transition.propose', {
publicKeyPem: 'client-controlled',
}),
authentication(),
),
ClusterPluginPackageManagementTransportRequestError,
);
});
@@ -0,0 +1,147 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
InvalidPluginPackageQuarantineError,
createPluginPackageQuarantineEvent,
createPluginPackageWithdrawalReceipt,
} = require('@qinglong/runtime-core/plugin-package-quarantine');
const {
CLUSTER_PLUGIN_PACKAGE_QUARANTINE_BATCH_LIMIT,
createClusterPluginPackageQuarantineService,
} = require('@qinglong/cluster-admin/plugin-package-quarantine');
function event(index, overrides = {}) {
return createPluginPackageQuarantineEvent({
mutationId: `quarantine-cluster-${index}`,
revocationReceiptDigest: 'a'.repeat(64),
impactDigest: 'b'.repeat(64),
target: {
projectId: 'project-cluster',
packageName: `package-${index}`,
installationId: `install-${index}`,
lockDigest: String(index % 10).repeat(64),
installState: 'queued',
installVersion: 1,
installRecordDigest: 'c'.repeat(64),
activeLockDigest: null,
...overrides.target,
},
proposer: { type: 'user', id: 'owner-a' },
confirmer: { type: 'user', id: 'owner-b' },
authorizationMode: 'dual_control',
reasonCode: 'confirmed_key_compromise',
occurredAtMs: 1_000 + index,
...overrides,
});
}
function receipt(value) {
return createPluginPackageWithdrawalReceipt({
eventDigest: value.eventDigest,
target: value.target,
capability: {
status: 'not_active',
taskWithdrawals: [],
previousActiveVectorDigest: null,
currentActiveVectorDigest: null,
currentToolSnapshotDigest: null,
retainedSourceCount: 0,
},
committedAtMs: value.occurredAtMs,
});
}
test('runs one bounded batch with authorization rechecked inside each repository transaction', async () => {
const events = [event(1), event(2)];
const calls = [];
const service = createClusterPluginPackageQuarantineService({
async findTargetsByLockDigest() {
return [];
},
async findByEventDigest() {
return null;
},
async quarantine(value, confirmAuthorization) {
calls.push(`begin:${value.eventDigest}`);
await confirmAuthorization();
calls.push(`write:${value.eventDigest}`);
await confirmAuthorization();
return {
status: 'created',
receipt: receipt(value),
};
},
});
const authorization = [];
const results = await service.quarantine(events, (value) => {
authorization.push(value.eventDigest);
});
assert.deepEqual(
results.map(({ status, eventDigest }) => ({ status, eventDigest })),
events.map(({ eventDigest }) => ({ status: 'created', eventDigest })),
);
assert.deepEqual(authorization, [
events[0].eventDigest,
events[0].eventDigest,
events[1].eventDigest,
events[1].eventDigest,
]);
assert.deepEqual(calls, [
`begin:${events[0].eventDigest}`,
`write:${events[0].eventDigest}`,
`begin:${events[1].eventDigest}`,
`write:${events[1].eventDigest}`,
]);
});
test('rejects duplicate targets, sparse input and batches above the hard limit before storage', async () => {
let calls = 0;
const service = createClusterPluginPackageQuarantineService({
async findTargetsByLockDigest() {
return [];
},
async findByEventDigest() {
return null;
},
async quarantine() {
calls += 1;
throw new Error('must not write');
},
});
const first = event(3);
const duplicateTarget = event(4, { target: first.target });
await assert.rejects(
service.quarantine([first, duplicateTarget], () => {}),
InvalidPluginPackageQuarantineError,
);
const sparse = [first, event(5)];
delete sparse[0];
await assert.rejects(
service.quarantine(sparse, () => {}),
InvalidPluginPackageQuarantineError,
);
await assert.rejects(
service.quarantine(
Array.from(
{ length: CLUSTER_PLUGIN_PACKAGE_QUARANTINE_BATCH_LIMIT + 1 },
(_, index) => event(index + 10),
),
() => {},
),
InvalidPluginPackageQuarantineError,
);
assert.equal(calls, 0);
});
test('publishes quarantine authority only through its explicit subpath', () => {
assert.equal(
require('@qinglong/cluster-admin').createClusterPluginPackageQuarantineService,
undefined,
);
assert.equal(
typeof require('@qinglong/cluster-admin/plugin-package-quarantine')
.createClusterPluginPackageQuarantineService,
'function',
);
});
@@ -0,0 +1,350 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
postgresqlControlSchemaContract,
postgresqlMainMigrationManifest,
} = require('@qinglong/cluster-postgres');
const {
MAX_PLUGIN_PACKAGE_RECOVERY_PAGES,
} = require('@qinglong/runtime-core/plugin-package-recovery');
const {
recoverClusterPluginPackages,
} = require('@qinglong/cluster-admin/plugin-package-recovery');
function history() {
return postgresqlMainMigrationManifest.migrations.map((migration, index) => ({
streamId: postgresqlMainMigrationManifest.id,
dialect: postgresqlMainMigrationManifest.dialect,
migrationId: migration.id,
checksum: migration.checksum,
appliedAtMs: index + 1,
}));
}
function executorPrivileges() {
const insertable = new Set([
'security_audit_events',
'plugin_package_installs',
'plugin_package_install_heads',
'plugin_package_install_mutations',
'approved_action_dispatches',
'approved_action_executions',
'plugin_package_admission_receipts',
'plugin_package_materialized_revisions',
'plugin_package_automation_publications',
'plugin_package_automation_publication_heads',
'project_tool_definition_snapshots',
'project_tool_definition_snapshot_sources',
'plugin_package_publisher_provenance',
'plugin_package_publisher_revocation_receipts',
'plugin_package_publisher_revocation_impacts',
'plugin_package_publisher_revocation_impact_items',
'plugin_package_publisher_trust_snapshots',
'plugin_package_publisher_trust_transition_receipts',
'plugin_package_lifecycle_plans',
]);
const readable = new Set([
'schema_migrations',
'schema_capabilities',
'projects',
'project_role_bindings',
'approval_requests',
'plugin_package_install_proposals',
'plugin_package_task_ownerships',
'plugin_package_task_reconciliations',
'plugin_package_task_reconciliation_items',
'plugin_package_quarantine_events',
'plugin_package_withdrawal_receipts',
'plugin_package_withdrawal_tasks',
'plugin_package_publisher_provenance',
'plugin_package_publisher_revocation_receipts',
'plugin_package_publisher_revocation_impacts',
'plugin_package_publisher_revocation_impact_items',
'plugin_package_publisher_trust_snapshots',
'plugin_package_publisher_trust_heads',
'plugin_package_publisher_revocation_proposals',
'plugin_package_publisher_trust_transition_proposals',
'plugin_package_publisher_trust_transition_receipts',
'plugin_package_lifecycle_events',
'plugin_package_lifecycle_heads',
'plugin_package_lifecycle_receipts',
'plugin_package_lifecycle_tasks',
'task_definitions',
'task_definition_revisions',
'task_execution_revisions',
...insertable,
]);
return postgresqlControlSchemaContract.tables.map(({ name: tableName }) => ({
tableName,
selectAllowed: readable.has(tableName),
insertAllowed: insertable.has(tableName),
updateAllowed: [
'plugin_package_installs',
'plugin_package_install_heads',
'approval_requests',
'approved_action_executions',
'plugin_package_publisher_trust_heads',
'plugin_package_automation_publication_heads',
].includes(tableName),
deleteAllowed: false,
isOwner: false,
}));
}
function database(serverVersionNum = '160014') {
const contract = postgresqlControlSchemaContract;
let closes = 0;
const resource = {
pool: {
async query(text) {
if (text.includes("current_setting('server_version_num')")) {
return {
rows: [
{
serverVersionNum,
currentUser: 'ql3_package_executor',
inRecovery: false,
transactionReadOnly: 'off',
},
],
};
}
if (text.includes('FROM "ql3"."schema_migrations"')) {
return { rows: history() };
}
if (text.includes('FROM "ql3"."schema_capabilities"')) {
return {
rows: [
{
contractName: contract.contractName,
contractVersion: contract.contractVersion,
migrationId: contract.migrationId,
capabilities: contract.capabilities,
},
],
};
}
if (text.includes('FROM pg_class tables')) {
return {
rows: contract.tables.flatMap((table) =>
table.columns.map((columnName) => ({
tableName: table.name,
columnName,
})),
),
};
}
if (text.includes('FROM pg_indexes')) {
return { rows: contract.indexes.map((indexName) => ({ indexName })) };
}
if (text.includes('FROM pg_constraint')) {
return {
rows: [
...contract.checks.map((constraintName) => ({
constraintName,
constraintType: 'check',
})),
...contract.foreignKeys.map((constraintName) => ({
constraintName,
constraintType: 'foreign_key',
})),
],
};
}
if (text.includes('FROM pg_proc routines')) {
return {
rows: contract.functions.map((definition) => ({
functionName: definition.name,
identityArguments: definition.identityArguments,
owner: definition.owner,
securityDefiner: definition.securityDefiner,
volatility: definition.volatility,
configuration: definition.configuration,
publicExecute: false,
})),
};
}
if (text.includes('FROM pg_catalog.pg_roles')) {
return {
rows: [
{
canLogin: true,
superuser: false,
createDatabase: false,
createRole: false,
replication: false,
bypassRowLevelSecurity: false,
databaseConnect: true,
},
],
};
}
if (text.includes('has_schema_privilege')) {
return { rows: [{ schemaUsage: true, schemaCreate: false }] };
}
if (text.includes('has_table_privilege')) {
return { rows: executorPrivileges() };
}
if (text.includes('has_function_privilege')) {
return {
rows: contract.functions.map(({ name: functionName }) => ({
functionName,
executeAllowed: ![
'enforce_plugin_package_stage_provenance',
'plugin_package_automation_start_allowed',
'plugin_package_run_start_allowed',
'plugin_package_tool_start_allowed',
'plugin_package_workflow_admission_snapshot',
'plugin_package_workflow_task_attempt_snapshot',
].includes(functionName),
isOwner: false,
})),
};
}
if (text.includes('plugin_package_install_heads')) {
return { rows: [] };
}
throw new Error(`unexpected query: ${text}`);
},
async connect() {
throw new Error('empty recovery must not open a transaction');
},
},
async close() {
closes += 1;
},
};
return { resource, closes: () => closes };
}
function options(db, overrides = {}) {
const unavailable = async () => {
throw new Error('empty recovery must not use Kubernetes');
};
return {
openDatabase: async () => db.resource,
api: {
readNamespacedConfigMap: unavailable,
createNamespacedConfigMap: unavailable,
replaceNamespacedConfigMap: unavailable,
},
stageAuthority: {
stage: unavailable,
verify: unavailable,
publisherEvidence: unavailable,
},
resourceByteSource: { open: unavailable },
clusterIdentity: 'cluster-test',
trustAuthorityId: 'cluster',
namespace: 'qinglong',
now: () => 1_000,
...overrides,
};
}
test('rejects invalid bounds before opening PostgreSQL', async () => {
const db = database();
let opens = 0;
await assert.rejects(
recoverClusterPluginPackages(
options(db, {
maxPages: MAX_PLUGIN_PACKAGE_RECOVERY_PAGES + 1,
openDatabase: async () => {
opens += 1;
return db.resource;
},
}),
),
/configuration is invalid/,
);
assert.equal(opens, 0);
assert.equal(db.closes(), 0);
});
test('proves an empty executor queue and closes PostgreSQL before returning', async () => {
const db = database();
const result = await recoverClusterPluginPackages(options(db));
assert.equal(result.evidence.currentUser, 'ql3_package_executor');
assert.deepEqual(result.provenanceRecovery, {
pages: 1,
scanned: 0,
created: 0,
existing: 0,
remaining: false,
safeToAdmit: true,
});
assert.deepEqual(result.recovery, {
pages: 1,
scanned: 0,
settled: 0,
retry: 0,
manualRequired: 0,
superseded: 0,
remaining: false,
safeToAdmit: true,
});
assert.deepEqual(result.taskPublicationRecovery, {
pages: 1,
scanned: 0,
settled: 0,
retry: 0,
manualRequired: 0,
superseded: 0,
remaining: false,
safeToAdmit: true,
});
assert.deepEqual(result.automationPublicationRecovery, {
pages: 1,
scanned: 0,
settled: 0,
retry: 0,
manualRequired: 0,
superseded: 0,
remaining: false,
safeToAdmit: true,
});
assert.deepEqual(result.toolSnapshotRecovery, {
pages: 1,
scanned: 0,
settled: 0,
retry: 0,
manualRequired: 0,
remaining: false,
safeToAdmit: true,
});
assert.equal(db.closes(), 1);
assert.equal(
require('@qinglong/cluster-admin').recoverClusterPluginPackages,
undefined,
);
});
test('resolves a database-bound stage authority after readiness', async () => {
const db = database();
let factories = 0;
const configured = options(db);
delete configured.stageAuthority;
configured.stageAuthorityFactory = async (pool) => {
factories += 1;
assert.equal(pool, db.resource.pool);
return {
stage: configured.resourceByteSource.open,
verify: configured.resourceByteSource.open,
publisherEvidence: configured.resourceByteSource.open,
};
};
await recoverClusterPluginPackages(configured);
assert.equal(factories, 1);
assert.equal(db.closes(), 1);
});
test('closes PostgreSQL when admin readiness fails', async () => {
const db = database('150018');
await assert.rejects(
recoverClusterPluginPackages(options(db)),
(error) => error.code === 'server_version_unsupported',
);
assert.equal(db.closes(), 1);
});
@@ -0,0 +1,260 @@
const assert = require('node:assert/strict');
const fs = require('node:fs');
const { generateKeyPairSync } = require('node:crypto');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const {
ClusterPluginPackageRecoveryProcessConfigError,
loadClusterPluginPackageRegistryCredentialFile,
loadClusterPluginPackagePublisherTrustFile,
loadClusterPluginPackageRecoveryProcessConfig,
runClusterPluginPackageRecoveryProcess,
} = require('@qinglong/cluster-admin/plugin-package-recovery-process');
function environment(overrides = {}) {
return {
QL3_CLUSTER_IDENTITY: 'cluster-production-001',
QL3_KUBERNETES_NAMESPACE: 'qinglong3-system',
QL3_PLUGIN_PACKAGE_OCI_REGISTRIES: 'ghcr.io,registry.example.com:5443',
QL3_PLUGIN_PACKAGE_PUBLISHER_TRUST_FILE: '/trust/publishers.json',
QL3_PLUGIN_PACKAGE_OCI_TIMEOUT_MS: '12000',
QL3_PLUGIN_PACKAGE_RECOVERY_PAGE_SIZE: '8',
QL3_PLUGIN_PACKAGE_RECOVERY_MAX_PAGES: '4',
QL3_POSTGRES_PACKAGE_EXECUTOR_URL:
'postgresql://ql3_package_executor:secret@postgres/qinglong',
QL3_POSTGRES_TLS_MODE: 'disable',
QL3_POSTGRES_ALLOW_INSECURE: 'true',
...overrides,
};
}
test('loads one explicit Package-executor-only recovery process configuration', () => {
const config = loadClusterPluginPackageRecoveryProcessConfig(environment());
assert.equal(config.clusterIdentity, 'cluster-production-001');
assert.equal(config.namespace, 'qinglong3-system');
assert.equal(config.publisherTrustAuthorityId, 'cluster');
assert.deepEqual(config.allowedRegistries, [
'ghcr.io',
'registry.example.com:5443',
]);
assert.equal(config.requestTimeoutMs, 12_000);
assert.equal(config.pageSize, 8);
assert.equal(config.maxPages, 4);
assert.equal(config.database.pool.maxConnections, 1);
assert.equal(config.database.connection.tls.mode, 'disable');
assert.equal('QL3_POSTGRES_RUNTIME_URL' in config.database.connection, false);
});
test('rejects runtime credentials, duplicate registries and implicit insecure PostgreSQL', async () => {
for (const invalid of [
environment({
QL3_POSTGRES_PACKAGE_EXECUTOR_URL: undefined,
QL3_POSTGRES_RUNTIME_URL:
'postgresql://ql3_runtime:secret@postgres/qinglong',
}),
environment({
QL3_PLUGIN_PACKAGE_OCI_REGISTRIES: 'ghcr.io,ghcr.io',
}),
environment({
QL3_POSTGRES_ALLOW_INSECURE: undefined,
}),
]) {
assert.throws(
() => loadClusterPluginPackageRecoveryProcessConfig(invalid),
ClusterPluginPackageRecoveryProcessConfigError,
);
}
let touched = false;
await assert.rejects(
runClusterPluginPackageRecoveryProcess({
environment: environment({
QL3_PLUGIN_PACKAGE_RECOVERY_MAX_PAGES: '65',
}),
openDatabase: async () => {
touched = true;
throw new Error('must not open');
},
api: {},
stageAuthority: {},
}),
ClusterPluginPackageRecoveryProcessConfigError,
);
assert.equal(touched, false);
});
test('loads a bounded read-only publisher trust file', (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-publisher-trust-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const filePath = path.join(root, 'publishers.json');
const { publicKey } = generateKeyPairSync('ed25519');
fs.writeFileSync(
filePath,
JSON.stringify({
schema: 'qinglong/plugin-package-publisher-trust@v1',
keys: [
{
publisher: 'packages.example.com',
keyId: 'release-2026',
publicKeyPem: publicKey.export({ format: 'pem', type: 'spki' }),
notBeforeMs: 100,
notAfterMs: 10_000,
},
],
}),
{ mode: 0o444 },
);
fs.chmodSync(filePath, 0o444);
const trust = loadClusterPluginPackagePublisherTrustFile(filePath);
assert.equal(trust.size, 1);
fs.chmodSync(filePath, 0o666);
assert.throws(
() => loadClusterPluginPackagePublisherTrustFile(filePath),
/read-only regular file/,
);
});
test('keeps process authority off the cluster-admin root', () => {
const root = require('@qinglong/cluster-admin');
const manifest = require('../package.json');
assert.equal(root.runClusterPluginPackageRecoveryProcess, undefined);
assert.equal(
manifest.bin['ql3-plugin-package-recover'],
'dist/plugin-package/recovery/pluginPackageRecoveryCli.js',
);
});
test('binds an optional registry credential file without enabling ambient credentials', () => {
const publicConfig = loadClusterPluginPackageRecoveryProcessConfig(
environment(),
);
assert.equal(publicConfig.registryCredentialFile, undefined);
const privateConfig = loadClusterPluginPackageRecoveryProcessConfig(
environment({
QL3_PLUGIN_PACKAGE_REGISTRY_CREDENTIAL_FILE:
'/var/run/secrets/qinglong3/registry/credentials.json',
}),
);
assert.equal(
privateConfig.registryCredentialFile,
'/var/run/secrets/qinglong3/registry/credentials.json',
);
assert.throws(
() =>
loadClusterPluginPackageRecoveryProcessConfig(
environment({
QL3_PLUGIN_PACKAGE_REGISTRY_CREDENTIAL_FILE: 'credentials.json',
}),
),
ClusterPluginPackageRecoveryProcessConfigError,
);
});
test('loads exact basic and bearer registry credentials and disposes retained bytes', (t) => {
const root = fs.mkdtempSync(
path.join(os.tmpdir(), 'ql3-registry-credentials-'),
);
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const filePath = path.join(root, 'credentials.json');
fs.writeFileSync(
filePath,
JSON.stringify({
schema: 'qinglong/plugin-package-registry-credentials@v1',
credentials: [
{
registry: 'ghcr.io',
scheme: 'bearer',
token: 'token.value-1',
},
{
registry: 'registry.example.com:5443',
scheme: 'basic',
username: 'ql3-admin',
password: 'private-password',
},
],
}),
{ mode: 0o440 },
);
fs.chmodSync(filePath, 0o440);
const credentials = loadClusterPluginPackageRegistryCredentialFile(filePath, [
'ghcr.io',
'registry.example.com:5443',
]);
assert.equal(credentials.authorizationFor('ghcr.io'), 'Bearer token.value-1');
assert.equal(
credentials.authorizationFor('registry.example.com:5443'),
`Basic ${Buffer.from('ql3-admin:private-password').toString('base64')}`,
);
assert.equal(credentials.authorizationFor('registry.example.com'), undefined);
credentials.dispose();
assert.equal(credentials.authorizationFor('ghcr.io'), undefined);
credentials.dispose();
});
test('rejects overbroad, duplicate and publicly readable registry credentials', (t) => {
const root = fs.mkdtempSync(
path.join(os.tmpdir(), 'ql3-invalid-registry-credentials-'),
);
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const filePath = path.join(root, 'credentials.json');
const write = (credentials, mode = 0o440) => {
if (fs.existsSync(filePath)) fs.chmodSync(filePath, 0o600);
fs.writeFileSync(
filePath,
JSON.stringify({
schema: 'qinglong/plugin-package-registry-credentials@v1',
credentials,
}),
{ mode },
);
fs.chmodSync(filePath, mode);
};
write([
{
registry: 'not-allowed.example.com',
scheme: 'bearer',
token: 'private-token',
},
]);
assert.throws(
() => loadClusterPluginPackageRegistryCredentialFile(filePath, ['ghcr.io']),
/binding is invalid/,
);
write([
{
registry: 'ghcr.io',
scheme: 'bearer',
token: 'private-token',
},
{
registry: 'ghcr.io',
scheme: 'basic',
username: 'owner',
password: 'private-password',
},
]);
assert.throws(
() => loadClusterPluginPackageRegistryCredentialFile(filePath, ['ghcr.io']),
/binding is invalid/,
);
write(
[
{
registry: 'ghcr.io',
scheme: 'bearer',
token: 'private-token',
},
],
0o444,
);
assert.throws(
() => loadClusterPluginPackageRegistryCredentialFile(filePath, ['ghcr.io']),
/private regular file/,
);
});
@@ -0,0 +1,336 @@
const assert = require('node:assert/strict');
const {
chmodSync,
mkdtempSync,
rmSync,
symlinkSync,
writeFileSync,
} = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const { createHash, generateKeyPairSync, sign } = require('node:crypto');
const { afterEach, test } = require('node:test');
const {
createPluginPackagePromptOutputArtifact,
} = require('../../ql3-ai/dist/prompt-output/pluginPackagePromptOutputArtifact.js');
const {
createPluginPackagePromptOutputExternalCustodyReceipt,
} = require('../../ql3-ai/dist/prompt-output/custody/pluginPackagePromptOutputExternalCustody.js');
const {
createPluginPackagePromptOutputExternalCustodyBundle,
} = require('../../ql3-ai/dist/prompt-output/custody/pluginPackagePromptOutputExternalCustodyBundle.js');
const {
createPluginPackagePromptOutputExternalRecoveryAuthorization,
} = require('../../ql3-ai/dist/prompt-output/custody/pluginPackagePromptOutputExternalRecoveryAuthorization.js');
const {
pluginPackagePromptOutputKeyRotationMaterialProof,
} = require('../../ql3-ai/dist/prompt-output/key-management/pluginPackagePromptOutputKeyRotation.js');
const {
disposeClusterPromptOutputExternalRecoveryInput,
readClusterPromptOutputExternalRecoveryCommand,
readClusterPromptOutputExternalRecoveryInput,
} = require('../dist/prompt-output/external-recovery/promptOutputExternalRecoveryInput.js');
const {
ClusterPromptOutputExternalRecoveryVerifierConfigError,
runClusterPromptOutputExternalRecoveryVerifier,
} = require('../dist/prompt-output/external-recovery/promptOutputExternalRecoveryVerifier.js');
const temporaryDirectories = [];
afterEach(() => {
for (const directory of temporaryDirectories.splice(0)) {
rmSync(directory, { recursive: true, force: true });
}
});
function privateFile(directory, name, value, mode = 0o440) {
const target = path.join(directory, name);
writeFileSync(target, value, { mode: 0o600 });
chmodSync(target, mode);
return target;
}
function fixture() {
const directory = mkdtempSync(path.join(os.tmpdir(), 'ql3-recovery-'));
temporaryDirectories.push(directory);
const now = Date.now();
const material = Buffer.alloc(32, 0x5c);
const wrapped = Buffer.from('provider-wrapped-key-material');
const custodyKeys = generateKeyPairSync('ed25519');
const approverAKeys = generateKeyPairSync('ed25519');
const approverBKeys = generateKeyPairSync('ed25519');
const catalogDigest = '1'.repeat(64);
const artifact = createPluginPackagePromptOutputArtifact(
{
projectId: 'project-offline-recovery',
runId: 'run-offline-recovery',
stepRunId: 'step-offline-recovery',
invocationId: 'invocation-offline-recovery',
requestedBy: { type: 'user', id: 'requester-user' },
result: {
provider: 'openai-compatible',
model: 'bounded-model',
text: 'private offline recovery output',
finishReason: 'stop',
usage: {
inputTokens: 4,
outputTokens: 5,
totalTokens: 9,
costMicros: 10,
},
},
retentionPolicy: {
revision: 'offline-recovery-v1',
retentionMs: 86_400_000,
},
keyId: 'offline-recovery-key',
key: Buffer.from(material),
sealedAtMs: now - 60_000,
},
() => Buffer.alloc(12, 0x39),
);
const receipt = createPluginPackagePromptOutputExternalCustodyReceipt(
{
custodyId: 'offline-custody',
keyId: 'offline-recovery-key',
materialProof: pluginPackagePromptOutputKeyRotationMaterialProof(
'offline-recovery-key',
material,
),
sourceGeneration: 4,
sourceCatalogDigest: catalogDigest,
wrappingProvider: 'external-kms',
wrappingKeyRefDigest: '2'.repeat(64),
wrappedMaterialDigest: createHash('sha256').update(wrapped).digest('hex'),
wrappedMaterialBytes: wrapped.length,
createdAtMs: now - 120_000,
},
{
publicKey: custodyKeys.publicKey,
sign: (message) => sign(null, message, custodyKeys.privateKey),
},
);
const custodyBundle = createPluginPackagePromptOutputExternalCustodyBundle(
receipt,
custodyKeys.publicKey,
wrapped,
);
const approvalSigner = (userId, authenticationId, keys, approvedAtMs) => ({
userId,
authenticationId,
authenticatedAtMs: approvedAtMs - 1_000,
approvedAtMs,
publicKey: keys.publicKey,
sign: (message) => sign(null, message, keys.privateKey),
});
const authorization =
createPluginPackagePromptOutputExternalRecoveryAuthorization(
{
recoveryId: 'offline-recovery-001',
requestId: 'offline-request-001',
custodyId: receipt.custodyId,
custodyReceiptDigest: receipt.receiptDigest,
keyId: receipt.keyId,
artifactId: artifact.artifactId,
artifactDigest: artifact.artifactDigest,
policyDigest: '3'.repeat(64),
requestedBy: {
userId: 'requester-user',
authenticationId: 'requester-auth',
authenticatedAtMs: now - 4_000,
},
requestedAtMs: now - 3_000,
expiresAtMs: now + 10 * 60_000,
},
[
approvalSigner(
'reviewer-a',
'reviewer-a-auth',
approverAKeys,
now - 2_000,
),
approvalSigner(
'reviewer-b',
'reviewer-b-auth',
approverBKeys,
now - 1_000,
),
],
);
const files = {
authorizationFile: privateFile(
directory,
'authorization.json',
JSON.stringify(authorization),
),
custodyBundleFile: privateFile(
directory,
'custody-bundle.json',
JSON.stringify(custodyBundle),
),
recoveredMaterialFile: privateFile(directory, 'material.bin', material),
durableKeyFactFile: privateFile(
directory,
'durable-fact.json',
JSON.stringify({
keyId: receipt.keyId,
materialProof: receipt.materialProof,
catalogDigest: receipt.sourceCatalogDigest,
}),
),
artifactFile: privateFile(
directory,
'artifact.json',
JSON.stringify(artifact),
),
custodyPublicKeyFile: privateFile(
directory,
'custody-public.pem',
custodyKeys.publicKey.export({ format: 'pem', type: 'spki' }),
),
approverAFile: privateFile(
directory,
'reviewer-a-public.pem',
approverAKeys.publicKey.export({ format: 'pem', type: 'spki' }),
),
approverBFile: privateFile(
directory,
'reviewer-b-public.pem',
approverBKeys.publicKey.export({ format: 'pem', type: 'spki' }),
),
};
const command = {
schemaVersion: 1,
operation: 'cluster.prompt-output-key.verify-recovery',
authorizationFile: files.authorizationFile,
custodyBundleFile: files.custodyBundleFile,
recoveredMaterialFile: files.recoveredMaterialFile,
durableKeyFactFile: files.durableKeyFactFile,
artifactFile: files.artifactFile,
custodyPublicKeyFile: files.custodyPublicKeyFile,
approverPublicKeyFiles: [
{ userId: 'reviewer-b', filePath: files.approverBFile },
{ userId: 'reviewer-a', filePath: files.approverAFile },
],
};
const commandFile = privateFile(
directory,
'command.json',
JSON.stringify(command),
0o444,
);
return { directory, now, files, command, commandFile };
}
test('reads one private recovery workspace and emits a content-free proof', () => {
const value = fixture();
const command = readClusterPromptOutputExternalRecoveryCommand(
value.commandFile,
);
assert.deepEqual(
command.approverPublicKeyFiles.map(({ userId }) => userId),
['reviewer-a', 'reviewer-b'],
);
const input = readClusterPromptOutputExternalRecoveryInput(command);
const proof = runClusterPromptOutputExternalRecoveryVerifier(
input,
value.now + 3_000,
);
assert.equal(proof.schema.includes('authorized'), true);
assert.equal(proof.authorizationDigest.length, 64);
assert.equal(JSON.stringify(proof).includes('private offline'), false);
disposeClusterPromptOutputExternalRecoveryInput(input);
assert.equal(
input.material.every((byte) => byte === 0),
true,
);
assert.equal(
input.wrappedMaterial.every((byte) => byte === 0),
true,
);
});
test('CLI verifies the workspace without database, Kubernetes or KMS input', () => {
const value = fixture();
const result = spawnSync(
process.execPath,
[
path.resolve(
__dirname,
'../dist/prompt-output/external-recovery/promptOutputExternalRecoveryCli.js',
),
'run',
'--command-file',
value.commandFile,
],
{ encoding: 'utf8' },
);
assert.equal(result.status, 0, result.stderr);
const report = JSON.parse(result.stdout);
assert.equal(report.event, 'recovery_verified');
assert.equal(report.proofDigest.length, 64);
assert.equal(result.stdout.includes('private offline'), false);
});
test('rejects writable or symlinked recovery material and custody bundles', () => {
for (const mutation of [
(value) => chmodSync(value.files.recoveredMaterialFile, 0o640),
(value) => chmodSync(value.files.custodyBundleFile, 0o640),
(value) => {
const link = path.join(value.directory, 'material-link.bin');
symlinkSync(value.files.recoveredMaterialFile, link);
const command = {
...value.command,
recoveredMaterialFile: link,
};
chmodSync(value.commandFile, 0o600);
writeFileSync(value.commandFile, JSON.stringify(command));
chmodSync(value.commandFile, 0o444);
},
(value) => {
const link = path.join(value.directory, 'custody-bundle-link.json');
symlinkSync(value.files.custodyBundleFile, link);
const command = {
...value.command,
custodyBundleFile: link,
};
chmodSync(value.commandFile, 0o600);
writeFileSync(value.commandFile, JSON.stringify(command));
chmodSync(value.commandFile, 0o444);
},
]) {
const value = fixture();
mutation(value);
const command = readClusterPromptOutputExternalRecoveryCommand(
value.commandFile,
);
assert.throws(() => readClusterPromptOutputExternalRecoveryInput(command));
}
});
test('rejects authorization and Artifact drift with a content-free error', () => {
const value = fixture();
const command = readClusterPromptOutputExternalRecoveryCommand(
value.commandFile,
);
const input = readClusterPromptOutputExternalRecoveryInput(command);
try {
assert.throws(
() =>
runClusterPromptOutputExternalRecoveryVerifier(
{
...input,
artifact: {
...input.artifact,
artifactDigest: '4'.repeat(64),
},
},
value.now + 3_000,
),
ClusterPromptOutputExternalRecoveryVerifierConfigError,
);
} finally {
disposeClusterPromptOutputExternalRecoveryInput(input);
}
});
@@ -0,0 +1,106 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
pluginPackagePromptOutputArtifactRetentionPolicyDigest,
} = require('@qinglong/ai/plugin-package-prompt-output-artifact');
const {
ClusterPromptOutputGcProcessConfigError,
runClusterPromptOutputGcProcess,
} = require('../dist/prompt-output/retention/promptOutputGcProcess');
function catalog() {
const policy = { revision: 'retention-v1', retentionMs: 3_600_000 };
return {
schemaVersion: 1,
policies: [
{
projectId: 'project-a',
policy,
policyDigest:
pluginPackagePromptOutputArtifactRetentionPolicyDigest(policy),
},
],
};
}
test('runs one bounded page through the maintenance-only PostgreSQL authority', async () => {
const statements = [];
let closed = false;
const pool = {
async query(sql, values) {
statements.push({ sql, values });
if (sql.includes('current_user AS "currentUser"')) {
return {
rows: [
{
currentUser: 'ql3_ai_maintenance',
maintenanceAuthority: true,
schemaAuthority: true,
artifactDeleteOnly: true,
tombstoneAppendOnly: true,
keyRetirementAppendOnly: true,
keyRotationAppendOnly: true,
terminalEvidenceReadOnly: true,
},
],
rowCount: 1,
};
}
if (sql.includes('clock_timestamp()')) {
return { rows: [{ observedAtMs: '1000' }], rowCount: 1 };
}
if (sql.includes('SELECT artifact_id AS "artifactId"')) {
assert.deepEqual(values, [1000, 5]);
return { rows: [], rowCount: 0 };
}
throw new Error(`unexpected query: ${sql}`);
},
async connect() {
throw new Error('empty scan must not acquire a transaction client');
},
};
const result = await runClusterPromptOutputGcProcess({
database: { connection: { host: 'postgres.example.test' } },
retentionPolicyCatalog: catalog(),
limit: 4,
async openDatabase() {
return {
pool,
async close() {
closed = true;
},
};
},
});
assert.deepEqual(
{
scanned: result.scanned,
tombstoned: result.tombstoned,
skipped: result.skipped,
hasMore: result.hasMore,
},
{ scanned: 0, tombstoned: 0, skipped: 0, hasMore: false },
);
assert.equal(result.readiness.maintenanceAuthority, true);
assert.equal(statements.length, 3);
assert.equal(closed, true);
});
test('rejects a rewritten policy before opening PostgreSQL', async () => {
let opened = false;
const invalid = catalog();
invalid.policies[0].policyDigest = '0'.repeat(64);
await assert.rejects(
runClusterPromptOutputGcProcess({
database: { connection: { host: 'postgres.example.test' } },
retentionPolicyCatalog: invalid,
async openDatabase() {
opened = true;
throw new Error('must not open');
},
}),
ClusterPromptOutputGcProcessConfigError,
);
assert.equal(opened, false);
});
@@ -0,0 +1,76 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const cli = path.resolve(
__dirname,
'../dist/prompt-output/key-management/promptOutputKeyRetirementCli.js',
);
test('Cluster Prompt output key retirement CLI exposes one command-file-only interface', (t) => {
const help = spawnSync(process.execPath, [cli, '--help'], {
encoding: 'utf8',
});
assert.equal(help.status, 0);
assert.match(
help.stdout,
/^Usage: ql3-prompt-output-key-retire run --command-file /,
);
assert.equal(help.stderr, '');
const usage = spawnSync(process.execPath, [cli, 'run'], {
encoding: 'utf8',
});
assert.equal(usage.status, 64);
const usageFact = JSON.parse(usage.stderr);
assert.equal(
usageFact.code,
'QL3_PROMPT_OUTPUT_KEY_RETIREMENT_CLI_USAGE_INVALID',
);
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-key-retire-cli-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const commandFile = path.join(root, 'command.json');
fs.writeFileSync(
commandFile,
JSON.stringify({
schemaVersion: 1,
operation: 'cluster.prompt-output-key.retire',
kubernetes: {
namespace: 'qinglong',
secretName: 'ql3-prompt-output-keyring',
expectedSecretUid: 'uid-keyring-1',
dataKey: 'keyring.json',
},
request: {
keyId: 'cluster-key-old',
retirementId: 'retirement-1',
requestId: 'request-1',
mutationId: 'mutation-1',
widened: true,
},
}),
{ mode: 0o444 },
);
const rejected = spawnSync(
process.execPath,
[cli, 'run', '--command-file', commandFile],
{ encoding: 'utf8' },
);
assert.equal(rejected.status, 1);
assert.equal(rejected.stdout, '');
assert.equal(rejected.stderr.includes(commandFile), false);
assert.equal(rejected.stderr.includes('cluster-key-old'), false);
const failure = JSON.parse(rejected.stderr);
assert.equal(failure.event, 'key_retirement_failed');
assert.equal(failure.name, 'TypeError');
const manifest = require('../package.json');
assert.equal(
manifest.bin['ql3-prompt-output-key-retire'],
'dist/prompt-output/key-management/promptOutputKeyRetirementCli.js',
);
});
@@ -0,0 +1,229 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
pluginPackagePromptOutputKeyRetirementAbsenceProof,
} = require('@qinglong/ai/plugin-package-prompt-output-key-retirement');
const {
ClusterPromptOutputKeyRetirementProcessConfigError,
runClusterPromptOutputKeyRetirementProcess,
} = require('../dist/prompt-output/key-management/promptOutputKeyRetirementProcess');
function preparationRow(value) {
return {
keyId: value.keyId,
retirementId: value.retirementId,
requestId: value.requestId,
mutationId: value.mutationId,
catalogDigest: value.catalogDigest,
materialProof: value.materialProof,
preparedAtMs: String(value.preparedAtMs),
preparationDigest: value.preparationDigest,
preparationJson: value,
};
}
function completionRow(value) {
return {
keyId: value.keyId,
retirementId: value.retirementId,
requestId: value.requestId,
mutationId: value.mutationId,
preparationDigest: value.preparationDigest,
retiredCatalogDigest: value.retiredCatalogDigest,
absenceProof: value.absenceProof,
completedAtMs: String(value.completedAtMs),
completionDigest: value.completionDigest,
completionJson: value,
};
}
function databaseHarness() {
let preparation = null;
let completion = null;
let closeCount = 0;
const statements = [];
const query = async (sql, values = []) => {
statements.push({ sql, values });
if (sql.includes('current_user AS "currentUser"')) {
return {
rows: [
{
currentUser: 'ql3_ai_maintenance',
maintenanceAuthority: true,
schemaAuthority: true,
artifactDeleteOnly: true,
tombstoneAppendOnly: true,
keyRetirementAppendOnly: true,
keyRotationAppendOnly: true,
terminalEvidenceReadOnly: true,
},
],
rowCount: 1,
};
}
if (sql.includes('key_retirement_preparations') && sql.includes('SELECT')) {
return {
rows:
preparation && preparation.keyId === values[0]
? [preparationRow(preparation)]
: [],
rowCount: preparation ? 1 : 0,
};
}
if (sql.includes('key_retirement_completions') && sql.includes('SELECT')) {
return {
rows:
completion && completion.keyId === values[0]
? [completionRow(completion)]
: [],
rowCount: completion ? 1 : 0,
};
}
if (sql.includes('count(*)::text AS count')) {
return { rows: [{ count: '0' }], rowCount: 1 };
}
if (
sql.includes('INSERT INTO') &&
sql.includes('key_retirement_preparations')
) {
preparation = JSON.parse(values[8]);
return { rows: [], rowCount: 1 };
}
if (
sql.includes('INSERT INTO') &&
sql.includes('key_retirement_completions')
) {
completion = JSON.parse(values[9]);
return { rows: [], rowCount: 1 };
}
if (
sql.startsWith('BEGIN') ||
sql.startsWith('SET LOCAL') ||
sql.startsWith('SELECT pg_advisory_xact_lock') ||
sql === 'COMMIT' ||
sql === 'ROLLBACK'
) {
return { rows: [], rowCount: 0 };
}
throw new Error(`unexpected query: ${sql}`);
};
const client = { query, release() {} };
const pool = {
query,
async connect() {
return client;
},
};
return {
statements,
get closeCount() {
return closeCount;
},
async openDatabase() {
return {
pool,
async close() {
closeCount += 1;
},
};
},
};
}
test('retires one inactive Cluster key through maintenance PostgreSQL and injected material authority', async () => {
const database = databaseHarness();
const keyId = 'cluster-key-old';
const initialCatalogDigest = '1'.repeat(64);
const materialProof = '2'.repeat(64);
const retiredCatalogDigest = '3'.repeat(64);
let retiredPreparation = null;
const materials = {
async inspect() {
if (retiredPreparation) {
return {
state: 'absent',
keyId,
catalogDigest: retiredCatalogDigest,
absenceProof: pluginPackagePromptOutputKeyRetirementAbsenceProof(
retiredPreparation,
retiredCatalogDigest,
),
};
}
return {
state: 'inactive',
keyId,
catalogDigest: initialCatalogDigest,
materialProof,
};
},
async retire({ preparation }) {
retiredPreparation = preparation;
return this.inspect(keyId);
},
};
const options = {
database: { connection: { host: 'postgres.example.test' } },
request: {
keyId,
retirementId: 'retirement-1',
requestId: 'request-1',
mutationId: 'mutation-1',
},
materials,
openDatabase: database.openDatabase,
};
const completed = await runClusterPromptOutputKeyRetirementProcess(options);
assert.equal(completed.status, 'completed');
assert.equal(completed.keyId, keyId);
assert.equal(completed.readiness.keyRetirementAppendOnly, true);
assert.equal(completed.readiness.keyRotationAppendOnly, true);
assert.match(completed.preparationDigest, /^[0-9a-f]{64}$/);
assert.match(completed.completionDigest, /^[0-9a-f]{64}$/);
const replay = await runClusterPromptOutputKeyRetirementProcess(options);
assert.equal(replay.status, 'existing');
assert.equal(replay.preparationDigest, completed.preparationDigest);
assert.equal(replay.completionDigest, completed.completionDigest);
assert.equal(database.closeCount, 2);
assert.equal(
database.statements.filter(
({ sql }) =>
sql.includes('INSERT INTO') &&
sql.includes('key_retirement_preparations'),
).length,
1,
);
assert.equal(
database.statements.filter(
({ sql }) =>
sql.includes('INSERT INTO') &&
sql.includes('key_retirement_completions'),
).length,
1,
);
});
test('rejects an invalid retirement request before opening PostgreSQL', async () => {
let opened = false;
await assert.rejects(
runClusterPromptOutputKeyRetirementProcess({
database: { connection: { host: 'postgres.example.test' } },
request: {
keyId: '../invalid',
retirementId: 'retirement-1',
requestId: 'request-1',
mutationId: 'mutation-1',
},
materials: { async inspect() {}, async retire() {} },
async openDatabase() {
opened = true;
throw new Error('must not open');
},
}),
ClusterPromptOutputKeyRetirementProcessConfigError,
);
assert.equal(opened, false);
});
@@ -0,0 +1,64 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const cli = path.resolve(
__dirname,
'../dist/prompt-output/key-management/promptOutputKeyRotationCli.js',
);
test('Cluster Prompt output key rotation CLI is command-file-only and content-free', (t) => {
const help = spawnSync(process.execPath, [cli, '--help'], {
encoding: 'utf8',
});
assert.equal(help.status, 0);
assert.match(
help.stdout,
/^Usage: ql3-prompt-output-key-rotate run --command-file /,
);
assert.equal(help.stderr, '');
const usage = spawnSync(process.execPath, [cli, 'run'], {
encoding: 'utf8',
});
assert.equal(usage.status, 64);
assert.equal(
JSON.parse(usage.stderr).code,
'QL3_PROMPT_OUTPUT_KEY_ROTATION_CLI_USAGE_INVALID',
);
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-key-rotate-cli-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const commandFile = path.join(root, 'command.json');
fs.writeFileSync(
commandFile,
JSON.stringify({
schemaVersion: 1,
operation: 'cluster.prompt-output-key.rotate',
kubernetes: {},
stagedMaterialFile: '/private/material.bin',
request: {},
widened: true,
}),
{ mode: 0o444 },
);
const rejected = spawnSync(
process.execPath,
[cli, 'run', '--command-file', commandFile],
{ encoding: 'utf8' },
);
assert.equal(rejected.status, 1);
assert.equal(rejected.stdout, '');
assert.equal(rejected.stderr.includes(commandFile), false);
assert.equal(rejected.stderr.includes('/private/material.bin'), false);
assert.equal(JSON.parse(rejected.stderr).event, 'key_rotation_failed');
const manifest = require('../package.json');
assert.equal(
manifest.bin['ql3-prompt-output-key-rotate'],
'dist/prompt-output/key-management/promptOutputKeyRotationCli.js',
);
});
@@ -0,0 +1,82 @@
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const {
readClusterPromptOutputKeyRotationCommand,
readClusterPromptOutputKeyRotationMaterial,
} = require('../dist/prompt-output/key-management/promptOutputKeyRotationInput.js');
function command(materialFile) {
return {
schemaVersion: 1,
operation: 'cluster.prompt-output-key.rotate',
kubernetes: {
namespace: 'qinglong',
secretName: 'ql3-prompt-output-keyring',
expectedSecretUid: 'uid-keyring-1',
dataKey: 'keyring.json',
},
stagedMaterialFile: materialFile,
request: {
rotationId: 'rotation-1',
requestId: 'request-1',
mutationId: 'mutation-1',
expectedActiveKeyId: 'cluster-key-current',
expectedCatalogDigest: '1'.repeat(64),
newKeyId: 'cluster-key-next',
},
};
}
test('reads one exact command and an owned 0440 staged material copy', (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-key-rotate-input-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const materialFile = path.join(root, 'material.bin');
fs.writeFileSync(materialFile, Buffer.alloc(32, 0x44), { mode: 0o440 });
fs.chmodSync(materialFile, 0o440);
const commandFile = path.join(root, 'command.json');
fs.writeFileSync(commandFile, JSON.stringify(command(materialFile)), {
mode: 0o444,
});
assert.deepEqual(
readClusterPromptOutputKeyRotationCommand(commandFile),
command(materialFile),
);
const material = readClusterPromptOutputKeyRotationMaterial(materialFile);
assert.deepEqual(material, Buffer.alloc(32, 0x44));
material.fill(0);
assert.deepEqual(fs.readFileSync(materialFile), Buffer.alloc(32, 0x44));
});
test('rejects widened commands and unsafe staged material files', (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-key-rotate-input-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const materialFile = path.join(root, 'material.bin');
fs.writeFileSync(materialFile, Buffer.alloc(32, 0x55), { mode: 0o640 });
fs.chmodSync(materialFile, 0o640);
assert.throws(
() => readClusterPromptOutputKeyRotationMaterial(materialFile),
/unavailable/,
);
const target = path.join(root, 'target.bin');
fs.writeFileSync(target, Buffer.alloc(32, 0x66), { mode: 0o440 });
const symlink = path.join(root, 'link.bin');
fs.symlinkSync(target, symlink);
assert.throws(() => readClusterPromptOutputKeyRotationMaterial(symlink));
const commandFile = path.join(root, 'command.json');
fs.writeFileSync(
commandFile,
JSON.stringify({ ...command(target), widened: true }),
{ mode: 0o444 },
);
assert.throws(
() => readClusterPromptOutputKeyRotationCommand(commandFile),
/shape/,
);
});
@@ -0,0 +1,183 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ClusterPromptOutputKeyRotationProcessConfigError,
runClusterPromptOutputKeyRotationProcess,
} = require('../dist/prompt-output/key-management/promptOutputKeyRotationProcess.js');
const {
pluginPackagePromptOutputKeyRotationMaterialProof,
} = require('@qinglong/ai/plugin-package-prompt-output-key-rotation');
function request() {
return {
rotationId: 'rotation-process-1',
requestId: 'rotation-process-request-1',
mutationId: 'rotation-process-mutation-1',
expectedSecretUid: 'rotation-secret-uid-1',
expectedActiveKeyId: 'key-before',
expectedCatalogDigest: '1'.repeat(64),
newKeyId: 'key-after',
};
}
function fakeDatabase(material) {
let preparation;
let completion;
let closed = 0;
const client = {
async query(sql, parameters = []) {
if (sql.includes('current_user AS "currentUser"')) {
return {
rows: [
{
currentUser: 'ql3_ai_maintenance',
maintenanceAuthority: true,
schemaAuthority: true,
artifactDeleteOnly: true,
tombstoneAppendOnly: true,
keyRetirementAppendOnly: true,
keyRotationAppendOnly: true,
terminalEvidenceReadOnly: true,
},
],
};
}
if (sql.includes('key_rotation_preparations') && sql.includes('SELECT')) {
return {
rows: preparation
? [
{
rotationId: preparation.rotationId,
requestId: preparation.requestId,
mutationId: preparation.mutationId,
expectedSecretUid: preparation.expectedSecretUid,
expectedActiveKeyId: preparation.expectedActiveKeyId,
expectedCatalogDigest: preparation.expectedCatalogDigest,
newKeyId: preparation.newKeyId,
materialProof: preparation.materialProof,
preparedAtMs: String(preparation.preparedAtMs),
preparationDigest: preparation.preparationDigest,
preparationJson: preparation,
},
]
: [],
};
}
if (sql.includes('key_rotation_completions') && sql.includes('SELECT')) {
return {
rows: completion
? [
{
rotationId: completion.rotationId,
requestId: completion.requestId,
mutationId: completion.mutationId,
preparationDigest: completion.preparationDigest,
generation: String(completion.generation),
previousActiveKeyId: completion.previousActiveKeyId,
activeKeyId: completion.activeKeyId,
catalogDigest: completion.catalogDigest,
materialProof: completion.materialProof,
completedAtMs: String(completion.completedAtMs),
completionDigest: completion.completionDigest,
completionJson: completion,
},
]
: [],
};
}
if (
sql.includes('INSERT INTO') &&
sql.includes('key_rotation_preparations')
) {
preparation = JSON.parse(parameters[10]);
}
if (
sql.includes('INSERT INTO') &&
sql.includes('key_rotation_completions')
) {
completion = JSON.parse(parameters[11]);
}
return { rows: [], rowCount: 1 };
},
release() {},
};
const pool = {
async connect() {
return client;
},
async query(sql, parameters) {
return client.query(sql, parameters);
},
};
return {
database: {
pool,
async close() {
closed += 1;
},
},
get closed() {
return closed;
},
get completion() {
return completion;
},
materials: {
async rotate(command) {
return {
generation: 2,
previousActiveKeyId: command.expectedActiveKeyId,
activeKeyId: command.newKeyId,
catalogDigest: '2'.repeat(64),
materialProof: pluginPackagePromptOutputKeyRotationMaterialProof(
command.newKeyId,
material,
),
};
},
},
};
}
test('Cluster rotation process commits content-free prepare and completion', async () => {
const material = Buffer.alloc(32, 0x71);
const fixture = fakeDatabase(material);
const result = await runClusterPromptOutputKeyRotationProcess({
database: { connection: { connectionString: 'postgres://unused' } },
request: request(),
material,
materials: fixture.materials,
openDatabase: async () => fixture.database,
});
assert.equal(result.status, 'completed');
assert.equal(result.generation, 2);
assert.equal(result.activeKeyId, 'key-after');
assert.equal(result.readiness.keyRotationAppendOnly, true);
assert.equal(fixture.closed, 1);
const durable = JSON.stringify(fixture.completion);
assert.equal(durable.includes(material.toString('base64url')), false);
assert.equal(durable.includes(material.toString('hex')), false);
});
test('Cluster rotation process rejects invalid material before opening PostgreSQL', async () => {
let opened = false;
await assert.rejects(
runClusterPromptOutputKeyRotationProcess({
database: { connection: { connectionString: 'postgres://unused' } },
request: request(),
material: Buffer.alloc(31),
materials: {
async rotate() {
throw new Error('unreachable');
},
},
openDatabase: async () => {
opened = true;
throw new Error('must not open');
},
}),
ClusterPromptOutputKeyRotationProcessConfigError,
);
assert.equal(opened, false);
});
@@ -0,0 +1,347 @@
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const {
canonicalPluginPackagePromptOutputKeyringManifest,
pluginPackagePromptOutputKeyringCatalogDigest,
} = require('@qinglong/ai/plugin-package-prompt-output-keyring-manifest');
const {
PluginPackagePromptOutputKeyRetirementConflictError,
PluginPackagePromptOutputKeyRetirementUnavailableError,
createPluginPackagePromptOutputKeyRetirementPreparation,
} = require('@qinglong/ai/plugin-package-prompt-output-key-retirement');
const {
createPluginPackagePromptOutputProjectedKeyring,
} = require('@qinglong/ai/plugin-package-prompt-output-projected-keyring');
const {
ClusterPromptOutputKubernetesSecretKeyring,
clusterPromptOutputKubernetesSecretKeyringMetadata,
} = require('../dist/prompt-output/key-management/promptOutputKubernetesSecretKeyring');
function copy(value) {
return structuredClone(value);
}
function apiError(code) {
return Object.assign(new Error(`Kubernetes API ${code}`), { code });
}
function initialManifest() {
return Object.freeze({
schema: 'qinglong/plugin-package-prompt-output-file-keyring@v1',
generation: 2,
activeKeyId: 'cluster-key-new',
keys: Object.freeze({
'cluster-key-old': Buffer.alloc(32, 7).toString('base64url'),
'cluster-key-new': Buffer.alloc(32, 8).toString('base64url'),
}),
retirements: Object.freeze({}),
});
}
function secret(manifest, resourceVersion = '1', uid = 'uid-keyring-1') {
const metadata = clusterPromptOutputKubernetesSecretKeyringMetadata;
const bytes = canonicalPluginPackagePromptOutputKeyringManifest(manifest);
try {
return {
apiVersion: 'v1',
kind: 'Secret',
type: 'Opaque',
immutable: false,
metadata: {
name: 'ql3-prompt-output-keyring',
namespace: 'qinglong',
uid,
resourceVersion,
labels: {
[metadata.managedByLabel]: metadata.managedByValue,
[metadata.keyringLabel]: metadata.keyringLabelValue,
},
annotations: {
[metadata.generationAnnotation]: String(manifest.generation),
[metadata.catalogDigestAnnotation]:
pluginPackagePromptOutputKeyringCatalogDigest(manifest),
},
},
data: { 'keyring.json': bytes.toString('base64') },
};
} finally {
bytes.fill(0);
}
}
class FakeSecretApi {
constructor(value) {
this.value = copy(value);
this.writeCount = 0;
this.replaceAttempts = 0;
this.failAfterReplace = false;
}
async readNamespacedSecret({ namespace, name }) {
if (
namespace !== this.value.metadata.namespace ||
name !== this.value.metadata.name
) {
throw apiError(404);
}
return copy(this.value);
}
async replaceNamespacedSecret({ namespace, name, body }) {
this.replaceAttempts += 1;
if (
namespace !== this.value.metadata.namespace ||
name !== this.value.metadata.name
) {
throw apiError(404);
}
if (body.metadata.resourceVersion !== this.value.metadata.resourceVersion) {
throw apiError(409);
}
this.writeCount += 1;
this.value = {
...copy(body),
metadata: {
...copy(body.metadata),
resourceVersion: String(Number(body.metadata.resourceVersion) + 1),
},
};
if (this.failAfterReplace) {
this.failAfterReplace = false;
throw apiError(409);
}
return copy(this.value);
}
}
class LostResponseSecretApi extends FakeSecretApi {
failed = false;
async replaceNamespacedSecret(request) {
const written = await super.replaceNamespacedSecret(request);
if (!this.failed) {
this.failed = true;
throw new Error('connection reset after Secret replacement');
}
return written;
}
}
function keyring(api, uid = 'uid-keyring-1') {
return new ClusterPromptOutputKubernetesSecretKeyring(api, {
namespace: 'qinglong',
secretName: 'ql3-prompt-output-keyring',
expectedSecretUid: uid,
});
}
async function preparation(authority) {
const state = await authority.inspect('cluster-key-old');
assert.equal(state.state, 'inactive');
return createPluginPackagePromptOutputKeyRetirementPreparation({
keyId: state.keyId,
retirementId: 'retirement-1',
requestId: 'request-1',
mutationId: 'mutation-1',
catalogDigest: state.catalogDigest,
materialProof: state.materialProof,
preparedAtMs: 1_000,
});
}
test('resourceVersion-fenced Secret keyring retires material and exactly replays', async () => {
const oldMaterial = initialManifest().keys['cluster-key-old'];
const api = new FakeSecretApi(secret(initialManifest()));
const authority = keyring(api);
const prepared = await preparation(authority);
const retired = await authority.retire({ preparation: prepared });
assert.equal(retired.state, 'absent');
assert.equal(
(await authority.inspect('cluster-key-old')).absenceProof,
retired.absenceProof,
);
assert.equal((await authority.inspect('cluster-key-new')).state, 'active');
assert.equal(api.writeCount, 1);
assert.equal(JSON.stringify(api.value).includes(oldMaterial), false);
assert.deepEqual(await authority.retire({ preparation: prepared }), retired);
assert.equal(api.writeCount, 1);
});
test('lost update response and concurrent exact retirement converge to one Secret write', async () => {
const api = new FakeSecretApi(secret(initialManifest()));
const authority = keyring(api);
const prepared = await preparation(authority);
api.failAfterReplace = true;
const [left, right] = await Promise.all([
authority.retire({ preparation: prepared }),
authority.retire({ preparation: prepared }),
]);
assert.deepEqual(left, right);
assert.equal(api.writeCount, 1);
assert.ok(api.replaceAttempts >= 1);
});
test('rejects active retirement and recreated or noncanonical Secret authority', async () => {
const api = new FakeSecretApi(secret(initialManifest()));
const authority = keyring(api);
const active = await authority.inspect('cluster-key-new');
assert.equal(active.state, 'active');
const activePreparation =
createPluginPackagePromptOutputKeyRetirementPreparation({
keyId: active.keyId,
retirementId: 'retirement-active',
requestId: 'request-active',
mutationId: 'mutation-active',
catalogDigest: active.catalogDigest,
materialProof: active.materialProof,
preparedAtMs: 1_000,
});
await assert.rejects(
authority.retire({ preparation: activePreparation }),
PluginPackagePromptOutputKeyRetirementConflictError,
);
await assert.rejects(
keyring(api, 'recreated-uid').inspect('cluster-key-old'),
PluginPackagePromptOutputKeyRetirementUnavailableError,
);
api.value.metadata.annotations[
'kubectl.kubernetes.io/last-applied-configuration'
] = '{}';
await assert.rejects(
authority.inspect('cluster-key-old'),
PluginPackagePromptOutputKeyRetirementUnavailableError,
);
});
test('retirement CAS and runtime projection consume one canonical Secret authority', async () => {
const root = await fs.mkdtemp(
path.join(os.tmpdir(), 'ql3-cluster-keyring-projection-'),
);
const api = new FakeSecretApi(secret(initialManifest()));
const authority = keyring(api);
let generation = 0;
const project = async () => {
generation += 1;
const generationName = `..2026_08_02_${generation}`;
const directory = path.join(root, generationName);
await fs.mkdir(directory, { mode: 0o750 });
const bytes = Buffer.from(api.value.data['keyring.json'], 'base64');
await fs.writeFile(path.join(directory, 'keyring.json'), bytes, {
mode: 0o440,
});
bytes.fill(0);
const next = path.join(root, '..data-next');
await fs.symlink(generationName, next);
await fs.rename(next, path.join(root, '..data'));
try {
await fs.symlink('..data/keyring.json', path.join(root, 'keyring.json'));
} catch (error) {
if (error.code !== 'EEXIST') throw error;
}
};
try {
await project();
const runtime = await createPluginPackagePromptOutputProjectedKeyring({
rootDirectory: root,
});
assert.equal((await runtime.active()).keyId, 'cluster-key-new');
const historical = await runtime.resolve('cluster-key-old');
assert.ok(historical);
historical.key.fill(0);
const prepared = await preparation(authority);
await authority.retire({ preparation: prepared });
await project();
assert.equal(await runtime.resolve('cluster-key-old'), null);
const active = await runtime.active();
assert.equal(active.keyId, 'cluster-key-new');
assert.deepEqual(Buffer.from(active.key), Buffer.alloc(32, 8));
active.key.fill(0);
assert.equal(api.writeCount, 1);
} finally {
await fs.rm(root, { recursive: true, force: true });
}
});
test('rotates externally staged material with one CAS and exact replay', async () => {
const before = initialManifest();
const api = new FakeSecretApi(secret(before));
const authority = keyring(api);
const staged = Buffer.alloc(32, 0x44);
const command = {
expectedActiveKeyId: before.activeKeyId,
expectedCatalogDigest:
pluginPackagePromptOutputKeyringCatalogDigest(before),
newKeyId: 'cluster-key-next',
material: staged,
};
const rotated = await authority.rotate(command);
assert.equal(rotated.generation, 3);
assert.equal(rotated.previousActiveKeyId, 'cluster-key-new');
assert.equal(rotated.activeKeyId, 'cluster-key-next');
assert.equal(api.writeCount, 1);
assert.equal(
JSON.stringify(rotated).includes(staged.toString('base64url')),
false,
);
assert.deepEqual(staged, Buffer.alloc(32, 0x44));
assert.deepEqual(await authority.rotate(command), rotated);
assert.equal(api.writeCount, 1);
});
test('recovers rotation and retirement from ambiguous lost responses', async () => {
const rotationManifest = initialManifest();
const rotationApi = new LostResponseSecretApi(secret(rotationManifest));
const rotated = await keyring(rotationApi).rotate({
expectedActiveKeyId: rotationManifest.activeKeyId,
expectedCatalogDigest:
pluginPackagePromptOutputKeyringCatalogDigest(rotationManifest),
newKeyId: 'cluster-key-next',
material: Buffer.alloc(32, 0x55),
});
assert.equal(rotated.activeKeyId, 'cluster-key-next');
assert.equal(rotationApi.writeCount, 1);
const retirementApi = new LostResponseSecretApi(secret(initialManifest()));
const retirementAuthority = keyring(retirementApi);
const prepared = await preparation(retirementAuthority);
assert.equal(
(await retirementAuthority.retire({ preparation: prepared })).state,
'absent',
);
assert.equal(retirementApi.writeCount, 1);
});
test('gives concurrent exact rotations one winner and rejects changed staged material', async () => {
const before = initialManifest();
const api = new FakeSecretApi(secret(before));
const authority = keyring(api);
const command = {
expectedActiveKeyId: before.activeKeyId,
expectedCatalogDigest:
pluginPackagePromptOutputKeyringCatalogDigest(before),
newKeyId: 'cluster-key-next',
material: Buffer.alloc(32, 0x66),
};
const [left, right] = await Promise.all([
authority.rotate(command),
authority.rotate(command),
]);
assert.deepEqual(left, right);
assert.equal(api.writeCount, 1);
await assert.rejects(
authority.rotate({ ...command, material: Buffer.alloc(32, 0x77) }),
PluginPackagePromptOutputKeyRetirementConflictError,
);
});
@@ -0,0 +1,142 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
WorkerCredentialMutationConflictError,
} = require('@qinglong/runtime-core/worker-credential');
const {
workerCredentialSecretDigest,
} = require('@qinglong/runtime-core/worker-credential-token');
const {
createWorkerCredentialAdministrationService,
} = require('../dist/worker-credential/workerCredentialAdministration');
const NOW = 1_000;
const PEPPER = Buffer.alloc(32, 1).toString('base64url');
const PRINCIPAL = {
subject: { type: 'user', id: 'usr_admin' },
authenticationId: 'session:admin:1',
authenticatedAtMs: 900,
expiresAtMs: 2_000,
assurance: 'multi_factor',
};
function request(overrides = {}) {
return {
mutationId: '123e4567-e89b-42d3-a456-426614174501',
requestId: 'request-worker-issue-1',
expectedCurrentVersion: 0,
credentialId: 'worker_primary',
workerId: 'edge-router-1',
principal: PRINCIPAL,
notBeforeAtMs: NOW,
expiresAtMs: 2_000,
...overrides,
};
}
function repository() {
let mutation = null;
const commands = [];
return {
commands,
port: {
async resolveMutation() { return mutation; },
async append(command) {
commands.push(command);
mutation = {
credential: command.credential,
mutation: command.mutation,
audit: command.audit,
};
return {
status: 'created',
credential: command.credential,
mutation: command.mutation,
};
},
},
setMutation(value) { mutation = value; },
};
}
test('issues a one-time ql3w token and stores only its Worker-domain digest', async () => {
const store = repository();
const generated = Buffer.alloc(32, 7);
const secret = generated.toString('base64url');
const service = createWorkerCredentialAdministrationService(
store.port,
PEPPER,
{ now: () => NOW, randomBytes: () => generated },
);
const result = await service.issue(request());
assert.equal(result.status, 'created');
assert.equal(result.token, `ql3w_worker_primary_${secret}`);
assert.equal(
store.commands[0].credential.secretDigest,
workerCredentialSecretDigest(PEPPER, 'worker_primary', secret),
);
assert.equal(store.commands[0].credential.workerId, 'edge-router-1');
assert.equal(store.commands[0].audit.operationId, 'worker_credential.issue');
assert.equal(generated.every((byte) => byte === 0), true);
});
test('accepts an approved activation time before delayed execution', async () => {
const store = repository();
const service = createWorkerCredentialAdministrationService(
store.port,
PEPPER,
{ now: () => 1_500, randomBytes: () => Buffer.alloc(32, 7) },
);
const result = await service.issue(request());
assert.equal(result.status, 'created');
assert.equal(result.credential.createdAtMs, 1_500);
assert.equal(result.credential.notBeforeAtMs, NOW);
assert.equal(result.credential.expiresAtMs, 2_000);
});
test('semantic replay returns no secret and conflicting replay fails closed', async () => {
const store = repository();
let randomCalls = 0;
const service = createWorkerCredentialAdministrationService(
store.port,
PEPPER,
{
now: () => NOW,
randomBytes() { randomCalls += 1; return Buffer.alloc(32, randomCalls); },
},
);
assert.ok((await service.issue(request())).token);
const replay = await service.issue(request());
assert.equal(replay.status, 'existing');
assert.equal(replay.token, null);
assert.equal(randomCalls, 1);
await assert.rejects(
service.issue(request({ workerId: 'other-worker' })),
WorkerCredentialMutationConflictError,
);
});
test('rejects widened requests, weak principals and operation/version confusion', async () => {
const store = repository();
const service = createWorkerCredentialAdministrationService(
store.port,
PEPPER,
{ now: () => NOW, randomBytes: () => Buffer.alloc(32, 1) },
);
await assert.rejects(service.issue(request({ debug: true })), /shape is invalid/);
await assert.rejects(
service.issue(request({ principal: { ...PRINCIPAL, assurance: 'single_factor' } })),
/strong principal/,
);
await assert.rejects(
service.issue(request({ expectedCurrentVersion: 1 })),
/operation fence is invalid/,
);
const { notBeforeAtMs, expiresAtMs, ...revoke } = request({
mutationId: '123e4567-e89b-42d3-a456-426614174502',
});
assert.equal(notBeforeAtMs, NOW);
assert.equal(expiresAtMs, 2_000);
await assert.rejects(service.revoke(revoke), /operation fence is invalid/);
assert.equal(store.commands.length, 0);
});
@@ -0,0 +1,538 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
WorkerCredentialDeliveryConflictError,
WorkerCredentialDeliveryUnavailableError,
} = require('@qinglong/runtime-core/worker-credential-delivery');
const {
createRecoverableWorkerCredentialIssuer,
createWorkerCredentialDeliveryRecoveryService,
createWorkerCredentialStageCleanupService,
} = require('../dist/worker-credential/workerCredentialDelivery');
const NOW = 1_000;
const PEPPER = Buffer.alloc(32, 1).toString('base64url');
const MUTATION_ID = '123e4567-e89b-42d3-a456-426614174701';
const PRINCIPAL = {
subject: { type: 'user', id: 'usr_admin' },
authenticationId: 'session:admin:1',
authenticatedAtMs: 900,
expiresAtMs: 2_000,
assurance: 'multi_factor',
};
function request(overrides = {}) {
return {
mutationId: MUTATION_ID,
requestId: 'request-worker-delivery-1',
expectedCurrentVersion: 0,
credentialId: 'worker_generation_2',
workerId: 'edge-router-1',
principal: PRINCIPAL,
notBeforeAtMs: NOW,
expiresAtMs: 2_000,
previousCredentialId: 'worker_generation_1',
deploymentTargetDigest: 'c'.repeat(64),
deploymentGeneration: 'secret-generation-2',
...overrides,
};
}
function committedRecord(overrides = {}) {
return {
deliveryId: MUTATION_ID,
version: 1,
state: 'credential_committed',
workerId: 'edge-router-1',
credentialId: 'worker_generation_2',
credentialVersion: 1,
previousCredentialId: 'worker_generation_1',
secretDigest: 'a'.repeat(64),
tokenDigest: 'b'.repeat(64),
deploymentTargetDigest: 'c'.repeat(64),
deploymentGeneration: 'secret-generation-2',
stagedAtMs: NOW,
credentialCommittedAtMs: NOW,
publishedAtMs: null,
publicationDigest: null,
observedAtMs: null,
observedSessionId: null,
observedSessionVersion: null,
previousRevokedAtMs: null,
...overrides,
};
}
function stagedIntent(overrides = {}) {
const {
version: _version,
state: _state,
credentialCommittedAtMs: _credentialCommittedAtMs,
publishedAtMs: _publishedAtMs,
publicationDigest: _publicationDigest,
observedAtMs: _observedAtMs,
observedSessionId: _observedSessionId,
observedSessionVersion: _observedSessionVersion,
previousRevokedAtMs: _previousRevokedAtMs,
...intent
} = committedRecord();
return { ...intent, ...overrides };
}
function fakeAuthority() {
let resolved = null;
let rawMutation = null;
let stageDiscard = null;
const revokeMutations = new Map();
const state = {
commits: 0,
marks: 0,
failMarkAfterCommit: false,
revokes: 0,
};
const port = {
async resolveMutation(mutationId) {
if (resolved?.mutation.mutationId === mutationId) return resolved;
return revokeMutations.get(mutationId) ?? rawMutation;
},
async append() {
throw new Error('raw append must remain unreachable');
},
async resolveDelivery() {
return resolved?.delivery ?? null;
},
async resolveDelivered() {
return resolved;
},
async commitDelivered(command) {
state.commits += 1;
if (resolved) {
return {
status: 'existing',
credential: resolved.credential,
mutation: resolved.mutation,
};
}
resolved = {
credential: command.credential.credential,
mutation: command.credential.mutation,
audit: command.credential.audit,
delivery: command.delivery,
};
return {
status: 'created',
credential: resolved.credential,
mutation: resolved.mutation,
};
},
async markPublished(command) {
state.marks += 1;
assert.ok(resolved);
assert.equal(command.expectedVersion, resolved.delivery.version);
resolved = {
...resolved,
delivery: {
...resolved.delivery,
version: 2,
state: 'published',
publishedAtMs: command.publishedAtMs,
publicationDigest: command.publicationDigest,
},
};
if (state.failMarkAfterCommit) {
state.failMarkAfterCommit = false;
throw new Error('commit response lost');
}
return resolved.delivery;
},
async listRecoveryPage(options = {}) {
const delivery = resolved?.delivery;
const recoverable = delivery &&
delivery.state !== 'previous_revoked' &&
!(delivery.state === 'observed' && delivery.previousCredentialId === null) &&
(!options.afterDeliveryId || delivery.deliveryId > options.afterDeliveryId)
? [delivery]
: [];
return {
observedAtMs: 1_500,
deliveries: recoverable,
truncated: false,
};
},
async revokePreviousDelivered(command) {
state.revokes += 1;
const replay = revokeMutations.get(command.credential.mutation.mutationId);
if (replay) {
return {
status: 'existing',
credential: replay.credential,
mutation: replay.mutation,
};
}
const value = {
credential: command.credential.credential,
mutation: command.credential.mutation,
audit: command.credential.audit,
};
revokeMutations.set(command.credential.mutation.mutationId, value);
resolved = { ...resolved, delivery: command.delivery };
return {
status: 'created',
credential: value.credential,
mutation: value.mutation,
};
},
async authorizeStageDiscard(intent) {
if (resolved || rawMutation) throw new WorkerCredentialDeliveryConflictError();
if (stageDiscard) {
const { version, state, authorizedAtMs, discardedAtMs, ...existing } = stageDiscard;
if (JSON.stringify(existing) !== JSON.stringify(intent)) {
throw new WorkerCredentialDeliveryConflictError();
}
return stageDiscard;
}
stageDiscard = {
...intent,
version: 1,
state: 'discard_authorized',
authorizedAtMs: 1_100,
discardedAtMs: null,
};
return stageDiscard;
},
async markStageDiscarded(command) {
if (!stageDiscard || command.expectedVersion !== 1) {
throw new WorkerCredentialDeliveryConflictError();
}
if (stageDiscard.state === 'discarded') return stageDiscard;
stageDiscard = {
...stageDiscard,
version: 2,
state: 'discarded',
discardedAtMs: 1_200,
};
return stageDiscard;
},
async listStageDiscardRecoveryPage() {
return {
observedAtMs: 1_300,
discards: stageDiscard?.state === 'discard_authorized'
? [stageDiscard]
: [],
truncated: false,
};
},
};
return {
port,
state,
current: () => resolved,
setRaw(value) { rawMutation = value; },
setDelivery(delivery) { resolved = { ...resolved, delivery }; },
stageDiscard: () => stageDiscard,
};
}
function fakeAdapter() {
let staged = null;
let token = null;
const state = {
stages: 0,
publishes: 0,
discards: 0,
failPublish: false,
passedToken: null,
};
const port = {
async inspect() { return staged; },
async stage(delivery, value) {
state.stages += 1;
if (staged) throw new Error('no replace');
staged = delivery;
state.passedToken = value;
token = Buffer.from(value);
},
async publish(delivery) {
state.publishes += 1;
if (state.failPublish) throw new Error('deployment unavailable');
assert.ok(staged);
assert.equal(delivery.deliveryId, staged.deliveryId);
assert.ok(token.toString('utf8').startsWith('ql3w_worker_generation_2_'));
return { publicationDigest: 'd'.repeat(64) };
},
async discard(delivery) {
state.discards += 1;
assert.equal(delivery.deliveryId, staged?.deliveryId);
token?.fill(0);
token = null;
staged = null;
},
async listStaged() {
return {
stages: staged ? [staged] : [],
truncated: false,
};
},
};
return {
port,
state,
seed(delivery) { staged = delivery; token = Buffer.from('seed'); },
clear() { staged = null; token?.fill(0); token = null; },
};
}
function issuer(authority, adapter, options = {}) {
return createRecoverableWorkerCredentialIssuer(
authority.port,
adapter.port,
PEPPER,
{ now: () => NOW, ...options },
);
}
function recovery(authority, adapter) {
return createWorkerCredentialDeliveryRecoveryService(
authority.port,
adapter.port,
PEPPER,
PRINCIPAL,
);
}
test('stages before one atomic credential commit and publishes without returning a token', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
const generated = Buffer.alloc(32, 7);
const result = await issuer(authority, adapter, {
randomBytes: () => generated,
}).issue(request());
assert.equal(result.status, 'published');
assert.equal(result.delivery.state, 'published');
assert.equal(authority.state.commits, 1);
assert.equal(authority.state.marks, 1);
assert.equal(adapter.state.stages, 1);
assert.equal(adapter.state.publishes, 1);
assert.equal(generated.every((byte) => byte === 0), true);
assert.equal(adapter.state.passedToken.every((byte) => byte === 0), true);
assert.equal(JSON.stringify(authority.current()).includes('ql3w_'), false);
});
test('publishes a preapproved credential after execution delay', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
const result = await issuer(authority, adapter, {
now: () => 1_500,
randomBytes: () => Buffer.alloc(32, 7),
}).issue(request());
assert.equal(result.status, 'published');
assert.equal(authority.current().credential.createdAtMs, 1_500);
assert.equal(authority.current().credential.notBeforeAtMs, NOW);
});
test('resumes a committed credential after publication failure without new entropy', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
let randomCalls = 0;
const service = issuer(authority, adapter, {
randomBytes() { randomCalls += 1; return Buffer.alloc(32, randomCalls); },
});
adapter.state.failPublish = true;
await assert.rejects(service.issue(request()), WorkerCredentialDeliveryUnavailableError);
assert.equal(authority.current().delivery.state, 'credential_committed');
adapter.state.failPublish = false;
const recovered = await service.issue(request());
assert.equal(recovered.status, 'existing');
assert.equal(recovered.delivery.state, 'published');
assert.equal(randomCalls, 1);
assert.equal(authority.state.commits, 1);
assert.equal(adapter.state.stages, 1);
assert.equal(adapter.state.publishes, 2);
});
test('recovers a lost publication-ledger response without republishing', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
let randomCalls = 0;
const service = issuer(authority, adapter, {
randomBytes() { randomCalls += 1; return Buffer.alloc(32, 9); },
});
authority.state.failMarkAfterCommit = true;
await assert.rejects(service.issue(request()), WorkerCredentialDeliveryUnavailableError);
assert.equal(authority.current().delivery.state, 'published');
const recovered = await service.issue(request());
assert.equal(recovered.status, 'existing');
assert.equal(recovered.delivery.state, 'published');
assert.equal(randomCalls, 1);
assert.equal(adapter.state.publishes, 1);
assert.equal(authority.state.commits, 1);
});
test('discards an orphaned pre-commit stage only after authoritative absence', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
adapter.seed(stagedIntent());
let randomCalls = 0;
const result = await issuer(authority, adapter, {
randomBytes() { randomCalls += 1; return Buffer.alloc(32, 1); },
}).issue(request());
assert.deepEqual(result, {
status: 'orphaned_stage_discarded',
delivery: null,
});
assert.equal(adapter.state.discards, 1);
assert.equal(authority.state.commits, 0);
assert.equal(randomCalls, 0);
});
test('fails closed for raw mutations, missing staged secrets and semantic drift', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
authority.setRaw({ credential: {}, mutation: {}, audit: {} });
await assert.rejects(
issuer(authority, adapter).issue(request()),
WorkerCredentialDeliveryConflictError,
);
const committedAuthority = fakeAuthority();
const committedAdapter = fakeAdapter();
await issuer(committedAuthority, committedAdapter, {
randomBytes: () => Buffer.alloc(32, 4),
}).issue(request());
committedAdapter.clear();
await assert.rejects(
issuer(committedAuthority, committedAdapter).issue(request()),
WorkerCredentialDeliveryConflictError,
);
await assert.rejects(
issuer(committedAuthority, fakeAdapter()).issue(request({
deploymentGeneration: 'other-generation',
})),
WorkerCredentialDeliveryConflictError,
);
});
test('preserves deployment semantic conflicts for manual review', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
adapter.port.publish = async () => {
throw new WorkerCredentialDeliveryConflictError();
};
await assert.rejects(
issuer(authority, adapter, {
randomBytes: () => Buffer.alloc(32, 6),
}).issue(request()),
WorkerCredentialDeliveryConflictError,
);
assert.equal(authority.current().delivery.state, 'credential_committed');
});
test('authorizes cleanup before discarding one globally inventoried stage', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
adapter.seed(stagedIntent());
const service = createWorkerCredentialStageCleanupService(
authority.port,
adapter.port,
);
const result = await service.cleanupInventoryPage({ limit: 1 });
assert.deepEqual(result, {
outcomes: [{ deliveryId: MUTATION_ID, result: 'discarded' }],
truncated: false,
});
assert.equal(authority.stageDiscard().state, 'discarded');
assert.equal(adapter.state.discards, 1);
assert.equal(await adapter.port.inspect(MUTATION_ID), null);
});
test('recovers an authorized discard after the stage removal response is lost', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
const intent = stagedIntent();
adapter.seed(intent);
await authority.port.authorizeStageDiscard(intent);
adapter.clear();
const service = createWorkerCredentialStageCleanupService(
authority.port,
adapter.port,
);
const result = await service.recoverAuthorizedPage({ limit: 1 });
assert.equal(result.observedAtMs, 1_300);
assert.deepEqual(result.outcomes, [
{ deliveryId: MUTATION_ID, result: 'discarded' },
]);
assert.equal(authority.stageDiscard().version, 2);
assert.equal(adapter.state.discards, 0);
});
test('fails closed when an authorized stage is semantically rewritten', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
const intent = stagedIntent();
await authority.port.authorizeStageDiscard(intent);
adapter.seed(stagedIntent({ tokenDigest: 'e'.repeat(64) }));
await assert.rejects(
createWorkerCredentialStageCleanupService(
authority.port,
adapter.port,
).recoverAuthorizedPage(),
WorkerCredentialDeliveryConflictError,
);
assert.equal(authority.stageDiscard().state, 'discard_authorized');
assert.equal(adapter.state.discards, 0);
});
test('rejects same-ID rotation and widened requests before touching authority', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
const service = issuer(authority, adapter);
await assert.rejects(
service.issue(request({ previousCredentialId: 'worker_generation_2' })),
/requires a new credential ID|identity is invalid/,
);
await assert.rejects(
service.issue(request({ debug: true })),
/shape is invalid/,
);
assert.equal(authority.state.commits, 0);
assert.equal(adapter.state.stages, 0);
});
test('recovers publication then atomically revokes the observed previous credential', async () => {
const authority = fakeAuthority();
const adapter = fakeAdapter();
adapter.state.failPublish = true;
await assert.rejects(
issuer(authority, adapter, {
randomBytes: () => Buffer.alloc(32, 8),
}).issue(request()),
WorkerCredentialDeliveryUnavailableError,
);
adapter.state.failPublish = false;
const publication = await recovery(authority, adapter).recoverPage();
assert.deepEqual(publication.outcomes, [{
deliveryId: MUTATION_ID,
state: 'published',
result: 'published',
}]);
authority.setDelivery({
...authority.current().delivery,
version: 3,
state: 'observed',
observedAtMs: 1_500,
observedSessionId: '019f7094-a853-72f3-82ab-dfa08e6bd1c1',
observedSessionVersion: 4,
});
const revoked = await recovery(authority, adapter).recoverPage();
assert.deepEqual(revoked.outcomes, [{
deliveryId: MUTATION_ID,
state: 'previous_revoked',
result: 'previous_revoked',
}]);
assert.equal(authority.state.revokes, 1);
assert.equal(authority.current().delivery.previousRevokedAtMs, 1_500);
const replay = await recovery(authority, adapter).recoverPage();
assert.deepEqual(replay.outcomes, []);
assert.equal(authority.state.revokes, 1);
});
@@ -0,0 +1,322 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const { chmod, mkdtemp, rm, writeFile } = require('node:fs/promises');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterWorkerCredentialExecutorProcessConfigError,
loadClusterWorkerCredentialExecutorProcessConfig,
runClusterWorkerCredentialExecutorProcess,
} = require('@qinglong/cluster-admin/worker-credential-executor-process');
const COMMAND = Object.freeze({
schemaVersion: 1,
actionRef: 'worker-credential:delivery-7',
approvalRequestId: 'approval-7',
consumptionId: 'consumption-7',
dispatchId: 'dispatch-7',
auditEventId: 'audit-7',
});
const CLI = resolve(__dirname, '../dist/worker-credential/workerCredentialExecutorCli.js');
function enabledEnvironment(paths, overrides = {}) {
return {
QL3_WORKER_CREDENTIAL_EXECUTOR_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
QL3_WORKER_CREDENTIAL_EXECUTOR_COMMAND_FILE: paths.commandFile,
QL3_WORKER_CREDENTIAL_EXECUTOR_PEPPER_FILE: paths.pepperFile,
QL3_WORKER_CREDENTIAL_EXECUTOR_CLUSTER_IDENTITY: 'cluster-primary',
QL3_WORKER_CREDENTIAL_EXECUTOR_STAGE_NAMESPACE: 'qinglong3-stage',
QL3_WORKER_CREDENTIAL_EXECUTOR_TARGET_NAMESPACE: 'qinglong3-worker',
QL3_WORKER_CREDENTIAL_EXECUTOR_TARGET_SECRET: 'worker-credential',
QL3_WORKER_CREDENTIAL_EXECUTOR_TARGET_DEPLOYMENT: 'worker-runtime',
QL3_WORKER_CREDENTIAL_EXECUTOR_TARGET_DATA_KEY: 'credential',
QL3_WORKER_CREDENTIAL_EXECUTOR_DELIVERY_SERVICE_ACCOUNT:
'worker-credential-delivery',
QL3_WORKER_CREDENTIAL_EXECUTOR_IDENTITY_SECRET: 'cluster-identity',
QL3_POSTGRES_WORKER_CREDENTIAL_EXECUTOR_URL:
'postgresql://worker_executor:secret@postgres.example.test/ql3',
QL3_POSTGRES_WORKER_CREDENTIAL_EXECUTOR_TLS_MODE: 'disable',
QL3_POSTGRES_WORKER_CREDENTIAL_EXECUTOR_ALLOW_INSECURE: 'true',
...overrides,
};
}
async function authorityFixture(run, options = {}) {
const directory = await mkdtemp(join(tmpdir(), 'ql3-worker-executor-'));
const paths = {
commandFile: join(directory, 'command.json'),
pepperFile: join(directory, 'pepper'),
};
try {
await writeFile(paths.commandFile, `${JSON.stringify(COMMAND)}\n`, {
mode: options.commandMode ?? 0o440,
});
await writeFile(
paths.pepperFile,
'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA',
{ mode: options.pepperMode ?? 0o440 },
);
await chmod(paths.commandFile, options.commandMode ?? 0o440);
await chmod(paths.pepperFile, options.pepperMode ?? 0o440);
return await run(paths);
} finally {
await rm(directory, { recursive: true, force: true });
}
}
test('disabled Worker executor reads no profile, files or authorities', async () => {
const reads = [];
const environment = new Proxy(
{ QL3_WORKER_CREDENTIAL_EXECUTOR_ENABLED: 'false' },
{
get(target, property) {
reads.push(property);
if (property === 'QL3_WORKER_CREDENTIAL_EXECUTOR_ENABLED') {
return target[property];
}
throw new Error(`disabled config read ${String(property)}`);
},
},
);
let created = 0;
let executed = 0;
const result = await runClusterWorkerCredentialExecutorProcess({
environment,
async createKubernetesAuthority() {
created += 1;
throw new Error('must not create');
},
async execute() {
executed += 1;
throw new Error('must not execute');
},
});
assert.deepEqual(result, { status: 'disabled' });
assert.equal(created, 0);
assert.equal(executed, 0);
assert.deepEqual(reads, ['QL3_WORKER_CREDENTIAL_EXECUTOR_ENABLED']);
});
test('loads one explicit caller-driven executor configuration', () => {
const config = loadClusterWorkerCredentialExecutorProcessConfig(
enabledEnvironment({
commandFile: '/run/ql3-worker-executor/command.json',
pepperFile: '/run/ql3-worker-executor/pepper',
}),
);
assert.equal(config.enabled, true);
assert.equal(config.profile, 'cluster-admin');
assert.equal(config.database.connection.tls.mode, 'disable');
assert.equal(config.database.pool.maxConnections, 1);
assert.equal(
config.database.pool.applicationName,
'qinglong3-worker-credential-executor',
);
assert.deepEqual(config.delivery, {
clusterIdentity: 'cluster-primary',
stageNamespace: 'qinglong3-stage',
namespace: 'qinglong3-worker',
targetSecretName: 'worker-credential',
targetDeploymentName: 'worker-runtime',
targetDataKey: 'credential',
});
});
test('rejects profile drift, relative authority files and implicit insecure database', () => {
const paths = {
commandFile: '/run/ql3-worker-executor/command.json',
pepperFile: '/run/ql3-worker-executor/pepper',
};
for (const environment of [
enabledEnvironment(paths, { QL3_PROFILE: 'cluster-control' }),
enabledEnvironment(paths, {
QL3_WORKER_CREDENTIAL_EXECUTOR_COMMAND_FILE: 'command.json',
}),
enabledEnvironment(paths, {
QL3_POSTGRES_WORKER_CREDENTIAL_EXECUTOR_ALLOW_INSECURE: 'false',
}),
enabledEnvironment(paths, {
QL3_WORKER_CREDENTIAL_EXECUTOR_TARGET_NAMESPACE: 'INVALID_NAMESPACE',
}),
]) {
assert.throws(
() => loadClusterWorkerCredentialExecutorProcessConfig(environment),
ClusterWorkerCredentialExecutorProcessConfigError,
);
}
});
test('composes exact one-shot execution and always disposes issuer authority', async () => {
await authorityFixture(async (paths) => {
const session = { async withDelivery() {} };
const confirmAuthorization = async () => {};
const openDatabase = async () => {
throw new Error('injected executor owns database use');
};
let disposed = 0;
let observed;
const run = Object.freeze({
database: { ready: true },
approval: { dispatchId: COMMAND.dispatchId },
execution: { status: 'completed' },
result: { status: 'published' },
tokenRequest: { issued: true },
});
const result = await runClusterWorkerCredentialExecutorProcess({
environment: enabledEnvironment(paths),
openDatabase,
kubernetesAuthority: {
session,
confirmAuthorization,
dispose() {
disposed += 1;
},
},
async execute(options) {
observed = options;
return run;
},
now: () => 7_000,
});
assert.equal(result.status, 'completed');
assert.equal(result.run, run);
assert.deepEqual(result.command, COMMAND);
assert.equal(disposed, 1);
assert.equal(observed.openDatabase, openDatabase);
assert.equal(observed.tokenRequestSession, session);
assert.equal(observed.confirmAuthorization, confirmAuthorization);
assert.equal(observed.workerCredentialPepper.length, 43);
assert.equal(observed.actionRef, COMMAND.actionRef);
assert.equal(observed.approvalRequestId, COMMAND.approvalRequestId);
assert.equal(observed.consumptionId, COMMAND.consumptionId);
assert.equal(observed.dispatchId, COMMAND.dispatchId);
assert.equal(observed.auditEventId, COMMAND.auditEventId);
assert.equal(observed.now(), 7_000);
});
});
test('preserves execution and issuer-disposal failures together', async () => {
await authorityFixture(async (paths) => {
const executionFailure = new Error('execution failed');
const disposalFailure = new Error('disposal failed');
await assert.rejects(
runClusterWorkerCredentialExecutorProcess({
environment: enabledEnvironment(paths),
kubernetesAuthority: {
session: { async withDelivery() {} },
async confirmAuthorization() {},
dispose() {
throw disposalFailure;
},
},
async execute() {
throw executionFailure;
},
}),
(error) => {
assert.equal(error instanceof AggregateError, true);
assert.deepEqual(error.errors, [executionFailure, disposalFailure]);
return true;
},
);
});
});
test('rejects expanded commands and publicly readable pepper material', async () => {
await authorityFixture(
async (paths) => {
await assert.rejects(
runClusterWorkerCredentialExecutorProcess({
environment: enabledEnvironment(paths),
kubernetesAuthority: {
session: { async withDelivery() {} },
async confirmAuthorization() {},
dispose() {},
},
async execute() {
throw new Error('must not execute');
},
}),
ClusterWorkerCredentialExecutorProcessConfigError,
);
},
{ pepperMode: 0o444 },
);
await authorityFixture(async (paths) => {
await chmod(paths.commandFile, 0o640);
await writeFile(
paths.commandFile,
`${JSON.stringify({ ...COMMAND, unexpected: true })}\n`,
{ mode: 0o440 },
);
await chmod(paths.commandFile, 0o440);
await assert.rejects(
runClusterWorkerCredentialExecutorProcess({
environment: enabledEnvironment(paths),
kubernetesAuthority: {
session: { async withDelivery() {} },
async confirmAuthorization() {},
dispose() {},
},
async execute() {
throw new Error('must not execute');
},
}),
ClusterWorkerCredentialExecutorProcessConfigError,
);
});
});
test('CLI exposes no authority paths or secret-bearing failure details', () => {
const help = spawnSync(process.execPath, [CLI, '--help'], {
encoding: 'utf8',
});
assert.equal(help.status, 0);
assert.equal(help.stdout, 'Usage: ql3-worker-credential-execute\n');
assert.equal(help.stderr, '');
const disabled = spawnSync(process.execPath, [CLI], {
encoding: 'utf8',
env: {
...process.env,
QL3_WORKER_CREDENTIAL_EXECUTOR_ENABLED: 'false',
},
});
assert.equal(disabled.status, 0);
assert.deepEqual(JSON.parse(disabled.stdout), {
schemaVersion: 1,
component: 'qinglong3-worker-credential-executor',
event: 'execution_disabled',
});
assert.equal(disabled.stderr, '');
const failure = spawnSync(process.execPath, [CLI], {
encoding: 'utf8',
env: {
...process.env,
QL3_WORKER_CREDENTIAL_EXECUTOR_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
},
});
assert.equal(failure.status, 1);
assert.equal(failure.stdout, '');
const fact = JSON.parse(failure.stderr);
assert.deepEqual(Object.keys(fact).sort(), [
'code',
'component',
'event',
'name',
'schemaVersion',
]);
assert.equal(
fact.code,
'QL3_WORKER_CREDENTIAL_EXECUTOR_PROCESS_CONFIG_INVALID',
);
assert.equal(failure.stderr.includes('/'), false);
assert.equal(failure.stderr.includes('pepper'), false);
});
@@ -0,0 +1,338 @@
const assert = require('node:assert/strict');
const { randomUUID } = require('node:crypto');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const {
WorkerCredentialDeliveryConflictError,
WorkerCredentialDeliveryUnavailableError,
workerCredentialDeliveryTokenDigest,
} = require('@qinglong/runtime-core/worker-credential-delivery');
const {
formatWorkerCredentialToken,
} = require('@qinglong/runtime-core/worker-credential-token');
const {
MAX_WORKER_CREDENTIAL_FILE_STAGES,
WorkerCredentialFileDeliveryAdapter,
} = require('@qinglong/cluster-admin/worker-credential-file-delivery');
const {
createRecoverableWorkerCredentialIssuer,
} = require('../dist/worker-credential/workerCredentialDelivery');
const DELIVERY_ID = '123e4567-e89b-42d3-a456-426614174901';
const CREDENTIAL_ID = 'worker_generation_2';
const PREVIOUS_CREDENTIAL_ID = 'worker_generation_1';
const PEPPER = Buffer.alloc(32, 7).toString('base64url');
function fixture(t) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-worker-delivery-'));
const stages = path.join(root, 'stages');
const target = path.join(root, 'target');
fs.mkdirSync(stages, { mode: 0o700 });
fs.mkdirSync(target, { mode: 0o700 });
const targetTokenFile = path.join(target, 'credential.token');
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const adapter = new WorkerCredentialFileDeliveryAdapter({
stageDirectory: stages,
targetTokenFile,
});
return { root, stages, target, targetTokenFile, adapter };
}
function token(credentialId, fill) {
return Buffer.from(formatWorkerCredentialToken(
credentialId,
Buffer.alloc(32, fill).toString('base64url'),
));
}
function intent(adapter, value, overrides = {}) {
return {
deliveryId: DELIVERY_ID,
workerId: 'edge-router-1',
credentialId: CREDENTIAL_ID,
credentialVersion: 1,
previousCredentialId: PREVIOUS_CREDENTIAL_ID,
secretDigest: 'a'.repeat(64),
tokenDigest: workerCredentialDeliveryTokenDigest(value),
deploymentTargetDigest: adapter.deploymentTargetDigest,
deploymentGeneration: 'secret-generation-2',
stagedAtMs: 1_000,
...overrides,
};
}
function committed(candidate, overrides = {}) {
return {
...candidate,
version: 1,
state: 'credential_committed',
credentialCommittedAtMs: 1_000,
publishedAtMs: null,
publicationDigest: null,
observedAtMs: null,
observedSessionId: null,
observedSessionVersion: null,
previousRevokedAtMs: null,
...overrides,
};
}
function deliveryAuthority() {
let resolved = null;
return {
port: {
async resolveMutation() { return resolved; },
async append() { throw new Error('raw append is forbidden'); },
async resolveDelivery() { return resolved?.delivery ?? null; },
async resolveDelivered() { return resolved; },
async commitDelivered(command) {
resolved = {
credential: command.credential.credential,
mutation: command.credential.mutation,
audit: command.credential.audit,
delivery: command.delivery,
};
return {
status: 'created',
credential: resolved.credential,
mutation: resolved.mutation,
};
},
async markPublished(command) {
resolved = {
...resolved,
delivery: {
...resolved.delivery,
version: 2,
state: 'published',
publishedAtMs: command.publishedAtMs,
publicationDigest: command.publicationDigest,
},
};
return resolved.delivery;
},
async listRecoveryPage() { throw new Error('not used'); },
async revokePreviousDelivered() { throw new Error('not used'); },
async authorizeStageDiscard() { throw new Error('not used'); },
async markStageDiscarded() { throw new Error('not used'); },
async listStageDiscardRecoveryPage() { throw new Error('not used'); },
},
current: () => resolved,
};
}
test('stages one private no-replace secret and inspects only its intent', async (t) => {
const { adapter, stages } = fixture(t);
const material = token(CREDENTIAL_ID, 2);
const candidate = intent(adapter, material);
await adapter.stage(candidate, material);
material.fill(0);
assert.deepEqual(await adapter.inspect(DELIVERY_ID), candidate);
const stagePath = path.join(stages, `${DELIVERY_ID}.stage`);
assert.equal(fs.statSync(stagePath).mode & 0o777, 0o600);
assert.deepEqual(fs.readdirSync(stages), [`${DELIVERY_ID}.stage`]);
const replay = token(CREDENTIAL_ID, 2);
await adapter.stage(candidate, replay);
await assert.rejects(
adapter.stage({ ...candidate, deploymentGeneration: 'other-generation' }, replay),
WorkerCredentialDeliveryConflictError,
);
replay.fill(0);
assert.equal((await adapter.inspect(DELIVERY_ID)).deploymentGeneration,
'secret-generation-2');
});
test('atomically replaces only the expected previous credential and replays publication', async (t) => {
const { adapter, targetTokenFile, stages, target } = fixture(t);
const previous = token(PREVIOUS_CREDENTIAL_ID, 1);
fs.writeFileSync(targetTokenFile, previous, { mode: 0o600 });
previous.fill(0);
const material = token(CREDENTIAL_ID, 2);
const candidate = intent(adapter, material);
await adapter.stage(candidate, material);
const publication = await adapter.publish(committed(candidate));
assert.match(publication.publicationDigest, /^[0-9a-f]{64}$/);
assert.deepEqual(fs.readFileSync(targetTokenFile), material);
assert.equal(fs.statSync(targetTokenFile).mode & 0o777, 0o600);
assert.deepEqual(fs.readdirSync(target), ['credential.token']);
assert.deepEqual(fs.readdirSync(stages), [`${DELIVERY_ID}.stage`]);
const restarted = new WorkerCredentialFileDeliveryAdapter({
stageDirectory: stages,
targetTokenFile,
});
assert.deepEqual(
await restarted.publish(committed(candidate)),
publication,
);
material.fill(0);
});
test('completes stage-before-commit issuance through the concrete file adapter', async (t) => {
const { adapter, targetTokenFile } = fixture(t);
const authority = deliveryAuthority();
const generated = Buffer.alloc(32, 9);
const service = createRecoverableWorkerCredentialIssuer(
authority.port,
adapter,
PEPPER,
{ now: () => 1_000, randomBytes: () => generated },
);
const result = await service.issue({
mutationId: DELIVERY_ID,
requestId: 'request-file-delivery-1',
expectedCurrentVersion: 0,
credentialId: CREDENTIAL_ID,
workerId: 'edge-router-1',
principal: {
subject: { type: 'user', id: 'usr_admin' },
authenticationId: 'session:admin:1',
authenticatedAtMs: 900,
expiresAtMs: 2_000,
assurance: 'multi_factor',
},
notBeforeAtMs: 1_000,
expiresAtMs: 2_000,
previousCredentialId: null,
deploymentTargetDigest: adapter.deploymentTargetDigest,
deploymentGeneration: 'secret-generation-2',
});
assert.equal(result.status, 'published');
assert.equal(result.delivery.state, 'published');
assert.match(fs.readFileSync(targetTokenFile, 'ascii'),
/^ql3w_worker_generation_2_[A-Za-z0-9_-]{43}$/);
assert.equal(JSON.stringify(authority.current()).includes('ql3w_'), false);
assert.equal(generated.every((byte) => byte === 0), true);
});
test('fails closed instead of overwriting an unexpected target generation', async (t) => {
const { adapter, targetTokenFile } = fixture(t);
const unexpected = token('worker_generation_other', 3);
fs.writeFileSync(targetTokenFile, unexpected, { mode: 0o600 });
const material = token(CREDENTIAL_ID, 2);
const candidate = intent(adapter, material);
await adapter.stage(candidate, material);
await assert.rejects(
adapter.publish(committed(candidate)),
WorkerCredentialDeliveryConflictError,
);
assert.deepEqual(fs.readFileSync(targetTokenFile), unexpected);
unexpected.fill(0);
material.fill(0);
});
test('discards only an unpublished exact orphan stage', async (t) => {
const { adapter, targetTokenFile } = fixture(t);
const material = token(CREDENTIAL_ID, 2);
const candidate = intent(adapter, material);
await adapter.stage(candidate, material);
await adapter.discard(candidate);
assert.equal(await adapter.inspect(DELIVERY_ID), null);
await adapter.discard(candidate);
await adapter.stage(candidate, material);
fs.writeFileSync(targetTokenFile, material, { mode: 0o600 });
await assert.rejects(
adapter.discard(candidate),
WorkerCredentialDeliveryConflictError,
);
assert.deepEqual(await adapter.inspect(DELIVERY_ID), candidate);
material.fill(0);
});
test('uses a durable target lock and leaves the old token untouched on contention', async (t) => {
const { adapter, target, targetTokenFile } = fixture(t);
const previous = token(PREVIOUS_CREDENTIAL_ID, 1);
fs.writeFileSync(targetTokenFile, previous, { mode: 0o600 });
fs.writeFileSync(
path.join(target, '.ql3-worker-credential-delivery.lock'),
`${JSON.stringify({ deliveryId: randomUUID() })}\n`,
{ mode: 0o600 },
);
const material = token(CREDENTIAL_ID, 2);
const candidate = intent(adapter, material);
await adapter.stage(candidate, material);
await assert.rejects(
adapter.publish(committed(candidate)),
WorkerCredentialDeliveryUnavailableError,
);
assert.deepEqual(fs.readFileSync(targetTokenFile), previous);
previous.fill(0);
material.fill(0);
});
test('lists only bounded ordered low-sensitive stage intents', async (t) => {
const { adapter, stages } = fixture(t);
const firstToken = token(CREDENTIAL_ID, 2);
const secondToken = token(CREDENTIAL_ID, 3);
const first = intent(adapter, firstToken);
const second = intent(adapter, secondToken, {
deliveryId: '123e4567-e89b-42d3-a456-426614174902',
tokenDigest: workerCredentialDeliveryTokenDigest(secondToken),
deploymentGeneration: 'secret-generation-3',
});
await adapter.stage(second, secondToken);
await adapter.stage(first, firstToken);
assert.deepEqual(await adapter.listStaged({ limit: 1 }), {
stages: [first],
truncated: true,
nextCursor: first.deliveryId,
});
assert.deepEqual(await adapter.listStaged({
afterDeliveryId: first.deliveryId,
limit: 1,
}), {
stages: [second],
truncated: false,
});
fs.writeFileSync(
path.join(stages, `.${DELIVERY_ID}.${randomUUID()}.tmp`),
'uncertain',
{ mode: 0o600 },
);
await assert.rejects(
adapter.listStaged(),
WorkerCredentialDeliveryUnavailableError,
);
firstToken.fill(0);
secondToken.fill(0);
});
test('enforces stage capacity, dedicated roots and live POSIX permissions', async (t) => {
const { adapter, stages, target, targetTokenFile } = fixture(t);
const material = token(CREDENTIAL_ID, 2);
const candidate = intent(adapter, material);
for (let index = 0; index < MAX_WORKER_CREDENTIAL_FILE_STAGES; index += 1) {
fs.writeFileSync(
path.join(stages, `${randomUUID()}.stage`),
'bounded',
{ mode: 0o600 },
);
}
await assert.rejects(
adapter.stage(candidate, material),
WorkerCredentialDeliveryUnavailableError,
);
fs.rmSync(stages, { recursive: true });
fs.mkdirSync(stages, { mode: 0o700 });
const changed = new WorkerCredentialFileDeliveryAdapter({
stageDirectory: stages,
targetTokenFile,
});
const changedCandidate = intent(changed, material);
await changed.stage(changedCandidate, material);
fs.chmodSync(target, 0o755);
await assert.rejects(
changed.publish(committed(changedCandidate)),
WorkerCredentialDeliveryUnavailableError,
);
fs.chmodSync(target, 0o700);
assert.throws(() => new WorkerCredentialFileDeliveryAdapter({
stageDirectory: target,
targetTokenFile,
}));
material.fill(0);
});
@@ -0,0 +1,256 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
WorkerCredentialDeliveryConflictError,
workerCredentialDeliveryTokenDigest,
} = require('@qinglong/runtime-core/worker-credential-delivery');
const {
formatWorkerCredentialToken,
} = require('@qinglong/runtime-core/worker-credential-token');
const {
WorkerCredentialKubernetesDeliveryAdapter,
} = require('@qinglong/cluster-admin/worker-credential-kubernetes-delivery');
const enabled = Boolean(
process.env.QL3_TEST_KUBECONFIG &&
process.env.QL3_TEST_KUBERNETES_NAMESPACE,
);
const integrationTest = enabled ? test : test.skip;
const IDS = [
'223e4567-e89b-42d3-a456-426614174901',
'223e4567-e89b-42d3-a456-426614174902',
'223e4567-e89b-42d3-a456-426614174903',
'223e4567-e89b-42d3-a456-426614174904',
];
function token(credentialId, fill) {
return Buffer.from(formatWorkerCredentialToken(
credentialId,
Buffer.alloc(32, fill).toString('base64url'),
));
}
function intent(adapter, deliveryId, credentialId, material, overrides = {}) {
return {
deliveryId,
workerId: 'integration-worker-1',
credentialId,
credentialVersion: 1,
previousCredentialId: null,
secretDigest: 'b'.repeat(64),
tokenDigest: workerCredentialDeliveryTokenDigest(material),
deploymentTargetDigest: adapter.deploymentTargetDigest,
deploymentGeneration: `generation-${credentialId}`,
stagedAtMs: 1_000,
...overrides,
};
}
function committed(candidate) {
return {
...candidate,
version: 1,
state: 'credential_committed',
credentialCommittedAtMs: candidate.stagedAtMs,
publishedAtMs: null,
publicationDigest: null,
observedAtMs: null,
observedSessionId: null,
observedSessionVersion: null,
previousRevokedAtMs: null,
};
}
integrationTest(
'real Kubernetes API enforces resourceVersion single winner and delete preconditions',
async () => {
const k8s = await import('@kubernetes/client-node');
const config = new k8s.KubeConfig();
config.loadFromFile(process.env.QL3_TEST_KUBECONFIG);
const api = config.makeApiClient(k8s.CoreV1Api);
const deployments = config.makeApiClient(k8s.AppsV1Api);
const namespace = process.env.QL3_TEST_KUBERNETES_NAMESPACE;
const stageNamespace =
process.env.QL3_TEST_KUBERNETES_STAGE_NAMESPACE ??
`${namespace.slice(0, 57)}-stage`;
await api.createNamespace({
body: {
apiVersion: 'v1',
kind: 'Namespace',
metadata: { name: stageNamespace },
},
}).catch((error) => {
if (error?.code !== 409) throw error;
});
await api.createNamespacedSecret({
namespace,
body: {
apiVersion: 'v1',
kind: 'Secret',
type: 'Opaque',
metadata: {
name: 'integration-worker-credential',
labels: {
'app.kubernetes.io/managed-by': 'qinglong3',
'qinglong.io/worker-credential-target': 'prepared-v3',
},
},
data: {},
},
});
await deployments.createNamespacedDeployment({
namespace,
body: {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: {
name: 'integration-worker',
namespace,
labels: { 'app.kubernetes.io/component': 'worker' },
},
spec: {
replicas: 1,
strategy: { type: 'Recreate' },
selector: {
matchLabels: {
app: 'integration-worker',
'app.kubernetes.io/component': 'worker',
},
},
template: {
metadata: {
labels: {
app: 'integration-worker',
'app.kubernetes.io/component': 'worker',
},
},
spec: {
containers: [{
name: 'worker',
image: 'registry.k8s.io/pause:3.10.1',
volumeMounts: [{
name: 'credential',
mountPath: '/credential',
readOnly: true,
}],
}],
volumes: [{
name: 'credential',
projected: {
sources: [{
secret: {
name: 'integration-worker-credential',
items: [{
key: 'credential.token',
path: 'credential-token',
}],
},
}],
},
}],
},
},
},
},
});
const adapter = new WorkerCredentialKubernetesDeliveryAdapter(
api,
deployments,
{
clusterIdentity: 'ql3-k3s-integration',
namespace,
stageNamespace,
targetSecretName: 'integration-worker-credential',
targetDeploymentName: 'integration-worker',
targetDataKey: 'credential.token',
},
);
const first = token('integration_generation_1', 1);
const firstIntent = intent(
adapter,
IDS[0],
'integration_generation_1',
first,
);
await adapter.stage(firstIntent, first);
const initial = await adapter.publish(committed(firstIntent));
assert.match(initial.publicationDigest, /^[0-9a-f]{64}$/);
const initialTarget = await api.readNamespacedSecret({
namespace,
name: 'integration-worker-credential',
});
const second = token('integration_generation_2', 2);
const third = token('integration_generation_3', 3);
const secondIntent = intent(
adapter,
IDS[1],
'integration_generation_2',
second,
{ previousCredentialId: 'integration_generation_1' },
);
const thirdIntent = intent(
adapter,
IDS[2],
'integration_generation_3',
third,
{ previousCredentialId: 'integration_generation_1' },
);
await adapter.stage(secondIntent, second);
await adapter.stage(thirdIntent, third);
const results = await Promise.allSettled([
adapter.publish(committed(secondIntent)),
adapter.publish(committed(thirdIntent)),
]);
assert.equal(
results.filter((result) => result.status === 'fulfilled').length,
1,
);
const rejected = results.find((result) => result.status === 'rejected');
assert.ok(rejected.reason instanceof WorkerCredentialDeliveryConflictError);
const winner = results[0].status === 'fulfilled' ? secondIntent : thirdIntent;
const target = await api.readNamespacedSecret({
namespace,
name: 'integration-worker-credential',
});
assert.notEqual(
target.metadata.resourceVersion,
initialTarget.metadata.resourceVersion,
);
assert.equal(
target.metadata.annotations['qinglong.io/worker-credential-delivery-id'],
winner.deliveryId,
);
const deployment = await deployments.readNamespacedDeployment({
namespace,
name: 'integration-worker',
});
assert.equal(
deployment.spec.template.metadata.annotations[
'qinglong.io/worker-credential-generation'
],
winner.deploymentGeneration,
);
const orphan = token('integration_orphan', 4);
const orphanIntent = intent(
adapter,
IDS[3],
'integration_orphan',
orphan,
);
await adapter.stage(orphanIntent, orphan);
await adapter.discard(orphanIntent);
assert.equal(await adapter.inspect(orphanIntent.deliveryId), null);
const page = await adapter.listStaged({ limit: 4 });
assert.equal(page.stages.some((item) =>
item.deliveryId === orphanIntent.deliveryId), false);
first.fill(0);
second.fill(0);
third.fill(0);
orphan.fill(0);
},
);
@@ -0,0 +1,568 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
WorkerCredentialDeliveryConflictError,
WorkerCredentialDeliveryUnavailableError,
workerCredentialDeliveryTokenDigest,
} = require('@qinglong/runtime-core/worker-credential-delivery');
const {
formatWorkerCredentialToken,
} = require('@qinglong/runtime-core/worker-credential-token');
const {
MAX_WORKER_CREDENTIAL_KUBERNETES_STAGES,
WorkerCredentialKubernetesDeliveryAdapter,
workerCredentialKubernetesDeploymentTargetDigest,
} = require('@qinglong/cluster-admin/worker-credential-kubernetes-delivery');
const IDS = [
'123e4567-e89b-42d3-a456-426614174901',
'123e4567-e89b-42d3-a456-426614174902',
'123e4567-e89b-42d3-a456-426614174903',
];
function apiError(code) {
return Object.assign(new Error(`Kubernetes API ${code}`), { code });
}
function copy(value) {
return structuredClone(value);
}
class FakeKubernetesSecretApi {
constructor() {
this.items = new Map();
this.deployments = new Map();
this.revision = 0;
this.uid = 0;
this.failAfterCreate = false;
this.failAfterReplace = false;
this.failBeforeDeploymentReplace = false;
this.failAfterDeploymentReplace = false;
this.failAfterDelete = false;
this.replacements = [];
this.deploymentReplacements = [];
this.deletions = [];
const preparedTarget = this.serverSecret('qinglong-workers', {
apiVersion: 'v1',
kind: 'Secret',
type: 'Opaque',
metadata: {
name: 'edge-router-1-credential',
annotations: {
'kubectl.kubernetes.io/last-applied-configuration':
'{"kind":"Secret","metadata":{"name":"edge-router-1-credential"}}',
},
labels: {
'app.kubernetes.io/managed-by': 'qinglong3',
'qinglong.io/worker-credential-target': 'prepared-v3',
},
},
data: {},
}, null);
this.items.set(
this.key('qinglong-workers', 'edge-router-1-credential'),
preparedTarget,
);
const deployment = this.serverDeployment('qinglong-workers', {
apiVersion: 'apps/v1',
kind: 'Deployment',
metadata: {
name: 'edge-router-1',
labels: { 'app.kubernetes.io/component': 'worker' },
},
spec: {
replicas: 1,
strategy: { type: 'Recreate' },
template: {
metadata: {
labels: { 'app.kubernetes.io/component': 'worker' },
},
spec: {
containers: [{ name: 'worker', image: 'worker:test' }],
volumes: [{
name: 'projected-authority',
projected: {
sources: [{
secret: {
name: 'edge-router-1-credential',
items: [{
key: 'credential.token',
path: 'credential-token',
}],
},
}],
},
}],
},
},
},
}, null);
this.deployments.set(this.key('qinglong-workers', 'edge-router-1'), deployment);
}
key(namespace, name) {
return `${namespace}/${name}`;
}
serverSecret(namespace, body, current) {
this.revision += 1;
const metadata = body.metadata ?? {};
return {
...copy(body),
metadata: {
...copy(metadata),
namespace,
uid: current?.metadata?.uid ?? `uid-${++this.uid}`,
resourceVersion: String(this.revision),
},
};
}
serverDeployment(namespace, body, current) {
this.revision += 1;
return {
...copy(body),
metadata: {
...copy(body.metadata),
namespace,
uid: current?.metadata?.uid ?? `uid-${++this.uid}`,
resourceVersion: String(this.revision),
},
};
}
async readNamespacedSecret({ namespace, name }) {
const value = this.items.get(this.key(namespace, name));
if (!value) throw apiError(404);
return copy(value);
}
async createNamespacedSecret({ namespace, body }) {
const key = this.key(namespace, body.metadata.name);
if (this.items.has(key)) throw apiError(409);
const created = this.serverSecret(namespace, body, null);
this.items.set(key, created);
if (this.failAfterCreate) {
this.failAfterCreate = false;
throw apiError(409);
}
return copy(created);
}
async replaceNamespacedSecret({ namespace, name, body }) {
const key = this.key(namespace, name);
const current = this.items.get(key);
if (!current) throw apiError(404);
this.replacements.push({
expectedResourceVersion: body.metadata.resourceVersion,
observedResourceVersion: current.metadata.resourceVersion,
});
if (body.metadata.resourceVersion !== current.metadata.resourceVersion) {
throw apiError(409);
}
const replaced = this.serverSecret(namespace, body, current);
this.items.set(key, replaced);
if (this.failAfterReplace) {
this.failAfterReplace = false;
throw apiError(409);
}
return copy(replaced);
}
async deleteNamespacedSecret({ namespace, name, body }) {
const key = this.key(namespace, name);
const current = this.items.get(key);
if (!current) throw apiError(404);
this.deletions.push(copy(body.preconditions));
if (
body.preconditions.uid !== current.metadata.uid ||
body.preconditions.resourceVersion !== current.metadata.resourceVersion
) {
throw apiError(409);
}
this.items.delete(key);
if (this.failAfterDelete) {
this.failAfterDelete = false;
throw apiError(404);
}
return { status: 'Success' };
}
async listNamespacedSecret({ namespace, limit }) {
const values = [...this.items.values()].filter((item) =>
item.metadata.namespace === namespace &&
item.metadata.labels?.['app.kubernetes.io/managed-by'] === 'qinglong3' &&
item.metadata.labels?.['qinglong.io/worker-credential-stage'] === 'v1');
return {
items: copy(values.slice(0, limit)),
metadata: values.length > limit ? { _continue: 'opaque' } : {},
};
}
async readNamespacedDeployment({ namespace, name }) {
const value = this.deployments.get(this.key(namespace, name));
if (!value) throw apiError(404);
return copy(value);
}
async replaceNamespacedDeployment({ namespace, name, body }) {
const key = this.key(namespace, name);
const current = this.deployments.get(key);
if (!current) throw apiError(404);
this.deploymentReplacements.push({
expectedResourceVersion: body.metadata.resourceVersion,
observedResourceVersion: current.metadata.resourceVersion,
});
if (body.metadata.resourceVersion !== current.metadata.resourceVersion) {
throw apiError(409);
}
if (this.failBeforeDeploymentReplace) {
this.failBeforeDeploymentReplace = false;
throw apiError(503);
}
const replaced = this.serverDeployment(namespace, body, current);
this.deployments.set(key, replaced);
if (this.failAfterDeploymentReplace) {
this.failAfterDeploymentReplace = false;
throw apiError(409);
}
return copy(replaced);
}
get(namespace, name) {
return copy(this.items.get(this.key(namespace, name)));
}
getDeployment(namespace, name) {
return copy(this.deployments.get(this.key(namespace, name)));
}
}
function adapter(api = new FakeKubernetesSecretApi()) {
return {
api,
adapter: new WorkerCredentialKubernetesDeliveryAdapter(api, api, {
clusterIdentity: 'cluster-production-a',
namespace: 'qinglong-workers',
stageNamespace: 'qinglong-workers-staging',
targetSecretName: 'edge-router-1-credential',
targetDeploymentName: 'edge-router-1',
targetDataKey: 'credential.token',
}),
};
}
function token(credentialId, fill) {
return Buffer.from(formatWorkerCredentialToken(
credentialId,
Buffer.alloc(32, fill).toString('base64url'),
));
}
function intent(deliveryAdapter, deliveryId, credentialId, material, overrides = {}) {
return {
deliveryId,
workerId: 'edge-router-1',
credentialId,
credentialVersion: 1,
previousCredentialId: null,
secretDigest: 'a'.repeat(64),
tokenDigest: workerCredentialDeliveryTokenDigest(material),
deploymentTargetDigest: deliveryAdapter.deploymentTargetDigest,
deploymentGeneration: `generation-${credentialId}`,
stagedAtMs: 1_000,
...overrides,
};
}
function committed(candidate) {
return {
...candidate,
version: 1,
state: 'credential_committed',
credentialCommittedAtMs: candidate.stagedAtMs,
publishedAtMs: null,
publicationDigest: null,
observedAtMs: null,
observedSessionId: null,
observedSessionVersion: null,
previousRevokedAtMs: null,
};
}
test('plans the exact deployment target digest without Kubernetes authority', () => {
const options = {
clusterIdentity: 'cluster-production-a',
namespace: 'qinglong-workers',
stageNamespace: 'qinglong-workers-staging',
targetSecretName: 'edge-router-1-credential',
targetDeploymentName: 'edge-router-1',
targetDataKey: 'credential.token',
};
const { adapter: delivery } = adapter();
assert.equal(
workerCredentialKubernetesDeploymentTargetDigest(options),
delivery.deploymentTargetDigest,
);
assert.throws(
() => workerCredentialKubernetesDeploymentTargetDigest({
...options,
namespace: options.stageNamespace,
}),
/delivery options are invalid/,
);
});
test('creates one immutable stage and lists only bounded normalized intent', async () => {
const { api, adapter: delivery } = adapter();
const material = token('worker_generation_1', 1);
const candidate = intent(delivery, IDS[0], 'worker_generation_1', material);
api.failAfterCreate = true;
await delivery.stage(candidate, material);
assert.deepEqual(await delivery.inspect(IDS[0]), candidate);
assert.deepEqual(await delivery.listStaged({ limit: 1 }), {
stages: [candidate],
truncated: false,
});
const stored = api.get(
'qinglong-workers-staging',
`ql3w-stage-${IDS[0].replaceAll('-', '')}`,
);
assert.equal(stored.immutable, true);
assert.equal(stored.type, 'qinglong.io/worker-credential-stage-v1');
assert.deepEqual(Object.keys(stored.data), ['credentialToken']);
await delivery.stage(candidate, material);
await assert.rejects(
delivery.stage({ ...candidate, deploymentGeneration: 'drift' }, material),
WorkerCredentialDeliveryConflictError,
);
material.fill(0);
});
test('publishes with resourceVersion CAS and replays a lost update response', async () => {
const { api, adapter: delivery } = adapter();
const first = token('worker_generation_1', 1);
const firstIntent = intent(delivery, IDS[0], 'worker_generation_1', first);
await delivery.stage(firstIntent, first);
const initial = await delivery.publish(committed(firstIntent));
assert.match(initial.publicationDigest, /^[0-9a-f]{64}$/);
const second = token('worker_generation_2', 2);
const secondIntent = intent(
delivery,
IDS[1],
'worker_generation_2',
second,
{ previousCredentialId: 'worker_generation_1' },
);
await delivery.stage(secondIntent, second);
api.failAfterReplace = true;
api.failAfterDeploymentReplace = true;
const rotated = await delivery.publish(committed(secondIntent));
assert.match(rotated.publicationDigest, /^[0-9a-f]{64}$/);
assert.deepEqual(
await delivery.publish(committed(secondIntent)),
rotated,
);
assert.equal(api.replacements.length, 2);
assert.equal(
api.replacements[0].expectedResourceVersion,
api.replacements[0].observedResourceVersion,
);
assert.equal(api.deploymentReplacements.length, 2);
assert.equal(
api.deploymentReplacements[1].expectedResourceVersion,
api.deploymentReplacements[1].observedResourceVersion,
);
const target = api.get('qinglong-workers', 'edge-router-1-credential');
assert.match(
target.metadata.labels['qinglong.io/worker-credential-target-digest'],
/^[A-Za-z0-9_-]{43}$/,
);
assert.deepEqual(
Buffer.from(target.data['credential.token'], 'base64'),
second,
);
const deployment = api.getDeployment('qinglong-workers', 'edge-router-1');
const annotations = deployment.spec.template.metadata.annotations;
assert.equal(
annotations['qinglong.io/worker-credential-generation'],
secondIntent.deploymentGeneration,
);
assert.equal(
annotations['qinglong.io/worker-credential-id'],
secondIntent.credentialId,
);
assert.equal(
annotations['qinglong.io/worker-credential-publication-digest'],
rotated.publicationDigest,
);
first.fill(0);
second.fill(0);
});
test('recovers a Secret-first crash before advancing the Recreate PodTemplate', async () => {
const { api, adapter: delivery } = adapter();
const material = token('worker_generation_1', 1);
const candidate = intent(delivery, IDS[0], 'worker_generation_1', material);
await delivery.stage(candidate, material);
api.failBeforeDeploymentReplace = true;
await assert.rejects(
delivery.publish(committed(candidate)),
WorkerCredentialDeliveryUnavailableError,
);
assert.ok(api.get('qinglong-workers', 'edge-router-1-credential'));
assert.equal(
api.getDeployment('qinglong-workers', 'edge-router-1')
.spec.template.metadata.annotations,
undefined,
);
const recovered = await delivery.publish(committed(candidate));
assert.match(recovered.publicationDigest, /^[0-9a-f]{64}$/);
assert.equal(api.replacements.length, 1);
assert.equal(api.deploymentReplacements.length, 2);
material.fill(0);
});
test('rejects Deployment drift before mutating the target Secret', async () => {
const { api, adapter: delivery } = adapter();
const material = token('worker_generation_1', 1);
const candidate = intent(delivery, IDS[0], 'worker_generation_1', material);
await delivery.stage(candidate, material);
api.deployments.get('qinglong-workers/edge-router-1').spec.strategy.type =
'RollingUpdate';
await assert.rejects(
delivery.publish(committed(candidate)),
WorkerCredentialDeliveryConflictError,
);
const prepared = api.get('qinglong-workers', 'edge-router-1-credential');
assert.equal(
prepared.metadata.labels['qinglong.io/worker-credential-target'],
'prepared-v3',
);
assert.deepEqual(prepared.data, {});
material.fill(0);
});
test('gives two concurrent rotations from one resourceVersion exactly one winner', async () => {
const { adapter: delivery } = adapter();
const first = token('worker_generation_1', 1);
const firstIntent = intent(delivery, IDS[0], 'worker_generation_1', first);
await delivery.stage(firstIntent, first);
await delivery.publish(committed(firstIntent));
const second = token('worker_generation_2', 2);
const third = token('worker_generation_3', 3);
const secondIntent = intent(delivery, IDS[1], 'worker_generation_2', second, {
previousCredentialId: 'worker_generation_1',
});
const thirdIntent = intent(delivery, IDS[2], 'worker_generation_3', third, {
previousCredentialId: 'worker_generation_1',
});
await delivery.stage(secondIntent, second);
await delivery.stage(thirdIntent, third);
const results = await Promise.allSettled([
delivery.publish(committed(secondIntent)),
delivery.publish(committed(thirdIntent)),
]);
assert.equal(results.filter((result) => result.status === 'fulfilled').length, 1);
const rejection = results.find((result) => result.status === 'rejected');
assert.ok(rejection.reason instanceof WorkerCredentialDeliveryConflictError);
first.fill(0);
second.fill(0);
third.fill(0);
});
test('deletes only an exact unpublished stage with UID and resourceVersion fences', async () => {
const { api, adapter: delivery } = adapter();
const material = token('worker_generation_1', 1);
const candidate = intent(delivery, IDS[0], 'worker_generation_1', material);
await delivery.stage(candidate, material);
const staged = api.get(
'qinglong-workers-staging',
`ql3w-stage-${IDS[0].replaceAll('-', '')}`,
);
api.failAfterDelete = true;
await delivery.discard(candidate);
assert.equal(await delivery.inspect(IDS[0]), null);
assert.deepEqual(api.deletions, [{
uid: staged.metadata.uid,
resourceVersion: staged.metadata.resourceVersion,
}]);
await delivery.discard(candidate);
material.fill(0);
});
test('never discards a stage whose token is already the published target', async () => {
const { adapter: delivery } = adapter();
const material = token('worker_generation_1', 1);
const candidate = intent(delivery, IDS[0], 'worker_generation_1', material);
await delivery.stage(candidate, material);
await delivery.publish(committed(candidate));
await assert.rejects(
delivery.discard(candidate),
WorkerCredentialDeliveryConflictError,
);
assert.deepEqual(await delivery.inspect(IDS[0]), candidate);
material.fill(0);
});
test('fails closed when the Kubernetes stage inventory exceeds its hard cap', async () => {
const { api, adapter: delivery } = adapter();
for (let index = 0; index <= MAX_WORKER_CREDENTIAL_KUBERNETES_STAGES; index += 1) {
api.items.set(`synthetic/${index}`, {
metadata: {
namespace: 'qinglong-workers-staging',
labels: {
'app.kubernetes.io/managed-by': 'qinglong3',
'qinglong.io/worker-credential-stage': 'v1',
},
},
});
}
await assert.rejects(
delivery.listStaged(),
WorkerCredentialDeliveryUnavailableError,
);
});
test('fails closed when the prepared target Secret is absent or has drifted', async () => {
const missing = new FakeKubernetesSecretApi();
missing.items.delete('qinglong-workers/edge-router-1-credential');
const { adapter: missingDelivery } = adapter(missing);
const missingMaterial = token('worker_generation_1', 1);
const missingIntent = intent(
missingDelivery,
IDS[0],
'worker_generation_1',
missingMaterial,
);
await missingDelivery.stage(missingIntent, missingMaterial);
await assert.rejects(
missingDelivery.publish(committed(missingIntent)),
WorkerCredentialDeliveryUnavailableError,
);
assert.equal(missing.replacements.length, 0);
const drifted = new FakeKubernetesSecretApi();
drifted.items.get('qinglong-workers/edge-router-1-credential')
.metadata.annotations['unexpected.example/authority'] = 'drift';
const { adapter: driftedDelivery } = adapter(drifted);
const driftedMaterial = token('worker_generation_2', 2);
const driftedIntent = intent(
driftedDelivery,
IDS[1],
'worker_generation_2',
driftedMaterial,
);
await driftedDelivery.stage(driftedIntent, driftedMaterial);
await assert.rejects(
driftedDelivery.publish(committed(driftedIntent)),
WorkerCredentialDeliveryConflictError,
);
assert.equal(drifted.replacements.length, 0);
missingMaterial.fill(0);
driftedMaterial.fill(0);
});
@@ -0,0 +1,284 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
WorkerCredentialDeliveryUnavailableError,
} = require('@qinglong/runtime-core/worker-credential-delivery');
const {
WORKER_CREDENTIAL_KUBERNETES_TOKEN_REQUEST_SECONDS,
WorkerCredentialKubernetesTokenRequestUnavailableError,
createWorkerCredentialKubernetesTokenRequestSession,
} = require(
'@qinglong/cluster-admin/worker-credential-kubernetes-token-request'
);
const NOW_MS = 1_800_000_000_000;
const DELIVERY = Object.freeze({
clusterIdentity: 'cluster-production-a',
stageNamespace: 'qinglong-worker-a-stage',
namespace: 'qinglong-worker-a',
targetSecretName: 'worker-a-credential',
targetDeploymentName: 'worker-a',
targetDataKey: 'credential-token',
});
function segment(value) {
return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url');
}
function jwt(overrides = {}) {
const issuedAt = Math.floor(NOW_MS / 1_000);
const header = { alg: 'RS256', typ: 'JWT', ...(overrides.header ?? {}) };
const claims = {
sub:
'system:serviceaccount:qinglong-worker-a-stage:' +
'ql3-worker-credential-admin',
iat: issuedAt,
exp: issuedAt + WORKER_CREDENTIAL_KUBERNETES_TOKEN_REQUEST_SECONDS,
aud: ['https://kubernetes.default.svc'],
...(overrides.claims ?? {}),
};
return `${segment(header)}.${segment(claims)}.${Buffer.from('signature').toString('base64url')}`;
}
function permission(attributes) {
if (
attributes.namespace === DELIVERY.stageNamespace &&
attributes.resource === 'secrets'
) {
return ['get', 'list', 'create', 'delete'].includes(attributes.verb);
}
if (
attributes.namespace === DELIVERY.namespace &&
attributes.resource === 'secrets' &&
attributes.name === DELIVERY.targetSecretName
) {
return ['get', 'update'].includes(attributes.verb);
}
return attributes.namespace === DELIVERY.namespace &&
attributes.group === 'apps' &&
attributes.resource === 'deployments' &&
attributes.name === DELIVERY.targetDeploymentName &&
['get', 'update'].includes(attributes.verb);
}
function issuerPermission(attributes) {
return attributes.namespace === DELIVERY.stageNamespace &&
attributes.verb === 'create' &&
attributes.resource === 'serviceaccounts' &&
attributes.subresource === 'token' &&
attributes.name === 'ql3-worker-credential-admin';
}
function fixture(options = {}) {
const value = options.token ?? jwt();
const issuedAt = Math.floor(NOW_MS / 1_000);
const response = {
apiVersion: 'authentication.k8s.io/v1',
kind: 'TokenRequest',
status: {
token: value,
expirationTimestamp:
options.expirationTimestamp ??
new Date(
(issuedAt + WORKER_CREDENTIAL_KUBERNETES_TOKEN_REQUEST_SECONDS) *
1_000,
),
},
};
const requests = [];
const receivedTokens = [];
let active = false;
let disposals = 0;
const tokenRequests = {
async createNamespacedServiceAccountToken(request) {
requests.push(structuredClone(request));
if (options.requestError) throw new Error('sensitive upstream failure');
return response;
},
};
const issuerAuthorization = {
async createSelfSubjectAccessReview(request) {
return {
status: {
allowed: options.issuerOverbroad === true
? true
: issuerPermission(request.body.spec.resourceAttributes),
},
};
},
};
const createRestrictedClients = (token) => {
receivedTokens.push(token);
active = true;
const assertActive = () => {
if (!active) {
throw Object.assign(new Error('disposed credential'), { code: 401 });
}
};
return {
secrets: {
async readNamespacedSecret() { assertActive(); throw Object.assign(new Error('absent'), { code: 404 }); },
async createNamespacedSecret() { assertActive(); throw new Error('unused'); },
async replaceNamespacedSecret() { assertActive(); throw new Error('unused'); },
async deleteNamespacedSecret() { assertActive(); throw new Error('unused'); },
async listNamespacedSecret() { assertActive(); return { items: [] }; },
},
deployments: {
async readNamespacedDeployment() { assertActive(); throw new Error('unused'); },
async replaceNamespacedDeployment() { assertActive(); throw new Error('unused'); },
},
authorization: {
async createSelfSubjectAccessReview(request) {
assertActive();
return {
status: {
allowed: options.overbroad === true
? true
: permission(request.body.spec.resourceAttributes),
},
};
},
},
dispose() {
disposals += 1;
active = false;
if (options.disposeError) throw new Error('sensitive dispose failure');
},
};
};
const session = createWorkerCredentialKubernetesTokenRequestSession(
tokenRequests,
issuerAuthorization,
createRestrictedClients,
{
serviceAccountName: 'ql3-worker-credential-admin',
identitySecretName: 'worker-a-identity',
delivery: DELIVERY,
now: () => NOW_MS,
},
);
return {
session,
response,
requests,
receivedTokens,
get active() { return active; },
get disposals() { return disposals; },
};
}
test('mints one bounded token, proves exact RBAC and disposes retained clients', async () => {
const state = fixture();
let retained;
const result = await state.session.withDelivery(async (context) => {
retained = context.delivery;
assert.deepEqual(context.evidence, {
tokenLifetimeSeconds: 600,
issuerAllowedChecks: 1,
issuerDeniedChecks: 8,
allowedChecks: 8,
deniedChecks: 20,
});
assert.match(context.delivery.deploymentTargetDigest, /^[0-9a-f]{64}$/);
return 'published';
});
assert.equal(result, 'published');
assert.deepEqual(state.requests, [{
name: 'ql3-worker-credential-admin',
namespace: DELIVERY.stageNamespace,
body: {
apiVersion: 'authentication.k8s.io/v1',
kind: 'TokenRequest',
spec: { expirationSeconds: 600 },
},
}]);
assert.equal(state.receivedTokens.length, 1);
assert.equal(state.response.status.token, '');
assert.equal(state.disposals, 1);
assert.equal(state.active, false);
await assert.rejects(
retained.inspect('123e4567-e89b-42d3-a456-426614174901'),
WorkerCredentialDeliveryUnavailableError,
);
state.receivedTokens[0] = '';
});
test('rejects an overbroad delivery token before invoking credential work', async () => {
const state = fixture({ overbroad: true });
let invoked = false;
await assert.rejects(
state.session.withDelivery(async () => {
invoked = true;
}),
WorkerCredentialKubernetesTokenRequestUnavailableError,
);
assert.equal(invoked, false);
assert.equal(state.response.status.token, '');
assert.equal(state.disposals, 1);
assert.equal(state.active, false);
state.receivedTokens[0] = '';
});
test('rejects an overbroad token issuer before requesting a token', async () => {
const state = fixture({ issuerOverbroad: true });
await assert.rejects(
state.session.withDelivery(async () => undefined),
WorkerCredentialKubernetesTokenRequestUnavailableError,
);
assert.equal(state.requests.length, 0);
assert.equal(state.receivedTokens.length, 0);
assert.equal(state.disposals, 0);
});
test('rejects malformed, wrong-subject and overlong token responses before client creation', async () => {
const issuedAt = Math.floor(NOW_MS / 1_000);
const cases = [
{ token: 'not-a-jwt' },
{ token: jwt({ header: { alg: 'none' } }) },
{ token: jwt({ claims: { sub: 'system:serviceaccount:other:other' } }) },
{ token: jwt({ claims: { exp: issuedAt + 601 } }), expirationTimestamp: new Date((issuedAt + 601) * 1_000) },
{ token: jwt(), expirationTimestamp: new Date((issuedAt + 599) * 1_000) },
];
for (const value of cases) {
const state = fixture(value);
await assert.rejects(
state.session.withDelivery(async () => undefined),
WorkerCredentialKubernetesTokenRequestUnavailableError,
);
assert.equal(state.receivedTokens.length, 0);
assert.equal(state.response.status.token, '');
}
});
test('maps TokenRequest failures to a stable secret-free error', async () => {
const state = fixture({ requestError: true });
await assert.rejects(
state.session.withDelivery(async () => undefined),
(error) => {
assert.ok(
error instanceof WorkerCredentialKubernetesTokenRequestUnavailableError,
);
assert.doesNotMatch(error.message, /sensitive|upstream/i);
return true;
},
);
assert.equal(state.receivedTokens.length, 0);
});
test('fails closed when restricted client disposal fails', async () => {
const state = fixture({ disposeError: true });
await assert.rejects(
state.session.withDelivery(async () => 'would-have-succeeded'),
(error) => {
assert.ok(
error instanceof WorkerCredentialKubernetesTokenRequestUnavailableError,
);
assert.doesNotMatch(error.message, /sensitive|dispose/i);
return true;
},
);
assert.equal(state.response.status.token, '');
assert.equal(state.disposals, 1);
assert.equal(state.active, false);
state.receivedTokens[0] = '';
});
@@ -0,0 +1,152 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
WorkerCredentialManagementAuthorizationError,
WorkerCredentialManagementConflictError,
WorkerCredentialManagementRequestError,
createClusterWorkerCredentialManagementService,
} = require('@qinglong/cluster-admin/worker-credential-management');
const NOW = 1_000;
const PRINCIPAL = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'operator-a' }),
authenticationId: 'session-operator-a',
authenticatedAtMs: 900,
expiresAtMs: 20_000,
assurance: 'multi_factor',
});
function fixture() {
const plans = new Map();
const queries = [];
const pool = {
async query(text, values) {
queries.push({ text, values });
if (text.includes('FROM "ql3"."projects" AS project')) {
return {
rows: [{
projectId: 'cluster-authority',
projectName: 'Cluster Authority',
projectSlug: 'cluster-authority',
projectStatus: 'active',
projectVersion: 3,
projectCreatedAtMs: 1,
projectUpdatedAtMs: 2,
bindingProjectId: 'cluster-authority',
bindingSubjectType: 'user',
bindingSubjectId: 'operator-a',
bindingVersion: 2,
bindingState: 'active',
bindingRole: 'admin',
bindingMutationId: 'binding-operator-a-v2',
bindingChangedByType: 'user',
bindingChangedById: 'owner-a',
bindingCreatedAtMs: 2,
}],
rowCount: 1,
};
}
if (text.includes('SELECT plan_json')) {
const plan = plans.get(values[0]);
return {
rows: plan ? [{ planJson: plan }] : [],
rowCount: plan ? 1 : 0,
};
}
if (text.includes('INSERT INTO "ql3"."worker_credential_management_plans"')) {
const actionRef = values[0];
if (plans.has(actionRef)) return { rows: [], rowCount: 0 };
plans.set(actionRef, JSON.parse(values[17]));
return { rows: [{ actionRef }], rowCount: 1 };
}
throw new Error(`unexpected query: ${text}`);
},
async connect() {
throw new Error('transaction is not expected by plan');
},
};
return { pool, plans, queries };
}
function request(overrides = {}) {
return {
actionRef: 'worker-credential:worker-a:generation-2',
authorityProjectId: 'cluster-authority',
action: 'rotate',
deliveryId: '123e4567-e89b-42d3-a456-426614174702',
workerId: 'worker-a',
credentialId: 'credential-b',
previousCredentialId: 'credential-a',
credentialNotBeforeAtMs: NOW,
credentialExpiresAtMs: 100_000,
deploymentTargetDigest: '1'.repeat(64),
deploymentGeneration: 'generation-2',
principal: PRINCIPAL,
...overrides,
};
}
test('plans one immutable Worker credential rotation under a strong User fence', async () => {
const state = fixture();
const service = createClusterWorkerCredentialManagementService({
pool: state.pool,
now: () => NOW,
planLifetimeMs: 10_000,
});
const created = await service.plan(request());
const replay = await service.plan(request());
assert.equal(created.status, 'created');
assert.equal(replay.status, 'existing');
assert.equal(created.plan.requestedBy.type, 'user');
assert.equal(created.plan.requestedBy.id, 'operator-a');
assert.equal(created.plan.plannedAtMs, NOW);
assert.equal(created.plan.expiresAtMs, 11_000);
assert.equal(created.plan.target.credentialNotBeforeAtMs, NOW);
assert.match(created.plan.planDigest, /^[0-9a-f]{64}$/);
assert.match(created.plan.previewDigest, /^[0-9a-f]{64}$/);
assert.doesNotMatch(JSON.stringify(created.plan), /token|kubeconfig|secret/i);
assert.equal(state.plans.size, 1);
});
test('rejects weak Users and widened manager configuration before persistence', async () => {
const state = fixture();
const service = createClusterWorkerCredentialManagementService({
pool: state.pool,
now: () => NOW,
});
await assert.rejects(
service.plan(request({
principal: { ...PRINCIPAL, assurance: 'single_factor' },
})),
WorkerCredentialManagementAuthorizationError,
);
assert.equal(state.plans.size, 0);
assert.throws(
() => createClusterWorkerCredentialManagementService({
pool: state.pool,
debug: true,
}),
WorkerCredentialManagementRequestError,
);
});
test('maps invalid plans and semantic replay drift to stable management errors', async () => {
const state = fixture();
const service = createClusterWorkerCredentialManagementService({
pool: state.pool,
now: () => NOW,
});
await assert.rejects(
service.plan(request({ credentialNotBeforeAtMs: NOW - 1 })),
WorkerCredentialManagementRequestError,
);
await service.plan(request());
await assert.rejects(
service.plan(request({ deploymentGeneration: 'generation-3' })),
WorkerCredentialManagementConflictError,
);
assert.equal(state.plans.size, 1);
});
@@ -0,0 +1,279 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createClusterWorkerCredentialManagementService,
} = require('@qinglong/cluster-admin/worker-credential-management');
const REQUESTER = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'operator-a' }),
authenticationId: 'session-operator-a',
authenticatedAtMs: 900,
expiresAtMs: 20_000,
assurance: 'multi_factor',
});
const REVIEWER = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'reviewer-b' }),
authenticationId: 'session-reviewer-b',
authenticatedAtMs: 900,
expiresAtMs: 20_000,
assurance: 'hardware',
});
function approvalFixture() {
const plans = new Map();
const approvals = new Map();
const audits = new Map();
const queries = [];
let releases = 0;
const query = async (text, values = []) => {
queries.push({ text, values });
if (text.includes('FROM "ql3"."projects" AS project')) {
const subjectId = values[2];
return {
rows: [{
projectId: 'cluster-authority',
projectName: 'Cluster Authority',
projectSlug: 'cluster-authority',
projectStatus: 'active',
projectVersion: 3,
projectCreatedAtMs: 1,
projectUpdatedAtMs: 2,
bindingProjectId: 'cluster-authority',
bindingSubjectType: 'user',
bindingSubjectId: subjectId,
bindingVersion: 2,
bindingState: 'active',
bindingRole: 'admin',
bindingMutationId: `binding-${subjectId}-v2`,
bindingChangedByType: 'user',
bindingChangedById: 'owner-a',
bindingCreatedAtMs: 2,
}],
rowCount: 1,
};
}
if (text.includes('SELECT plan_json')) {
const plan = plans.get(values[0]);
return { rows: plan ? [{ planJson: plan }] : [], rowCount: plan ? 1 : 0 };
}
if (text.includes('INSERT INTO "ql3"."worker_credential_management_plans"')) {
if (plans.has(values[0])) return { rows: [], rowCount: 0 };
plans.set(values[0], JSON.parse(values[17]));
return { rows: [{ actionRef: values[0] }], rowCount: 1 };
}
if (text.includes('FROM "ql3"."approval_requests"')) {
const stored = approvals.get(values[0]);
return {
rows: stored
? [{ requestJson: stored.request, requestDigest: stored.digest }]
: [],
rowCount: stored ? 1 : 0,
};
}
if (text.includes('FROM "ql3"."security_audit_events"')) {
const stored = audits.get(values[0]);
return { rows: stored ? [stored] : [], rowCount: stored ? 1 : 0 };
}
if (text.includes('"ql3"."lock_approval_policy_fence"')) {
return { rows: [{ matches: true }], rowCount: 1 };
}
if (text.includes('INSERT INTO "ql3"."approval_requests"')) {
approvals.set(values[0], {
request: JSON.parse(values[14]),
digest: values[15],
});
return { rows: [], rowCount: 1 };
}
if (text.includes('UPDATE "ql3"."approval_requests"')) {
const current = approvals.get(values[8]);
if (!current || current.request.version !== values[9]) {
return { rows: [], rowCount: 0 };
}
approvals.set(values[8], {
request: JSON.parse(values[5]),
digest: values[6],
});
return { rows: [], rowCount: 1 };
}
if (text.includes('INSERT INTO "ql3"."security_audit_events"')) {
audits.set(values[0], {
eventId: values[0],
requestId: values[1],
operationId: values[2],
projectId: values[3],
subjectType: values[4],
subjectId: values[5],
authenticationId: values[6],
outcome: values[7],
reasonsJson: JSON.parse(values[8]),
fenceProjectVersion: values[9],
fenceBindingVersion: values[10],
occurredAtMs: values[11],
});
return { rows: [], rowCount: 1 };
}
if (
text === 'BEGIN ISOLATION LEVEL SERIALIZABLE' ||
text === 'COMMIT' ||
text === 'ROLLBACK' ||
text.includes("SELECT set_config(")
) {
return { rows: [], rowCount: 0 };
}
throw new Error(`unexpected query: ${text}`);
};
const pool = {
query,
async connect() {
return {
query,
release() {
releases += 1;
},
};
},
};
return { pool, plans, approvals, audits, queries, releases: () => releases };
}
function planRequest() {
return {
actionRef: 'worker-credential:worker-a:generation-2',
authorityProjectId: 'cluster-authority',
action: 'rotate',
deliveryId: '123e4567-e89b-42d3-a456-426614174702',
workerId: 'worker-a',
credentialId: 'credential-b',
previousCredentialId: 'credential-a',
credentialNotBeforeAtMs: 1_000,
credentialExpiresAtMs: 100_000,
deploymentTargetDigest: '1'.repeat(64),
deploymentGeneration: 'generation-2',
principal: REQUESTER,
};
}
test('binds proposal, separate approval and inspection to one immutable plan', async () => {
const state = approvalFixture();
let now = 1_000;
const service = createClusterWorkerCredentialManagementService({
pool: state.pool,
now: () => now,
planLifetimeMs: 10_000,
approvalLifetimeMs: 5_000,
});
const planned = await service.plan(planRequest());
const proposed = await service.propose({
actionRef: planned.plan.actionRef,
authorityProjectId: planned.plan.authorityProjectId,
approvalRequestId: 'approval-worker-a-generation-2',
approvalAuditEventId: '123e4567-e89b-42d3-a456-426614174703',
principal: REQUESTER,
});
assert.equal(proposed.approvalStatus, 'created');
assert.equal(proposed.approvalRequest.state, 'pending');
assert.equal(proposed.approvalRequest.version, 1);
assert.equal(proposed.approvalRequest.decisionMode, 'separation_of_duty');
assert.equal(proposed.approvalRequest.risk, 'high');
assert.deepEqual(proposed.approvalRequest.action, {
permission: 'worker.manage',
actionType: 'worker_credential.delivery.rotate',
actionRef: planned.plan.actionRef,
actionDigest: planned.plan.planDigest,
previewDigest: planned.plan.previewDigest,
});
now = 1_100;
const decided = await service.decide({
actionRef: planned.plan.actionRef,
authorityProjectId: planned.plan.authorityProjectId,
approvalRequestId: proposed.approvalRequest.id,
expectedVersion: 1,
decisionId: 'decision-worker-a-generation-2',
auditEventId: '123e4567-e89b-42d3-a456-426614174704',
decision: 'approved',
reasonCode: 'reviewed',
principal: REVIEWER,
});
const replay = await service.decide({
actionRef: planned.plan.actionRef,
authorityProjectId: planned.plan.authorityProjectId,
approvalRequestId: proposed.approvalRequest.id,
expectedVersion: 1,
decisionId: 'decision-worker-a-generation-2',
auditEventId: '123e4567-e89b-42d3-a456-426614174704',
decision: 'approved',
reasonCode: 'reviewed',
principal: REVIEWER,
});
const inspection = await service.inspectAuthorized({
actionRef: planned.plan.actionRef,
authorityProjectId: planned.plan.authorityProjectId,
approvalRequestId: proposed.approvalRequest.id,
inspectionId: 'inspection-worker-a-generation-2',
principal: REVIEWER,
});
assert.equal(decided.status, 'decided');
assert.equal(decided.request.state, 'approved');
assert.equal(decided.request.version, 2);
assert.equal(decided.request.decidedBy.id, 'reviewer-b');
assert.equal(replay.status, 'existing');
assert.equal(inspection.stale, false);
assert.equal(inspection.plan.planDigest, planned.plan.planDigest);
assert.equal(inspection.approvalRequest.state, 'approved');
assert.equal(state.approvals.size, 1);
assert.equal(state.audits.size, 2);
assert.equal(state.releases(), 2);
});
test('authorizes and consumes durable quota before reading management state', async () => {
const state = approvalFixture();
const quotaCalls = [];
const service = createClusterWorkerCredentialManagementService({
pool: state.pool,
now: () => 1_000,
quota: {
async consume(command) {
assert.equal(
state.queries.some(({ text }) =>
text.includes('FROM "ql3"."projects" AS project'),
),
true,
);
quotaCalls.push({
...command,
planReads: state.queries.filter(({ text }) =>
text.includes('SELECT plan_json'),
).length,
});
return { admitted: true, retryAfterMs: null };
},
},
});
const planned = await service.plan(planRequest());
assert.equal(quotaCalls[0].operation, 'worker-credential.plan');
const readsBefore = state.queries.filter(({ text }) =>
text.includes('SELECT plan_json'),
).length;
await assert.rejects(
service.propose({
actionRef: planned.plan.actionRef,
authorityProjectId: 'other-project',
approvalRequestId: 'approval-other-project',
approvalAuditEventId: '123e4567-e89b-42d3-a456-426614174799',
principal: REQUESTER,
}),
);
assert.equal(
state.queries.filter(({ text }) => text.includes('SELECT plan_json')).length,
readsBefore + 1,
);
assert.equal(quotaCalls.length, 2);
assert.equal(quotaCalls[1].operation, 'worker-credential.propose');
assert.equal(quotaCalls[1].planReads, readsBefore);
});
@@ -0,0 +1,280 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const {
chmodSync,
mkdtempSync,
readFileSync,
realpathSync,
rmSync,
writeFileSync,
} = require('node:fs');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { afterEach, test } = require('node:test');
const {
executeClusterWorkerCredentialManagementClient,
validateClusterWorkerCredentialManagementClientResult,
} = require('@qinglong/cluster-admin/worker-credential-management-client');
const CLIENT_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/client-key.pem',
);
const CLIENT_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/client-cert.pem',
);
const SERVER_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-key.pem',
);
const CA_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/ca-cert.pem',
);
const temporaryDirectories = [];
const CLI = resolve(
__dirname,
'../dist/worker-credential/workerCredentialManagementClientCli.js',
);
const subject = Object.freeze({ type: 'user', id: 'operator-a' });
const plan = Object.freeze({
actionRef: 'worker-credential:worker-1:rotate:1',
authorityProjectId: 'cluster-authority',
action: 'rotate',
target: Object.freeze({
deliveryId: 'delivery-1',
workerId: 'worker-1',
credentialId: 'credential-2',
previousCredentialId: 'credential-1',
credentialNotBeforeAtMs: 1_000,
credentialExpiresAtMs: 2_000,
deploymentTargetDigest: 'a'.repeat(64),
deploymentGeneration: 'generation-2',
}),
requestedBy: subject,
plannedAtMs: 900,
expiresAtMs: 1_800,
previewDigest: 'b'.repeat(64),
planDigest: 'c'.repeat(64),
});
const approval = Object.freeze({
id: 'approval-1',
projectId: 'cluster-authority',
version: 2,
state: 'approved',
risk: 'high',
decisionMode: 'four_eyes',
requestedBy: subject,
requestedAtMs: 910,
expiresAtMs: 1_800,
decision: 'approved',
decisionReasonCode: 'reviewed',
decidedBy: Object.freeze({ type: 'user', id: 'reviewer-b' }),
decidedAtMs: 920,
dispatchId: null,
consumedAtMs: null,
actionType: 'worker_credential.delivery.rotate',
actionRef: plan.actionRef,
actionDigest: 'd'.repeat(64),
previewDigest: plan.previewDigest,
});
function command(operation) {
return {
operation,
request: {},
};
}
function privateWrite(filePath, value) {
writeFileSync(filePath, value, { mode: 0o600 });
chmodSync(filePath, 0o600);
}
function clientFiles(options = {}) {
const directory = realpathSync(
mkdtempSync(join(tmpdir(), 'ql3-worker-client-')),
);
temporaryDirectories.push(directory);
const paths = {
configFile: join(directory, 'client.json'),
commandFile: join(directory, 'command.json'),
assertionFile: join(directory, 'assertion.jwt'),
};
const caFile = join(directory, 'ca.crt');
const clientCertificateFile = join(directory, 'client.crt');
const clientPrivateKeyFile = join(directory, 'client.key');
privateWrite(caFile, readFileSync(CA_CERT));
privateWrite(clientCertificateFile, readFileSync(CLIENT_CERT));
privateWrite(
clientPrivateKeyFile,
readFileSync(options.mismatchedKey ? SERVER_KEY : CLIENT_KEY),
);
const config = {
schemaVersion: 1,
endpoint:
'https://manager.example.test:8444/api/v3/worker-credentials/management',
servername: 'manager.example.test',
caFile,
...(options.omitClientIdentity
? {}
: { clientCertificateFile, clientPrivateKeyFile }),
requestTimeoutMs: 1_000,
};
privateWrite(paths.configFile, `${JSON.stringify(config)}\n`);
privateWrite(
paths.commandFile,
`${JSON.stringify({
schemaVersion: 1,
operation: 'worker-credential.inspect',
request: {
actionRef: 'worker-credential:worker-1:rotate:1',
authorityProjectId: 'cluster-authority',
approvalRequestId: 'approval-worker-1',
inspectionId: 'inspection-worker-1',
},
})}\n`,
);
privateWrite(
paths.assertionFile,
'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl',
);
return { paths, clientPrivateKeyFile };
}
afterEach(() => {
for (const directory of temporaryDirectories.splice(0)) {
rmSync(directory, { recursive: true, force: true });
}
});
test('validates all four low-sensitive Worker management results', () => {
const fixtures = [
[
'worker-credential.plan',
{ schemaVersion: 1, operation: 'worker-credential.plan', status: 'created', plan },
],
[
'worker-credential.propose',
{
schemaVersion: 1,
operation: 'worker-credential.propose',
approvalStatus: 'created',
plan,
approval,
},
],
[
'worker-credential.decide',
{
schemaVersion: 1,
operation: 'worker-credential.decide',
status: 'decided',
approval,
},
],
[
'worker-credential.inspect',
{
schemaVersion: 1,
operation: 'worker-credential.inspect',
plan,
approval,
stale: false,
},
],
];
for (const [operation, result] of fixtures) {
assert.equal(
validateClusterWorkerCredentialManagementClientResult(
result,
command(operation),
).operation,
operation,
);
assert.doesNotMatch(JSON.stringify(result), /authenticationId|token|secret/i);
}
});
test('rejects widened and secret-bearing response shapes', () => {
assert.throws(() =>
validateClusterWorkerCredentialManagementClientResult(
{
schemaVersion: 1,
operation: 'worker-credential.inspect',
plan: null,
approval: { ...approval, authenticationId: 'must-not-leak' },
stale: false,
},
command('worker-credential.inspect'),
),
);
assert.throws(() =>
validateClusterWorkerCredentialManagementClientResult(
{
schemaVersion: 1,
operation: 'worker-credential.execute',
status: 'completed',
},
command('worker-credential.inspect'),
),
);
});
test('requires one matching private client certificate identity before connect', async () => {
{
const files = clientFiles();
let connects = 0;
await assert.rejects(
executeClusterWorkerCredentialManagementClient(files.paths, {
async connect(target) {
connects += 1;
assert.deepEqual(target, {
hostname: 'manager.example.test',
port: 8444,
});
throw new Error('expected-connect-stop');
},
}),
{ code: 'QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED' },
);
assert.equal(connects, 1);
}
for (const options of [
{ omitClientIdentity: true },
{ mismatchedKey: true },
]) {
const files = clientFiles(options);
let connects = 0;
await assert.rejects(
executeClusterWorkerCredentialManagementClient(files.paths, {
async connect() {
connects += 1;
throw new Error('must not connect');
},
}),
{ code: 'QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_CONFIG_INVALID' },
);
assert.equal(connects, 0);
}
});
test('CLI exposes path-only usage and no credential material', () => {
const help = spawnSync(process.execPath, [CLI, '--help'], {
encoding: 'utf8',
});
assert.equal(help.status, 0);
assert.match(help.stdout, /^Usage: ql3-worker-credential-client /);
assert.doesNotMatch(help.stdout, /token|secret|credential-value/i);
const invalid = spawnSync(process.execPath, [CLI, '--assertion=value'], {
encoding: 'utf8',
});
assert.equal(invalid.status, 64);
assert.match(invalid.stderr, /USAGE_INVALID/);
assert.doesNotMatch(invalid.stderr, /assertion=value/);
});
@@ -0,0 +1,58 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
WorkerCredentialManagementRequestError,
} = require('@qinglong/cluster-admin/worker-credential-management');
const {
runClusterWorkerCredentialExecution,
} = require('@qinglong/cluster-admin/worker-credential-management-executor');
function options(overrides = {}) {
let opens = 0;
let sessions = 0;
const value = {
openDatabase: async () => {
opens += 1;
throw new Error('database must not open');
},
tokenRequestSession: {
async withDelivery() {
sessions += 1;
throw new Error('TokenRequest must not start');
},
},
workerCredentialPepper: 'pepper-value',
actionRef: 'worker-credential:worker-a:generation-2',
approvalRequestId: 'approval-worker-a-generation-2',
consumptionId: 'consume-worker-a-generation-2',
dispatchId: 'dispatch-worker-a-generation-2',
auditEventId: '123e4567-e89b-42d3-a456-426614174705',
confirmAuthorization: async () => {
throw new Error('operator session expired');
},
now: () => 1_000,
...overrides,
};
return { value, opens: () => opens, sessions: () => sessions };
}
test('fails before PostgreSQL and TokenRequest when caller authorization is absent', async () => {
const fixture = options();
await assert.rejects(
runClusterWorkerCredentialExecution(fixture.value),
/operator session expired/,
);
assert.equal(fixture.opens(), 0);
assert.equal(fixture.sessions(), 0);
});
test('rejects widened executor inputs before acquiring any authority', async () => {
const fixture = options({ debug: true });
await assert.rejects(
runClusterWorkerCredentialExecution(fixture.value),
WorkerCredentialManagementRequestError,
);
assert.equal(fixture.opens(), 0);
assert.equal(fixture.sessions(), 0);
});
@@ -0,0 +1,402 @@
const assert = require('node:assert/strict');
const { readFileSync } = require('node:fs');
const { request: httpsRequest } = require('node:https');
const { resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterPluginPackageManagementHttpConfigurationError,
startClusterPluginPackageManagementHttp,
} = require('@qinglong/cluster-admin/plugin-package-management-http');
const {
startClusterWorkerCredentialManagementHttp,
} = require('@qinglong/cluster-admin/worker-credential-management-http');
const {
WorkerCredentialManagementAuthorizationError,
WorkerCredentialManagementConflictError,
WorkerCredentialManagementQuotaExceededError,
WorkerCredentialManagementRequestError,
WorkerCredentialManagementUnavailableError,
} = require('@qinglong/cluster-admin/worker-credential-management');
const SERVER_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-key.pem',
);
const SERVER_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem',
);
const CLIENT_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/client-key.pem',
);
const CLIENT_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/client-cert.pem',
);
const CLIENT_CA = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/ca-cert.pem',
);
const EMPTY_CRL = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/empty-crl.pem',
);
const REVOKED_CLIENT_CRL = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/revoked-client-crl.pem',
);
const NEXT_CLIENT_CERT = resolve(
__dirname,
'fixtures/management-service-cert.pem',
);
const NEXT_CLIENT_KEY = resolve(
__dirname,
'fixtures/management-service-key.pem',
);
const WORKER_PATH = '/api/v3/worker-credentials/management';
function command() {
return {
schemaVersion: 1,
operation: 'worker-credential.inspect',
request: {
actionRef: 'worker-credential:worker-1:rotate:1',
authorityProjectId: 'cluster-authority',
approvalRequestId: 'approval-worker-1',
inspectionId: 'inspection-worker-1',
},
};
}
function identities() {
return {
async reload() {},
bind(assertion) {
assert.equal(assertion, 'assertion-value');
return {
async authenticate() {
return {
subject: { type: 'user', id: 'cluster-reviewer' },
authenticationId: 'authentication-1',
authenticatedAtMs: 900,
expiresAtMs: 10_000,
assurance: 'multi_factor',
};
},
};
},
};
}
async function start(execute, options = {}) {
return startClusterWorkerCredentialManagementHttp({
host: '127.0.0.1',
port: 0,
tls: {
privateKey: Buffer.from(readFileSync(SERVER_KEY)),
certificate: Buffer.from(readFileSync(SERVER_CERT)),
clientCertificateAuthority: Buffer.from(readFileSync(CLIENT_CA)),
clientCertificateRevocationList: Buffer.from(
readFileSync(options.revoked ? REVOKED_CLIENT_CRL : EMPTY_CRL),
),
},
identities: options.identities ?? identities(),
transport: { execute },
limits: { requestTimeoutMs: 2_000, drainTimeoutMs: 500 },
now: () => 1_000,
createRequestId: () => 'worker-request-1',
});
}
async function request(application, path = WORKER_PATH, client = true) {
const body = Buffer.from(JSON.stringify(command()));
return new Promise((resolvePromise, reject) => {
const outgoing = httpsRequest(
{
host: '127.0.0.1',
port: application.address.port,
path,
method: 'POST',
rejectUnauthorized: false,
...(client
? {
cert: readFileSync(CLIENT_CERT),
key: readFileSync(CLIENT_KEY),
}
: {}),
agent: false,
headers: {
authorization: 'Bearer assertion-value',
'content-type': 'application/json',
'content-length': String(body.length),
},
},
(incoming) => {
const chunks = [];
incoming.on('data', (chunk) => chunks.push(chunk));
incoming.once('end', () => {
resolvePromise({
statusCode: incoming.statusCode,
body: JSON.parse(Buffer.concat(chunks).toString('utf8')),
});
});
},
);
outgoing.once('error', reject);
outgoing.end(body);
});
}
async function health(application, path) {
return new Promise((resolvePromise, reject) => {
const outgoing = httpsRequest(
{
host: '127.0.0.1',
port: application.address.port,
path,
method: 'GET',
rejectUnauthorized: false,
agent: false,
},
(incoming) => {
incoming.resume();
incoming.once('end', () => resolvePromise(incoming.statusCode));
},
);
outgoing.once('error', reject);
outgoing.end();
});
}
async function startWithClientTrust(certificateAuthority, execute) {
return startClusterWorkerCredentialManagementHttp({
host: '127.0.0.1',
port: 0,
tls: {
privateKey: Buffer.from(readFileSync(SERVER_KEY)),
certificate: Buffer.from(readFileSync(SERVER_CERT)),
clientCertificateAuthority: certificateAuthority,
clientCertificateRevocationList: Buffer.from(readFileSync(EMPTY_CRL)),
},
identities: identities(),
transport: { execute },
limits: { requestTimeoutMs: 2_000, drainTimeoutMs: 500 },
now: () => 1_000,
createRequestId: () => 'worker-request-rotation',
});
}
async function requestWithClientIdentity(application, certificate, key) {
const body = Buffer.from(JSON.stringify(command()));
return new Promise((resolvePromise, reject) => {
const outgoing = httpsRequest(
{
host: '127.0.0.1',
port: application.address.port,
path: WORKER_PATH,
method: 'POST',
rejectUnauthorized: false,
cert: readFileSync(certificate),
key: readFileSync(key),
agent: false,
headers: {
authorization: 'Bearer assertion-value',
'content-type': 'application/json',
'content-length': String(body.length),
},
},
(incoming) => {
const chunks = [];
incoming.on('data', (chunk) => chunks.push(chunk));
incoming.once('end', () => {
resolvePromise({
statusCode: incoming.statusCode,
body: JSON.parse(Buffer.concat(chunks).toString('utf8')),
});
});
},
);
outgoing.once('error', reject);
outgoing.end(body);
});
}
test('serves only the fixed Worker credential management route', async () => {
const calls = [];
const application = await start(async (value, authentication) => {
calls.push({ value, principal: await authentication.authenticate() });
return {
schemaVersion: 1,
operation: 'worker-credential.inspect',
plan: null,
approval: null,
stale: false,
};
});
try {
const response = await request(application);
assert.equal(response.statusCode, 200);
assert.equal(response.body.requestId, 'worker-request-1');
assert.equal(response.body.result.operation, 'worker-credential.inspect');
assert.deepEqual(calls[0].value, command());
assert.deepEqual(calls[0].principal.subject, {
type: 'user',
id: 'cluster-reviewer',
});
assert.equal(
(await request(application, '/api/v3/plugin-packages/management'))
.statusCode,
404,
);
assert.equal(calls.length, 1);
} finally {
await application.close();
}
});
test('requires an authorized client certificate before OIDC or body parsing', async () => {
let identityBinds = 0;
let transportCalls = 0;
const identity = identities();
const application = await start(
async () => {
transportCalls += 1;
},
{
identities: {
...identity,
bind(assertion) {
identityBinds += 1;
return identity.bind(assertion);
},
},
},
);
try {
assert.equal(await health(application, '/livez'), 200);
assert.equal(await health(application, '/readyz'), 200);
const response = await request(application, WORKER_PATH, false);
assert.equal(response.statusCode, 401);
assert.equal(response.body.error.code, 'client_certificate_required');
assert.equal(identityBinds, 0);
assert.equal(transportCalls, 0);
} finally {
await application.close();
}
});
test('rejects a CRL-revoked client certificate before OIDC', async () => {
let identityBinds = 0;
const identity = identities();
const application = await start(async () => assert.fail('must not execute'), {
revoked: true,
identities: {
...identity,
bind(assertion) {
identityBinds += 1;
return identity.bind(assertion);
},
},
});
try {
const response = await request(application);
assert.equal(response.statusCode, 401);
assert.equal(response.body.error.code, 'client_certificate_required');
assert.equal(identityBinds, 0);
} finally {
await application.close();
}
});
test('maps Worker credential management failures to stable HTTP errors', async () => {
for (const [failure, statusCode, code] of [
[new WorkerCredentialManagementRequestError('invalid'), 400, 'request_invalid'],
[new WorkerCredentialManagementAuthorizationError(), 403, 'forbidden'],
[new WorkerCredentialManagementConflictError('conflict'), 409, 'conflict'],
[new WorkerCredentialManagementQuotaExceededError(1_250), 429, 'quota_exceeded'],
[new WorkerCredentialManagementUnavailableError(), 503, 'unavailable'],
]) {
const application = await start(async () => {
throw failure;
});
try {
const response = await request(application);
assert.equal(response.statusCode, statusCode);
assert.equal(response.body.error.code, code);
} finally {
await application.close();
}
}
});
test('rejects arbitrary management paths at configuration time', async () => {
const privateKey = Buffer.from(readFileSync(SERVER_KEY));
try {
await assert.rejects(
startClusterPluginPackageManagementHttp({
host: '127.0.0.1',
port: 0,
tls: {
privateKey,
certificate: Buffer.from(readFileSync(SERVER_CERT)),
},
identities: identities(),
transport: { async execute() {} },
managementPath: '/api/v3/arbitrary',
}),
ClusterPluginPackageManagementHttpConfigurationError,
);
} finally {
privateKey.fill(0);
}
});
test('accepts both client CAs during overlap then rejects the retired CA', async () => {
const execute = async () => ({
schemaVersion: 1,
operation: 'worker-credential.inspect',
plan: null,
approval: null,
stale: false,
});
const oldAuthority = Buffer.from(readFileSync(CLIENT_CA));
const nextAuthority = Buffer.from(readFileSync(NEXT_CLIENT_CERT));
const overlap = await startWithClientTrust(
Buffer.concat([oldAuthority, nextAuthority]),
execute,
);
try {
assert.equal((await request(overlap)).statusCode, 200);
assert.equal(
(
await requestWithClientIdentity(
overlap,
NEXT_CLIENT_CERT,
NEXT_CLIENT_KEY,
)
).statusCode,
200,
);
} finally {
await overlap.close();
}
const retired = await startWithClientTrust(nextAuthority, execute);
try {
assert.equal((await request(retired)).statusCode, 401);
assert.equal(
(
await requestWithClientIdentity(
retired,
NEXT_CLIENT_CERT,
NEXT_CLIENT_KEY,
)
).statusCode,
200,
);
} finally {
await retired.close();
}
});
@@ -0,0 +1,662 @@
const assert = require('node:assert/strict');
const { generateKeyPairSync, sign } = require('node:crypto');
const { readFile, writeFile, chmod, mkdtemp, rm } = require('node:fs/promises');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const { test } = require('node:test');
const {
ClusterWorkerCredentialManagementProcessConfigError,
loadClusterWorkerCredentialManagementProcessConfig,
startClusterWorkerCredentialManagementProcess,
} = require('@qinglong/cluster-admin/worker-credential-management-process');
const SERVER_KEY = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-key.pem',
);
const SERVER_CERT = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem',
);
const CLIENT_CA = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/ca-cert.pem',
);
const EMPTY_CRL = resolve(
__dirname,
'../../ql3-cluster-control/test/fixtures/mtls/empty-crl.pem',
);
const MANAGEMENT_SERVICE_CERT = resolve(
__dirname,
'fixtures/management-service-cert.pem',
);
const NOW_MS = Date.UTC(2030, 0, 1);
const WORKER_ISSUER = 'https://identity.example.test/';
const WORKER_AUDIENCE = 'qinglong3-worker-credential-management';
function workerIdentityFixture() {
const kid = 'worker-identity-key-1';
const { privateKey, publicKey } = generateKeyPairSync('ed25519');
const keyset = {
schemaVersion: 1,
generation: 1,
issuer: WORKER_ISSUER,
audience: WORKER_AUDIENCE,
keys: [
{
...publicKey.export({ format: 'jwk' }),
alg: 'EdDSA',
kid,
use: 'sig',
},
],
revokedKids: [],
assuranceMappings: [
{
acr: 'urn:ql3:mfa',
assurance: 'multi_factor',
requiredAmr: ['pwd', 'otp'],
},
],
constraints: {
maxAssertionBytes: 8 * 1024,
maxLifetimeMs: 5 * 60 * 1000,
maxAuthenticationAgeMs: 5 * 60 * 1000,
clockSkewMs: 5 * 1000,
},
};
const assertion = (type, purpose) => {
const header = Buffer.from(
JSON.stringify({ alg: 'EdDSA', kid, typ: type }),
).toString('base64url');
const now = Math.floor(NOW_MS / 1_000);
const payload = Buffer.from(
JSON.stringify({
acr: 'urn:ql3:mfa',
amr: ['pwd', 'otp'],
aud: WORKER_AUDIENCE,
auth_time: now - 10,
exp: now + 120,
iat: now,
iss: WORKER_ISSUER,
jti: `worker-process-${purpose}`,
ql3_purpose: purpose,
sub: 'worker-operator-1',
}),
).toString('base64url');
const signed = `${header}.${payload}`;
return `${signed}.${sign(
null,
Buffer.from(signed, 'ascii'),
privateKey,
).toString('base64url')}`;
};
return {
keyset,
workerAssertion: assertion(
'ql3-worker-credential-management+jwt',
'worker-credential-management',
),
pluginAssertion: assertion(
'ql3-plugin-package-management+jwt',
'plugin-package-management',
),
};
}
function identityLedgerPool() {
let row;
const client = {
async query(statement, parameters = []) {
if (statement === 'BEGIN' || statement === 'COMMIT') return { rows: [] };
if (statement === 'ROLLBACK') return { rows: [] };
if (statement.includes('INSERT INTO')) {
row ??= {
generation: parameters[1],
digest: parameters[2],
issuer: parameters[3],
audience: parameters[4],
activeKeyIds: JSON.parse(parameters[5]),
revokedKeyIds: JSON.parse(parameters[6]),
};
return { rows: [] };
}
if (statement.includes('SELECT generation')) {
return { rows: row === undefined ? [] : [row] };
}
throw new Error('unexpected identity ledger query');
},
release() {},
};
return {
async query() {
throw new Error('construction must not query PostgreSQL');
},
async connect() {
return client;
},
};
}
function enabledEnvironment(paths, overrides = {}) {
return {
QL3_WORKER_CREDENTIAL_MANAGEMENT_ENABLED: 'true',
QL3_PROFILE: 'cluster-admin',
QL3_WORKER_CREDENTIAL_MANAGEMENT_HOST: '127.0.0.1',
QL3_WORKER_CREDENTIAL_MANAGEMENT_PORT: '8444',
QL3_WORKER_CREDENTIAL_MANAGEMENT_TLS_CERT_FILE: paths.certificateFile,
QL3_WORKER_CREDENTIAL_MANAGEMENT_TLS_KEY_FILE: paths.privateKeyFile,
QL3_WORKER_CREDENTIAL_MANAGEMENT_CLIENT_CA_FILE:
paths.clientCertificateAuthorityFile,
QL3_WORKER_CREDENTIAL_MANAGEMENT_CLIENT_CRL_FILE:
paths.clientCertificateRevocationListFile,
QL3_WORKER_CREDENTIAL_MANAGEMENT_IDENTITY_KEYSET_FILE:
paths.identityKeysetFile,
QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_URL:
'postgresql://ql3_worker_credential_manager:secret@postgres.example.test/ql3',
QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_TLS_MODE: 'disable',
QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_ALLOW_INSECURE: 'true',
...overrides,
};
}
function readiness() {
return {
ready: true,
writablePrimary: true,
serverVersionNum: 180004,
serverMajor: 18,
currentUser: 'ql3_worker_credential_manager',
contractName: 'control-core',
contractVersion: 49,
migrationIds: ['pg-0050-worker-credential-management-boundary'],
};
}
function identities() {
let reloads = 0;
return {
get reloads() {
return reloads;
},
provider: {
async reload() {
reloads += 1;
return {
schemaVersion: 1,
generation: 7,
digest: 'worker-keyset-digest',
issuer: 'https://identity.example.test/',
audience: 'qinglong3-worker-credential-management',
activeKeyIds: ['identity-key-7'],
revokedKeyIds: ['identity-key-6'],
};
},
bind() {
throw new Error('HTTP stub must not authenticate');
},
},
};
}
async function tlsFixture(run) {
const directory = await mkdtemp(join(tmpdir(), 'ql3-worker-manager-'));
const paths = {
certificateFile: join(directory, 'tls.crt'),
privateKeyFile: join(directory, 'tls.key'),
clientCertificateAuthorityFile: join(directory, 'client-ca.crt'),
clientCertificateRevocationListFile: join(directory, 'client.crl'),
identityKeysetFile: join(directory, 'keyset.json'),
};
try {
await writeFile(paths.certificateFile, await readFile(SERVER_CERT), {
mode: 0o644,
});
await writeFile(paths.privateKeyFile, await readFile(SERVER_KEY), {
mode: 0o640,
});
await writeFile(
paths.clientCertificateAuthorityFile,
await readFile(CLIENT_CA),
{ mode: 0o644 },
);
await writeFile(
paths.clientCertificateRevocationListFile,
await readFile(EMPTY_CRL),
{ mode: 0o644 },
);
await writeFile(paths.identityKeysetFile, '{}\n', { mode: 0o644 });
return await run(paths);
} finally {
await rm(directory, { recursive: true, force: true });
}
}
test('disabled Worker manager reads no profile, TLS or PostgreSQL authority', async () => {
const reads = [];
const environment = new Proxy(
{ QL3_WORKER_CREDENTIAL_MANAGEMENT_ENABLED: 'false' },
{
get(target, property) {
reads.push(property);
if (property === 'QL3_WORKER_CREDENTIAL_MANAGEMENT_ENABLED') {
return target[property];
}
throw new Error(`disabled config read ${String(property)}`);
},
},
);
let opened = 0;
const runtime = await startClusterWorkerCredentialManagementProcess({
environment,
async openDatabase() {
opened += 1;
throw new Error('must not open');
},
});
assert.equal(runtime.status, 'disabled');
assert.equal(opened, 0);
assert.deepEqual(reads, ['QL3_WORKER_CREDENTIAL_MANAGEMENT_ENABLED']);
await runtime.close();
});
test('loads one explicit Worker manager-only HTTPS and database configuration', () => {
const config = loadClusterWorkerCredentialManagementProcessConfig(
enabledEnvironment({
certificateFile: '/run/ql3-worker-manager/tls.crt',
privateKeyFile: '/run/ql3-worker-manager/tls.key',
clientCertificateAuthorityFile:
'/run/ql3-worker-manager/client-ca.crt',
clientCertificateRevocationListFile:
'/run/ql3-worker-manager/client.crl',
identityKeysetFile: '/run/ql3-worker-manager/keyset.json',
}),
);
assert.equal(config.enabled, true);
assert.equal(config.profile, 'cluster-admin');
assert.equal(config.port, 8444);
assert.equal(
config.clientCertificateAuthorityFile,
'/run/ql3-worker-manager/client-ca.crt',
);
assert.equal(
config.clientCertificateRevocationListFile,
'/run/ql3-worker-manager/client.crl',
);
assert.equal(config.database.connection.tls.mode, 'disable');
assert.equal(config.database.pool.maxConnections, 2);
assert.equal(
config.database.pool.applicationName,
'qinglong3-worker-credential-manager',
);
assert.deepEqual(config.quota, {
windowMs: 60_000,
planLimit: 30,
proposeLimit: 30,
decideLimit: 60,
inspectLimit: 600,
});
assert.equal(config.planLifetimeMs, 15 * 60_000);
assert.equal(config.approvalLifetimeMs, 15 * 60_000);
});
test('rejects profile drift, implicit insecure PostgreSQL and unsafe bounds', () => {
const paths = {
certificateFile: '/run/ql3-worker-manager/tls.crt',
privateKeyFile: '/run/ql3-worker-manager/tls.key',
clientCertificateAuthorityFile:
'/run/ql3-worker-manager/client-ca.crt',
clientCertificateRevocationListFile:
'/run/ql3-worker-manager/client.crl',
identityKeysetFile: '/run/ql3-worker-manager/keyset.json',
};
for (const environment of [
enabledEnvironment(paths, { QL3_PROFILE: 'cluster-control' }),
enabledEnvironment(paths, {
QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_ALLOW_INSECURE: 'false',
}),
enabledEnvironment(paths, {
QL3_WORKER_CREDENTIAL_MANAGEMENT_TLS_KEY_FILE: 'relative.key',
}),
enabledEnvironment(paths, {
QL3_WORKER_CREDENTIAL_MANAGEMENT_CLIENT_CA_FILE: 'relative-ca.crt',
}),
enabledEnvironment(paths, {
QL3_WORKER_CREDENTIAL_MANAGEMENT_CLIENT_CRL_FILE: 'relative.crl',
}),
enabledEnvironment(paths, {
QL3_WORKER_CREDENTIAL_MANAGEMENT_INSPECT_QUOTA: '1001',
}),
]) {
assert.throws(
() => loadClusterWorkerCredentialManagementProcessConfig(environment),
ClusterWorkerCredentialManagementProcessConfigError,
);
}
});
test('starts after manager readiness and identity validation then closes in order', async () => {
await tlsFixture(async (paths) => {
const order = [];
const identity = identities();
const pool = {
async query() {
throw new Error('construction must not query PostgreSQL');
},
async connect() {
throw new Error('construction must not acquire PostgreSQL');
},
};
let privateKey;
let httpOptions;
const runtime = await startClusterWorkerCredentialManagementProcess({
environment: enabledEnvironment(paths),
identities: identity.provider,
async openDatabase() {
order.push('database.open');
return {
pool,
async close() {
order.push('database.close');
},
};
},
async assertReady(observedPool) {
order.push('database.ready');
assert.equal(observedPool, pool);
return readiness();
},
async startHttp(options) {
order.push('http.start');
httpOptions = options;
privateKey = options.tls.privateKey;
assert.equal(
privateKey.some((value) => value !== 0),
true,
);
return {
status: 'active',
address: { host: '127.0.0.1', port: 9444 },
availabilityStatus: () => 'ready',
withdraw() {},
async close() {
order.push('http.close');
},
};
},
now: () => NOW_MS,
});
assert.equal(runtime.status, 'active');
assert.deepEqual(order, ['database.open', 'database.ready', 'http.start']);
assert.equal(identity.reloads, 1);
assert.equal(
privateKey.every((value) => value === 0),
true,
);
assert.equal(typeof httpOptions.transport.execute, 'function');
assert.equal(httpOptions.identities, identity.provider);
assert.deepEqual(runtime.identity.activeKeyIds, ['identity-key-7']);
assert.equal(runtime.database.contractVersion, 49);
await Promise.all([runtime.close(), runtime.close()]);
assert.deepEqual(order, [
'database.open',
'database.ready',
'http.start',
'http.close',
'database.close',
]);
});
});
test('assembles the default Worker identity purpose without accepting Plugin assertions', async () => {
await tlsFixture(async (paths) => {
const identity = workerIdentityFixture();
await writeFile(
paths.identityKeysetFile,
`${JSON.stringify(identity.keyset)}\n`,
{ mode: 0o644 },
);
const pool = identityLedgerPool();
let capturedIdentities;
const runtime = await startClusterWorkerCredentialManagementProcess({
environment: enabledEnvironment(paths),
async openDatabase() {
return { pool, async close() {} };
},
async assertReady() {
return readiness();
},
async startHttp(options) {
capturedIdentities = options.identities;
return {
status: 'active',
address: { host: '127.0.0.1', port: 9444 },
availabilityStatus: () => 'ready',
withdraw() {},
async close() {},
};
},
now: () => NOW_MS,
});
assert.ok(capturedIdentities);
assert.deepEqual(
(await capturedIdentities.bind(identity.workerAssertion).authenticate())
.subject,
{ type: 'user', id: 'worker-operator-1' },
);
await assert.rejects(
capturedIdentities.bind(identity.pluginAssertion).authenticate(),
{ code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
);
await runtime.close();
});
});
test('closes manager database when readiness or HTTP startup fails', async () => {
await tlsFixture(async (paths) => {
for (const failureAt of ['readiness', 'http']) {
let closes = 0;
await assert.rejects(
startClusterWorkerCredentialManagementProcess({
environment: enabledEnvironment(paths),
identities: identities().provider,
async openDatabase() {
return {
pool: {
async query() {
throw new Error('must not query');
},
async connect() {
throw new Error('must not connect');
},
},
async close() {
closes += 1;
},
};
},
async assertReady() {
if (failureAt === 'readiness') {
throw new Error('readiness failed');
}
return readiness();
},
async startHttp() {
throw new Error('HTTP failed');
},
}),
new RegExp(`${failureAt} failed`, 'i'),
);
assert.equal(closes, 1);
}
});
});
test('rejects publicly readable private TLS authority before listener start', async () => {
await tlsFixture(async (paths) => {
await chmod(paths.privateKeyFile, 0o644);
let starts = 0;
await assert.rejects(
startClusterWorkerCredentialManagementProcess({
environment: enabledEnvironment(paths),
identities: identities().provider,
async openDatabase() {
return {
pool: {
async query() {
throw new Error('must not query');
},
async connect() {
throw new Error('must not connect');
},
},
async close() {},
};
},
async assertReady() {
return readiness();
},
async startHttp() {
starts += 1;
throw new Error('must not start');
},
}),
ClusterWorkerCredentialManagementProcessConfigError,
);
assert.equal(starts, 0);
});
});
test('accepts a bounded client CA overlap bundle before listener start', async () => {
await tlsFixture(async (paths) => {
await writeFile(
paths.clientCertificateAuthorityFile,
Buffer.concat([
await readFile(CLIENT_CA),
await readFile(MANAGEMENT_SERVICE_CERT),
]),
{ mode: 0o644 },
);
let starts = 0;
const runtime = await startClusterWorkerCredentialManagementProcess({
environment: enabledEnvironment(paths),
identities: identities().provider,
async openDatabase() {
return {
pool: {
async query() {
throw new Error('must not query');
},
async connect() {
throw new Error('must not connect');
},
},
async close() {},
};
},
async assertReady() {
return readiness();
},
async startHttp() {
starts += 1;
return {
status: 'active',
address: { host: '127.0.0.1', port: 9444 },
availabilityStatus: () => 'ready',
withdraw() {},
async close() {},
};
},
now: () => NOW_MS,
});
assert.equal(starts, 1);
await runtime.close();
});
});
test('rejects malformed or unbounded client trust before listener start', async () => {
await tlsFixture(async (paths) => {
const authority = await readFile(CLIENT_CA);
const leaf = await readFile(SERVER_CERT);
const revocationList = await readFile(EMPTY_CRL);
const invalidConfigurations = [
{
authority: Buffer.concat([authority, authority]),
revocationList,
},
{
authority: Buffer.concat([authority, Buffer.from('unexpected\n')]),
revocationList,
},
{ authority: leaf, revocationList },
{
authority: Buffer.concat(Array.from({ length: 17 }, () => authority)),
revocationList,
},
{
authority,
revocationList: Buffer.concat([revocationList, revocationList]),
},
{
authority,
revocationList: Buffer.from(
'-----BEGIN X509 CRL-----\ninvalid\n-----END X509 CRL-----\n',
),
},
{
authority,
revocationList,
now: Date.UTC(2050, 0, 1),
},
{ authority, revocationList, now: -1 },
];
for (const invalid of invalidConfigurations) {
await writeFile(paths.clientCertificateAuthorityFile, invalid.authority, {
mode: 0o644,
});
await writeFile(
paths.clientCertificateRevocationListFile,
invalid.revocationList,
{ mode: 0o644 },
);
let starts = 0;
let closes = 0;
await assert.rejects(
startClusterWorkerCredentialManagementProcess({
environment: enabledEnvironment(paths),
identities: identities().provider,
async openDatabase() {
return {
pool: {
async query() {
throw new Error('must not query');
},
async connect() {
throw new Error('must not connect');
},
},
async close() {
closes += 1;
},
};
},
async assertReady() {
return readiness();
},
async startHttp() {
starts += 1;
throw new Error('must not start');
},
now: () => invalid.now ?? NOW_MS,
}),
ClusterWorkerCredentialManagementProcessConfigError,
);
assert.equal(starts, 0);
assert.equal(closes, 1);
}
});
});
@@ -0,0 +1,263 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
createWorkerCredentialManagementPlan,
} = require('@qinglong/runtime-core/worker-credential-management-plan');
const {
ClusterWorkerCredentialManagementTransportAuthenticationError,
ClusterWorkerCredentialManagementTransportRequestError,
ClusterWorkerCredentialManagementTransportUnavailableError,
createClusterWorkerCredentialManagementTransport,
} = require('@qinglong/cluster-admin/worker-credential-management-transport');
const REQUESTER = Object.freeze({ type: 'user', id: 'operator-a' });
const REVIEWER = Object.freeze({ type: 'user', id: 'reviewer-b' });
const FENCE = Object.freeze({ projectVersion: 1, bindingVersion: 1 });
function principal(subject = REQUESTER, assurance = 'multi_factor') {
return Object.freeze({
subject,
authenticationId: `session-${subject.id}`,
authenticatedAtMs: 900,
expiresAtMs: 10_000,
assurance,
});
}
function plan() {
return createWorkerCredentialManagementPlan({
actionRef: 'worker-credential:worker-a:generation-2',
authorityProjectId: 'cluster-authority',
action: 'rotate',
target: {
deliveryId: '123e4567-e89b-42d3-a456-426614174901',
workerId: 'worker-a',
credentialId: 'credential-generation-2',
previousCredentialId: 'credential-generation-1',
credentialNotBeforeAtMs: 1_000,
credentialExpiresAtMs: 9_000,
deploymentTargetDigest: 'd'.repeat(64),
deploymentGeneration: 'generation-2',
},
requestedBy: REQUESTER,
plannedAtMs: 1_000,
expiresAtMs: 5_000,
});
}
function approval(planValue) {
return createApprovalRequest({
id: 'approval-worker-a-generation-2',
projectId: planValue.authorityProjectId,
action: {
permission: 'worker.manage',
actionType: 'worker_credential.delivery.rotate',
actionRef: planValue.actionRef,
actionDigest: planValue.planDigest,
previewDigest: planValue.previewDigest,
},
risk: 'high',
decisionMode: 'separation_of_duty',
requestedBy: REQUESTER,
requestedAtMs: 1_001,
expiresAtMs: 5_000,
requestFence: FENCE,
});
}
function commands() {
return [
{
schemaVersion: 1,
operation: 'worker-credential.plan',
request: {
actionRef: 'worker-credential:worker-a:generation-2',
authorityProjectId: 'cluster-authority',
action: 'rotate',
deliveryId: '123e4567-e89b-42d3-a456-426614174901',
workerId: 'worker-a',
credentialId: 'credential-generation-2',
previousCredentialId: 'credential-generation-1',
credentialNotBeforeAtMs: 1_000,
credentialExpiresAtMs: 9_000,
deploymentTargetDigest: 'd'.repeat(64),
deploymentGeneration: 'generation-2',
},
},
{
schemaVersion: 1,
operation: 'worker-credential.propose',
request: {
actionRef: 'worker-credential:worker-a:generation-2',
authorityProjectId: 'cluster-authority',
approvalRequestId: 'approval-worker-a-generation-2',
approvalAuditEventId: '123e4567-e89b-42d3-a456-426614174902',
},
},
{
schemaVersion: 1,
operation: 'worker-credential.decide',
request: {
actionRef: 'worker-credential:worker-a:generation-2',
authorityProjectId: 'cluster-authority',
approvalRequestId: 'approval-worker-a-generation-2',
expectedVersion: 1,
decisionId: 'decision-worker-a-generation-2',
auditEventId: '123e4567-e89b-42d3-a456-426614174903',
decision: 'approved',
reasonCode: 'reviewed',
},
},
{
schemaVersion: 1,
operation: 'worker-credential.inspect',
request: {
actionRef: 'worker-credential:worker-a:generation-2',
authorityProjectId: 'cluster-authority',
approvalRequestId: 'approval-worker-a-generation-2',
inspectionId: 'inspection-worker-a-generation-2',
},
},
];
}
test('routes the four public commands with strong User authority and low-sensitive results', async () => {
const planValue = plan();
const approvalValue = approval(planValue);
const calls = [];
const service = {
async plan(request) {
calls.push(['plan', request]);
return { status: 'created', plan: planValue };
},
async propose(request) {
calls.push(['propose', request]);
return {
plan: planValue,
approvalStatus: 'created',
approvalRequest: approvalValue,
};
},
async decide(request) {
calls.push(['decide', request]);
return { status: 'decided', request: approvalValue };
},
async inspectAuthorized(request) {
calls.push(['inspect', request]);
return {
plan: planValue,
approvalRequest: approvalValue,
stale: false,
};
},
};
const transport = createClusterWorkerCredentialManagementTransport({
service,
now: () => 1_100,
});
const authentication = {
async authenticate() {
return principal();
},
};
const results = [];
for (const command of commands()) {
results.push(await transport.execute(command, authentication));
}
assert.deepEqual(
calls.map(([kind]) => kind),
['plan', 'propose', 'decide', 'inspect'],
);
for (const [, request] of calls) {
assert.deepEqual(request.principal, principal());
}
assert.deepEqual(
results.map(({ operation }) => operation),
[
'worker-credential.plan',
'worker-credential.propose',
'worker-credential.decide',
'worker-credential.inspect',
],
);
assert.equal(results[0].plan.planDigest, planValue.planDigest);
assert.equal(results[1].approval.actionDigest, planValue.planDigest);
assert.equal(results[3].stale, false);
const serialized = JSON.stringify(results);
assert.doesNotMatch(serialized, /authenticationId|credential-token|secret/i);
});
test('rejects weak or unavailable identity before management authority', async () => {
let calls = 0;
const service = Object.fromEntries(
['plan', 'propose', 'decide', 'inspectAuthorized'].map((name) => [
name,
async () => {
calls += 1;
throw new Error('must not call service');
},
]),
);
const transport = createClusterWorkerCredentialManagementTransport({
service,
now: () => 1_100,
});
await assert.rejects(
transport.execute(commands()[0], {
async authenticate() {
return principal(REVIEWER, 'service');
},
}),
ClusterWorkerCredentialManagementTransportAuthenticationError,
);
await assert.rejects(
transport.execute(commands()[0], {
async authenticate() {
throw new Error('identity provider unavailable');
},
}),
ClusterWorkerCredentialManagementTransportUnavailableError,
);
assert.equal(calls, 0);
});
test('rejects widened and internal commands before authentication', async () => {
const transport = createClusterWorkerCredentialManagementTransport({
service: {
async plan() {},
async propose() {},
async decide() {},
async inspectAuthorized() {},
},
});
let authentications = 0;
const authentication = {
async authenticate() {
authentications += 1;
return principal();
},
};
await assert.rejects(
transport.execute(
{ ...commands()[0], debug: true },
authentication,
),
ClusterWorkerCredentialManagementTransportRequestError,
);
await assert.rejects(
transport.execute(
{
schemaVersion: 1,
operation: 'worker-credential.execute',
request: {},
},
authentication,
),
ClusterWorkerCredentialManagementTransportRequestError,
);
assert.equal(authentications, 0);
});