qinglong/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs

const assert = require('node:assert/strict');
const { chmod, mkdtemp, rm, writeFile } = require('node:fs/promises');
const { tmpdir } = require('node:os');
const { join } = require('node:path');
const { test } = require('node:test');

const {
  ProductionClusterAiConfigError,
  loadProductionClusterAiConfig,
  startProductionClusterAiControlApplication,
} = require('@qinglong/cluster-control/ai-production');
const {
  canonicalPluginPackagePromptOutputKeyringManifest,
  PLUGIN_PACKAGE_PROMPT_OUTPUT_KEYRING_MANIFEST_SCHEMA,
} = require('@qinglong/ai/plugin-package-prompt-output-keyring-manifest');
const {
  canonicalClusterToolInvocationKeyringManifest,
  CLUSTER_TOOL_INVOCATION_KEYRING_MANIFEST_SCHEMA,
} = require('@qinglong/cluster-control/trusted-tool-invocation-keyring');
const {
  canonicalClusterToolResultKeyringManifest,
  CLUSTER_TOOL_RESULT_KEYRING_MANIFEST_SCHEMA,
} = require('@qinglong/cluster-control/trusted-tool-result-keyring');
const {
  canonicalClusterCopilotFailureDiagnosisOutputKeyringManifest,
  CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_KEYRING_MANIFEST_SCHEMA,
} = require('@qinglong/cluster-control/failure-diagnosis-output-keyring');

function enabledEnvironment(overrides = {}) {
  return {
    QL3_CLUSTER_AI_ENABLED: 'true',
    QL3_CLUSTER_AI_PROVIDER_AUTHORITY_FILE:
      '/var/run/qinglong/ai/providers.json',
    QL3_CLUSTER_AI_SECRET_ROOT: '/var/run/qinglong/ai/provider-secrets',
    ...overrides,
  };
}

function controlConfig() {
  return {
    enabled: true,
    profile: 'cluster-control',
    http: { host: '127.0.0.1', port: 5800 },
    database: {
      connection: {
        host: '127.0.0.1',
        port: 5432,
        database: 'qinglong',
        user: 'ql3_runtime',
        password: 'test-only',
        tls: { mode: 'disable' },
      },
      pool: { maxConnections: 8 },
    },
    security: { apiCredentialPepper: 'test-only' },
  };
}

test('AI config is fail-closed and bounded behind the explicit process flag', () => {
  assert.throws(
    () => loadProductionClusterAiConfig({}),
    (error) =>
      error instanceof ProductionClusterAiConfigError &&
      error.code === 'QL3_CLUSTER_AI_CONFIG_INVALID',
  );
  assert.deepEqual(loadProductionClusterAiConfig(enabledEnvironment()), {
    enabled: true,
    providerAuthorityFile: '/var/run/qinglong/ai/providers.json',
    secretRootDirectory: '/var/run/qinglong/ai/provider-secrets',
    maxConcurrent: 4,
    recoveryLimit: 32,
    databaseMaxConnections: 4,
  });
  assert.throws(
    () =>
      loadProductionClusterAiConfig(
        enabledEnvironment({ QL3_CLUSTER_AI_MAX_CONCURRENT: '65' }),
      ),
    /between 1 and 64/,
  );
  assert.throws(
    () =>
      loadProductionClusterAiConfig(
        enabledEnvironment({
          QL3_CLUSTER_AI_PROMPT_OUTPUT_ENABLED: 'true',
        }),
      ),
    /QL3_CLUSTER_AI_PROMPT_OUTPUT_KEYRING_ROOT is invalid/,
  );
  assert.throws(
    () =>
      loadProductionClusterAiConfig(
        enabledEnvironment({ QL3_CLUSTER_AI_COPILOT_ENABLED: 'true' }),
      ),
    /QL3_CLUSTER_AI_COPILOT_CONFIG_FILE is invalid/,
  );
  assert.deepEqual(
    loadProductionClusterAiConfig(
      enabledEnvironment({
        QL3_CLUSTER_AI_COPILOT_ENABLED: 'true',
        QL3_CLUSTER_AI_COPILOT_CONFIG_FILE: '/run/ql3/copilot/config.json',
        QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT: '/run/ql3/invocation',
        QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT: '/run/ql3/result',
        QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT: '/run/ql3/output',
      }),
    ).copilot,
    {
      configFile: '/run/ql3/copilot/config.json',
      invocationKeyringRootDirectory: '/run/ql3/invocation',
      resultKeyringRootDirectory: '/run/ql3/result',
      outputKeyringRootDirectory: '/run/ql3/output',
    },
  );
});

async function projectedFile(root, name, bytes) {
  await writeFile(join(root, name), bytes, { mode: 0o440 });
  await chmod(join(root, name), 0o440);
}

test('Copilot composition is explicit, shares the Prompt gateway and injects one route capability', async () => {
  const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-'));
  const configRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-'));
  const invocationRoot = await mkdtemp(
    join(tmpdir(), 'ql3-copilot-invocation-'),
  );
  const resultRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-result-'));
  const outputRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-output-'));
  const key = Buffer.alloc(32, 0x55).toString('base64url');
  const config = Buffer.from(
    `${JSON.stringify({
      schema: 'qinglong/cluster-copilot-failure-diagnosis-config@v1',
      provider: 'provider-primary',
      model: 'model-diagnosis',
      modelBoundary: 'external',
      responseLanguage: 'zh-CN',
      maxOutputTokens: 512,
      executionTimeoutMs: 60_000,
      egressPolicy: {
        schema: 'qinglong/copilot-model-egress-policy@v1',
        revision: 'cluster-copilot-v1',
        potentiallySensitiveDataBoundaries: ['external'],
        maxInputBytes: 64 * 1024,
        maxOutputTokens: 1024,
      },
    })}\n`,
  );
  const invocation = canonicalClusterToolInvocationKeyringManifest({
    schema: CLUSTER_TOOL_INVOCATION_KEYRING_MANIFEST_SCHEMA,
    activeKeyId: 'invocation-key-1',
    keys: { 'invocation-key-1': key },
  });
  const result = canonicalClusterToolResultKeyringManifest({
    schema: CLUSTER_TOOL_RESULT_KEYRING_MANIFEST_SCHEMA,
    keys: { 'result-key-1': key },
  });
  const output = canonicalClusterCopilotFailureDiagnosisOutputKeyringManifest({
    schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_KEYRING_MANIFEST_SCHEMA,
    activeKeyId: 'output-key-1',
    keys: { 'output-key-1': key },
  });
  const gateway = {
    generate() {},
    supportsSuccessfulCompletionSink(sink) {
      return sink === registeredSink;
    },
  };
  const fakePool = { query() {}, connect() {} };
  const artifactStore = { put() {}, inspect() {}, readLogRange() {} };
  const copilot = Object.freeze({ execute() {} });
  const copilotRead = Object.freeze({ inspect() {}, readOutput() {} });
  const copilotCancellation = Object.freeze({ cancel() {} });
  let registeredSink;
  let created;
  let createdRead;
  let createdCancellation;
  let controlOptions;
  try {
    await Promise.all([
      projectedFile(configRoot, 'config.json', config),
      projectedFile(invocationRoot, 'keyring.json', invocation),
      projectedFile(resultRoot, 'keyring.json', result),
      projectedFile(outputRoot, 'keyring.json', output),
    ]);
    const application = await startProductionClusterAiControlApplication({
      control: {
        config: controlConfig(),
        workerIngress: {
          config: { enabled: true },
          artifactStore,
        },
      },
      ai: {
        enabled: true,
        providerAuthorityFile: '/unused/providers.json',
        secretRootDirectory: secretRoot,
        copilot: {
          configFile: join(configRoot, 'config.json'),
          invocationKeyringRootDirectory: invocationRoot,
          resultKeyringRootDirectory: resultRoot,
          outputKeyringRootDirectory: outputRoot,
        },
        maxConcurrent: 1,
        recoveryLimit: 1,
        databaseMaxConnections: 1,
      },
      audit() {},
      async openAiDatabase() {
        return { pool: fakePool, async close() {} };
      },
      async bootstrapPrompt(options) {
        await options.openDatabase();
        registeredSink = options.createAdditionalSuccessfulCompletion({
          async recordWithAtomicSuccess() {},
        });
        return {
          status: 'active',
          profile: 'cluster',
          readiness: {},
          capability: gateway,
          prompts: {},
          promptCatalog: {},
          promptExecutions: {},
          promptExecutionInspections: {},
          async stop() {
            return 'stopped';
          },
        };
      },
      async createCopilot(options) {
        created = options;
        return copilot;
      },
      createCopilotRead(options) {
        createdRead = options;
        return copilotRead;
      },
      createCopilotCancellation(options) {
        createdCancellation = options;
        return copilotCancellation;
      },
      async startControl(options) {
        controlOptions = options;
        return {
          status: 'active',
          address: { host: '127.0.0.1', port: 5800 },
          evidence: {},
          recovery: { safe: true, remaining: 0, failed: 0 },
          unavailable: new Promise(() => {}),
          availabilityStatus() {
            return 'ready';
          },
          async stop() {
            return 'stopped';
          },
        };
      },
    });
    assert.equal(application.copilot, copilot);
    assert.equal(created.pool, fakePool);
    assert.equal(created.gateway, gateway);
    assert.equal(created.successfulCompletion, registeredSink);
    assert.equal(created.artifactStore, artifactStore);
    assert.equal(createdRead.pool, fakePool);
    assert.equal(typeof createdRead.prepared.outputKeys.resolve, 'function');
    assert.equal(createdCancellation.pool, fakePool);
    assert.equal(controlOptions.copilotFailureDiagnosis.capability, copilot);
    assert.equal(
      controlOptions.copilotFailureDiagnosis.readCapability,
      copilotRead,
    );
    assert.equal(
      controlOptions.copilotFailureDiagnosis.cancellationCapability,
      copilotCancellation,
    );
    assert.equal(await application.stop(), 'stopped');
  } finally {
    config.fill(0);
    invocation.fill(0);
    result.fill(0);
    output.fill(0);
    await Promise.all([
      rm(secretRoot, { recursive: true, force: true }),
      rm(configRoot, { recursive: true, force: true }),
      rm(invocationRoot, { recursive: true, force: true }),
      rm(resultRoot, { recursive: true, force: true }),
      rm(outputRoot, { recursive: true, force: true }),
    ]);
  }
});

test('explicit AI composition injects one reviewed Prompt capability and drains it after HTTP control', async () => {
  const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-'));
  const events = [];
  const promptCatalog = Object.freeze({ inspect() {} });
  const promptExecutions = Object.freeze({ execute() {} });
  const promptExecutionInspections = Object.freeze({ inspectAuthorized() {} });
  let promptOptions;
  let controlOptions;
  const neverUnavailable = new Promise(() => {});
  try {
    const application = await startProductionClusterAiControlApplication({
      control: { config: controlConfig() },
      ai: {
        enabled: true,
        providerAuthorityFile: '/unused/providers.json',
        secretRootDirectory: secretRoot,
        maxConcurrent: 3,
        recoveryLimit: 11,
        databaseMaxConnections: 2,
      },
      audit() {},
      async bootstrapPrompt(options) {
        promptOptions = options;
        return {
          status: 'active',
          profile: 'cluster',
          readiness: {},
          capability: {},
          prompts: {},
          promptCatalog,
          promptExecutions,
          promptExecutionInspections,
          async stop() {
            events.push('stop-prompt');
            return 'stopped';
          },
        };
      },
      async startControl(options) {
        controlOptions = options;
        return {
          status: 'active',
          address: { host: '127.0.0.1', port: 5800 },
          evidence: {},
          recovery: { safe: true, remaining: 0, failed: 0 },
          unavailable: neverUnavailable,
          availabilityStatus() {
            return 'ready';
          },
          async stop() {
            events.push('stop-control');
            return 'stopped';
          },
        };
      },
    });

    assert.equal(application.status, 'active');
    assert.equal(application.availabilityStatus(), 'ready');
    assert.equal(promptOptions.enabled, true);
    assert.equal(promptOptions.maxConcurrent, 3);
    assert.equal(promptOptions.recoveryLimit, 11);
    assert.equal(controlOptions.promptCatalog.capability, promptCatalog);
    assert.equal(controlOptions.promptExecution.capability, promptExecutions);
    assert.equal(
      controlOptions.promptExecutionInspection.capability,
      promptExecutionInspections,
    );
    assert.equal('promptOutputRead' in controlOptions, false);
    assert.equal(await application.stop(), 'stopped');
    assert.equal(await application.stop(), 'stopped');
    assert.deepEqual(events, ['stop-control', 'stop-prompt']);
  } finally {
    await rm(secretRoot, { recursive: true, force: true });
  }
});

test('output-enabled AI composition wires exact and request-keyed protected reads', async () => {
  const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-'));
  const outputRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-output-'));
  const manifest = canonicalPluginPackagePromptOutputKeyringManifest({
    schema: PLUGIN_PACKAGE_PROMPT_OUTPUT_KEYRING_MANIFEST_SCHEMA,
    generation: 1,
    activeKeyId: 'prompt-key-1',
    keys: { 'prompt-key-1': Buffer.alloc(32, 7).toString('base64url') },
    retirements: {},
  });
  await writeFile(join(outputRoot, 'keyring.json'), manifest, { mode: 0o440 });
  await chmod(join(outputRoot, 'keyring.json'), 0o440);
  let promptOptions;
  let controlOptions;
  const promptOutputs = Object.freeze({ read() {} });
  const promptExecutionOutputs = Object.freeze({ read() {} });
  try {
    const application = await startProductionClusterAiControlApplication({
      control: { config: controlConfig() },
      ai: {
        enabled: true,
        providerAuthorityFile: '/unused/providers.json',
        secretRootDirectory: secretRoot,
        promptOutputKeyringRootDirectory: outputRoot,
        maxConcurrent: 1,
        recoveryLimit: 1,
        databaseMaxConnections: 1,
      },
      audit() {},
      async bootstrapPrompt(options) {
        promptOptions = options;
        return {
          status: 'active',
          profile: 'cluster',
          readiness: {},
          capability: {},
          prompts: {},
          promptCatalog: { inspect() {} },
          promptExecutions: { execute() {} },
          promptExecutionInspections: { inspectAuthorized() {} },
          promptOutputs,
          promptExecutionOutputs,
          async stop() {
            return 'stopped';
          },
        };
      },
      async startControl(options) {
        controlOptions = options;
        return {
          status: 'active',
          address: { host: '127.0.0.1', port: 5800 },
          evidence: {},
          recovery: { safe: true, remaining: 0, failed: 0 },
          unavailable: new Promise(() => {}),
          availabilityStatus() {
            return 'ready';
          },
          async stop() {
            return 'stopped';
          },
        };
      },
    });
    assert.equal(typeof promptOptions.promptOutputKeys.resolve, 'function');
    assert.equal(
      typeof promptOptions.promptOutputRead.authorizer.authorize,
      'function',
    );
    assert.equal(controlOptions.promptOutputRead.capability, promptOutputs);
    assert.equal(
      controlOptions.promptExecutionOutputRead.capability,
      promptExecutionOutputs,
    );
    assert.equal(await application.stop(), 'stopped');
  } finally {
    manifest.fill(0);
    await rm(secretRoot, { recursive: true, force: true });
    await rm(outputRoot, { recursive: true, force: true });
  }
});

test('AI composition fails closed and drains a non-active Prompt bootstrap', async () => {
  const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-'));
  let stops = 0;
  try {
    await assert.rejects(
      startProductionClusterAiControlApplication({
        control: { config: controlConfig() },
        ai: {
          enabled: true,
          providerAuthorityFile: '/unused/providers.json',
          secretRootDirectory: secretRoot,
          maxConcurrent: 1,
          recoveryLimit: 1,
          databaseMaxConnections: 1,
        },
        audit() {},
        async bootstrapPrompt() {
          return {
            status: 'disabled',
            profile: 'cluster',
            async stop() {
              stops += 1;
              return 'stopped';
            },
          };
        },
        async startControl() {
          throw new Error('control must not start');
        },
      }),
      /Prompt application did not activate/,
    );
    assert.equal(stops, 1);
  } finally {
    await rm(secretRoot, { recursive: true, force: true });
  }
});