qinglong/packages/ql3-cluster-control/test/httpSurface.test.cjs

const assert = require('node:assert/strict');
const { readFileSync } = require('node:fs');
const http = require('node:http');
const https = require('node:https');
const path = require('node:path');
const { test } = require('node:test');
const {
  ClusterControlAdmissionDrainTimeoutError,
  ClusterControlHttpConfigurationError,
  startClusterControlHttpSurface,
} = require('@qinglong/cluster-control/http');

const EVIDENCE = Object.freeze({
  contractName: 'control-core',
  contractVersion: 2,
  serverMajor: 16,
  migrationIds: Object.freeze([
    'pg-0001-schema-capability',
    'pg-0002-run-core',
    'pg-0003-run-retry-policy',
  ]),
});
const MTLS_FIXTURES = path.join(__dirname, 'fixtures', 'mtls');
const MTLS = Object.freeze({
  privateKey: readFileSync(path.join(MTLS_FIXTURES, 'server-key.pem')),
  certificateChain: readFileSync(path.join(MTLS_FIXTURES, 'server-cert.pem')),
  clientCertificateAuthorities: Object.freeze([
    readFileSync(path.join(MTLS_FIXTURES, 'ca-cert.pem')),
  ]),
});

function pipeline(handler) {
  return {
    async prepare(metadata) {
      return {
        handle(body) {
          return handler({ ...metadata, body });
        },
      };
    },
  };
}

function request(address, options = {}) {
  const rawBody = options.rawBody;
  const body =
    rawBody === undefined
      ? options.body === undefined
        ? undefined
        : Buffer.from(JSON.stringify(options.body))
      : Buffer.from(rawBody);
  const headers = { connection: 'close', ...options.headers };
  if (body && headers['content-length'] === undefined) {
    headers['content-length'] = String(body.byteLength);
  }
  if (options.body !== undefined && headers['content-type'] === undefined) {
    headers['content-type'] = 'application/json';
  }
  return new Promise((resolve, reject) => {
    const outgoing = http.request(
      {
        host: address.host,
        port: address.port,
        method: options.method ?? 'GET',
        path: options.path ?? '/',
        headers,
      },
      (response) => {
        const chunks = [];
        response.on('data', (chunk) => chunks.push(chunk));
        response.on('end', () => {
          const text = Buffer.concat(chunks).toString('utf8');
          resolve({
            statusCode: response.statusCode,
            headers: response.headers,
            body: text.length === 0 ? null : JSON.parse(text),
          });
        });
      },
    );
    outgoing.on('error', reject);
    if (body) outgoing.write(body);
    outgoing.end();
  });
}

function secureRequest(address, withClientCertificate, options = {}) {
  return new Promise((resolve, reject) => {
    const outgoing = https.request(
      {
        host: '127.0.0.1',
        servername: 'localhost',
        port: address.port,
        path: options.path ?? '/livez',
        method: options.method ?? 'GET',
        agent: options.agent,
        ca: readFileSync(path.join(MTLS_FIXTURES, 'ca-cert.pem')),
        ...(withClientCertificate
          ? {
              key: readFileSync(path.join(MTLS_FIXTURES, 'client-key.pem')),
              cert: readFileSync(path.join(MTLS_FIXTURES, 'client-cert.pem')),
            }
          : {}),
        minVersion: 'TLSv1.3',
        maxVersion: 'TLSv1.3',
      },
      (response) => {
        const tlsProtocol = response.socket.getProtocol();
        const chunks = [];
        response.on('data', (chunk) => chunks.push(chunk));
        response.on('end', () => {
          resolve({
            statusCode: response.statusCode,
            tlsProtocol,
            headers: response.headers,
            body: JSON.parse(Buffer.concat(chunks).toString('utf8')),
          });
        });
      },
    );
    outgoing.once('error', reject);
    outgoing.end();
  });
}

test('requires a trusted client certificate on the TLS 1.3 surface', async (t) => {
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
    mutualTls: MTLS,
  });
  t.after(() => surface.close());

  await assert.rejects(secureRequest(surface.address, false));
  const trusted = await secureRequest(surface.address, true);
  assert.equal(trusted.statusCode, 200);
  assert.equal(trusted.tlsProtocol, 'TLSv1.3');
  assert.deepEqual(trusted.body, { status: 'live' });
});

test('reloads mTLS trust and CRLs without rebinding the listener', async (t) => {
  const emptyCrl = readFileSync(path.join(MTLS_FIXTURES, 'empty-crl.pem'));
  const revokedClientCrl = readFileSync(
    path.join(MTLS_FIXTURES, 'revoked-client-crl.pem'),
  );
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
    mutualTls: {
      ...MTLS,
      certificateRevocationLists: [emptyCrl],
    },
  });
  t.after(() => surface.close());
  const address = { ...surface.address };

  assert.equal((await secureRequest(address, true)).statusCode, 200);
  assert.equal(
    surface.reloadMutualTls({
      ...MTLS,
      certificateRevocationLists: [revokedClientCrl],
    }),
    2,
  );
  assert.deepEqual(surface.address, address);
  await assert.rejects(secureRequest(address, true));

  assert.equal(
    surface.reloadMutualTls({
      ...MTLS,
      certificateRevocationLists: [emptyCrl],
    }),
    3,
  );
  assert.equal((await secureRequest(address, true)).statusCode, 200);

  assert.throws(
    () =>
      surface.reloadMutualTls({
        ...MTLS,
        certificateChain: 'not a certificate',
        certificateRevocationLists: [emptyCrl],
      }),
    ClusterControlHttpConfigurationError,
  );
  assert.equal((await secureRequest(address, true)).statusCode, 200);
});

test('forces pre-reload keep-alive sockets to reconnect before routing', async (t) => {
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
    mutualTls: MTLS,
  });
  t.after(() => surface.close());
  const agent = new https.Agent({ keepAlive: true, maxSockets: 1 });
  t.after(() => agent.destroy());
  let entered;
  let release;
  const enteredPromise = new Promise((resolve) => {
    entered = resolve;
  });
  const releasePromise = new Promise((resolve) => {
    release = resolve;
  });
  const dispose = surface.installAdmission(
    EVIDENCE,
    pipeline(async () => {
      entered();
      await releasePromise;
      return { statusCode: 200, body: { status: 'completed' } };
    }),
  );
  t.after(() => dispose());

  const active = secureRequest(surface.address, true, {
    agent,
    path: '/api/v3/hold',
  });
  await enteredPromise;
  assert.equal(surface.reloadMutualTls(MTLS), 2);
  release();
  assert.equal((await active).statusCode, 200);

  const stale = await secureRequest(surface.address, true, { agent });
  assert.equal(stale.statusCode, 503);
  assert.deepEqual(stale.body, { code: 'tls_context_reloaded' });
  assert.equal(stale.headers.connection, 'close');
  assert.equal(
    (await secureRequest(surface.address, true, { agent })).statusCode,
    200,
  );
});

test('exposes probes but rejects API work until admission is installed', async (t) => {
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
  });
  t.after(() => surface.close());

  const live = await request(surface.address, { path: '/livez' });
  assert.equal(live.statusCode, 200);
  assert.deepEqual(live.body, { status: 'live' });
  assert.equal(live.headers['cache-control'], 'no-store');
  const invalidProbeMethod = await request(surface.address, {
    method: 'POST',
    path: '/livez',
  });
  assert.equal(invalidProbeMethod.statusCode, 405);
  assert.deepEqual(invalidProbeMethod.body, { code: 'method_not_allowed' });
  const waiting = await request(surface.address, { path: '/readyz' });
  assert.equal(waiting.statusCode, 503);
  assert.deepEqual(waiting.body, { status: 'not_ready' });
  const rejected = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/runs',
    headers: { connection: 'keep-alive' },
    body: { mustNotBeRead: true },
  });
  assert.equal(rejected.statusCode, 503);
  assert.equal(rejected.headers.connection, 'close');

  let observed;
  const dispose = surface.installAdmission(
    EVIDENCE,
    pipeline(async (incoming) => {
      observed = incoming;
      return { statusCode: 201, body: { accepted: true } };
    }),
  );
  const ready = await request(surface.address, { path: '/readyz' });
  assert.equal(ready.statusCode, 200);
  assert.deepEqual(ready.body, { status: 'ready' });
  const admitted = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/runs?tag=a&tag=b',
    headers: { 'x-request-id': 'request-123' },
    body: { taskId: 'task-1' },
  });
  assert.equal(admitted.statusCode, 201);
  assert.deepEqual(admitted.body, { accepted: true });
  assert.equal(admitted.headers['x-request-id'], 'request-123');
  assert.equal(observed.requestId, 'request-123');
  assert.equal(observed.method, 'POST');
  assert.equal(observed.path, '/api/v3/runs');
  assert.deepEqual(observed.query.tag, ['a', 'b']);
  assert.deepEqual(observed.body, { taskId: 'task-1' });

  await dispose();
  assert.equal(
    (await request(surface.address, { path: '/readyz' })).statusCode,
    503,
  );
  assert.equal(
    (
      await request(surface.address, {
        method: 'POST',
        path: '/api/v3/runs',
        body: { ignored: true },
      })
    ).statusCode,
    503,
  );
});

test('enforces bounded JSON requests and responses without leaking failures', async (t) => {
  const diagnostics = [];
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
    maxBodyBytes: 1024,
    maxResponseBytes: 1024,
    onError(diagnostic) {
      diagnostics.push(diagnostic);
    },
  });
  t.after(() => surface.close());
  const dispose = surface.installAdmission(
    EVIDENCE,
    pipeline(async (incoming) => {
      if (incoming.path.endsWith('/large')) {
        return { statusCode: 200, body: { value: 'x'.repeat(2048) } };
      }
      if (incoming.path.endsWith('/throw')) {
        throw new Error('secret driver detail');
      }
      return { statusCode: 200, body: incoming.body };
    }),
  );
  t.after(() => dispose());

  const invalidJson = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/test',
    headers: { 'content-type': 'application/json' },
    rawBody: '{',
  });
  assert.equal(invalidJson.statusCode, 400);
  assert.deepEqual(invalidJson.body, { code: 'invalid_json' });

  const unsupported = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/test',
    headers: { 'content-type': 'text/plain' },
    rawBody: 'hello',
  });
  assert.equal(unsupported.statusCode, 415);
  assert.deepEqual(unsupported.body, { code: 'unsupported_content_type' });

  const oversized = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/test',
    headers: { 'content-type': 'application/json' },
    rawBody: JSON.stringify({ value: 'x'.repeat(1024) }),
  });
  assert.equal(oversized.statusCode, 413);
  assert.deepEqual(oversized.body, { code: 'request_too_large' });

  const largeResponse = await request(surface.address, {
    path: '/api/v3/large',
  });
  assert.equal(largeResponse.statusCode, 500);
  assert.deepEqual(largeResponse.body, { code: 'response_too_large' });

  const internal = await request(surface.address, {
    path: '/api/v3/throw',
  });
  assert.equal(internal.statusCode, 500);
  assert.deepEqual(internal.body, { code: 'internal_error' });
  assert.equal(
    JSON.stringify(internal).includes('secret driver detail'),
    false,
  );
  assert.equal(
    diagnostics.some(
      (diagnostic) =>
        diagnostic.phase === 'request' && diagnostic.path === '/api/v3/throw',
    ),
    true,
  );
});

test('completes admission preflight before reading an untrusted body', async (t) => {
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
  });
  t.after(() => surface.close());
  const dispose = surface.installAdmission(EVIDENCE, {
    async prepare() {
      throw Object.assign(new Error('credential rejected'), {
        statusCode: 401,
        code: 'authentication_required',
      });
    },
  });
  t.after(() => dispose());

  const rejected = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/runs',
    headers: {
      connection: 'keep-alive',
      'content-type': 'application/json',
      'content-length': String(1024 * 1024),
    },
  });
  assert.equal(rejected.statusCode, 401);
  assert.deepEqual(rejected.body, { code: 'authentication_required' });
  assert.equal(rejected.headers.connection, 'close');
});

test('streams one route-bounded body without widening the JSON body cap', async (t) => {
  const observations = [];
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
    maxBodyBytes: 1024,
  });
  t.after(() => surface.close());
  const dispose = surface.installAdmission(EVIDENCE, {
    async prepare(metadata) {
      observations.push(`prepare:${metadata.path}`);
      return {
        bodyMode: 'stream',
        contentType: 'application/vnd.qinglong.worker-artifact',
        maximumBodyBytes: 8 * 1024,
        async handleStream(body) {
          observations.push(`handle:${body.contentLength}`);
          let total = 0;
          let chunks = 0;
          for await (const chunk of body.chunks) {
            total += chunk.byteLength;
            chunks += 1;
          }
          return {
            statusCode: 200,
            body: { total, chunks, contentType: body.contentType },
          };
        },
      };
    },
  });
  t.after(() => dispose());

  const bytes = Buffer.alloc(4 * 1024, 7);
  const result = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/worker-ingress/artifacts',
    headers: {
      'content-type': 'application/vnd.qinglong.worker-artifact',
    },
    rawBody: bytes,
  });
  assert.equal(result.statusCode, 200);
  assert.deepEqual(result.body, {
    total: bytes.byteLength,
    chunks: 1,
    contentType: 'application/vnd.qinglong.worker-artifact',
  });
  assert.deepEqual(observations, [
    'prepare:/api/v3/worker-ingress/artifacts',
    `handle:${bytes.byteLength}`,
  ]);
});

test('rejects invalid stream envelopes and incomplete consumption', async (t) => {
  let handles = 0;
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
  });
  t.after(() => surface.close());
  const dispose = surface.installAdmission(EVIDENCE, {
    async prepare() {
      return {
        bodyMode: 'stream',
        contentType: 'application/vnd.qinglong.worker-artifact',
        maximumBodyBytes: 1024,
        async handleStream() {
          handles += 1;
          return { statusCode: 200, body: { accepted: true } };
        },
      };
    },
  });
  t.after(() => dispose());

  const oversized = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/worker-ingress/artifacts',
    headers: {
      'content-type': 'application/vnd.qinglong.worker-artifact',
      'content-length': '1025',
    },
  });
  assert.equal(oversized.statusCode, 413);
  assert.deepEqual(oversized.body, { code: 'request_too_large' });
  assert.equal(handles, 0);

  const unsupported = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/worker-ingress/artifacts',
    headers: { 'content-type': 'application/octet-stream' },
    rawBody: 'log',
  });
  assert.equal(unsupported.statusCode, 415);
  assert.deepEqual(unsupported.body, { code: 'unsupported_content_type' });
  assert.equal(handles, 0);

  const incomplete = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/worker-ingress/artifacts',
    headers: {
      'content-type': 'application/vnd.qinglong.worker-artifact',
    },
    rawBody: 'log',
  });
  assert.equal(incomplete.statusCode, 500);
  assert.deepEqual(incomplete.body, { code: 'internal_error' });
  assert.equal(incomplete.headers.connection, 'close');
  assert.equal(handles, 1);
});

test('refunds successful admission and limits failed preflight before body reads', async (t) => {
  const events = [];
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
    authenticationRatePerPeer: 1,
    authenticationRateGlobal: 10,
    authenticationRateMaxPeers: 4,
    onAuthenticationShieldEvent(event) {
      events.push(event);
    },
  });
  t.after(() => surface.close());

  const beforeRead = await request(surface.address, {
    path: '/api/v3/test',
    headers: { 'x-forwarded-for': '198.51.100.10' },
  });
  assert.equal(beforeRead.statusCode, 503);
  assert.deepEqual(beforeRead.body, { code: 'not_ready' });

  let prepares = 0;
  const dispose = surface.installAdmission(EVIDENCE, {
    async prepare(metadata) {
      prepares += 1;
      if (metadata.path === '/api/v3/rejected') {
        throw Object.assign(new Error('authentication failed'), {
          statusCode: 401,
          code: 'authentication_required',
        });
      }
      return { handle: () => ({ statusCode: 204 }) };
    },
  });
  t.after(() => dispose());

  const admitted = await request(surface.address, {
    path: '/api/v3/test',
    headers: { 'x-forwarded-for': '198.51.100.11' },
  });
  assert.equal(admitted.statusCode, 204);
  assert.equal(prepares, 1);

  const rejected = await request(surface.address, {
    path: '/api/v3/rejected',
  });
  assert.equal(rejected.statusCode, 401);
  assert.deepEqual(rejected.body, { code: 'authentication_required' });
  assert.equal(prepares, 2);

  const limited = await request(surface.address, {
    method: 'POST',
    path: '/api/v3/rejected',
    headers: {
      connection: 'keep-alive',
      'content-type': 'application/json',
      'content-length': String(1024 * 1024),
      'x-forwarded-for': '203.0.113.99',
    },
  });
  assert.equal(limited.statusCode, 429);
  assert.deepEqual(limited.body, { code: 'authentication_rate_limited' });
  assert.equal(limited.headers['retry-after'], '60');
  assert.equal(limited.headers.connection, 'close');
  assert.equal(prepares, 2);
  assert.deepEqual(events, [{ outcome: 'rate_limited', reason: 'peer' }]);

  const probe = await request(surface.address, { path: '/livez' });
  assert.equal(probe.statusCode, 200);
});

test('withdraws admission immediately and drains in-flight requests', async (t) => {
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
    maxInFlightRequests: 1,
    drainTimeoutMs: 1000,
  });
  t.after(() => surface.close());
  let entered;
  const enteredPromise = new Promise((resolve) => {
    entered = resolve;
  });
  let release;
  const gate = new Promise((resolve) => {
    release = resolve;
  });
  const dispose = surface.installAdmission(
    EVIDENCE,
    pipeline(async () => {
      entered();
      await gate;
      return { statusCode: 200, body: { completed: true } };
    }),
  );

  const first = request(surface.address, { path: '/api/v3/slow' });
  await enteredPromise;
  const capacity = await request(surface.address, { path: '/api/v3/second' });
  assert.equal(capacity.statusCode, 503);
  assert.deepEqual(capacity.body, { code: 'admission_capacity_exhausted' });

  let drained = false;
  const draining = dispose().then(() => {
    drained = true;
  });
  assert.equal(
    (await request(surface.address, { path: '/readyz' })).statusCode,
    503,
  );
  await new Promise((resolve) => setImmediate(resolve));
  assert.equal(drained, false);
  release();
  const withdrawn = await first;
  assert.equal(withdrawn.statusCode, 503);
  assert.deepEqual(withdrawn.body, { code: 'admission_draining' });
  await draining;
  assert.equal(drained, true);
});

test('reports a drain timeout when a handler ignores cancellation', async (t) => {
  const surface = await startClusterControlHttpSurface({
    host: '127.0.0.1',
    port: 0,
    drainTimeoutMs: 100,
  });
  t.after(() => surface.close());
  let entered;
  const enteredPromise = new Promise((resolve) => {
    entered = resolve;
  });
  const dispose = surface.installAdmission(
    EVIDENCE,
    pipeline(async () => {
      entered();
      await new Promise(() => {});
      return { statusCode: 200 };
    }),
  );
  const response = request(surface.address, { path: '/api/v3/stuck' });
  await enteredPromise;
  await assert.rejects(dispose(), ClusterControlAdmissionDrainTimeoutError);
  const timedOut = await response;
  assert.equal(timedOut.statusCode, 503);
  assert.deepEqual(timedOut.body, { code: 'admission_draining' });
});

test('rejects unsafe listener and resource configurations before binding', async () => {
  const plaintext = await startClusterControlHttpSurface({ port: 0 });
  await assert.rejects(
    Promise.resolve().then(() => plaintext.reloadMutualTls(MTLS)),
    /does not use mutual TLS/,
  );
  await plaintext.close();
  await assert.rejects(
    startClusterControlHttpSurface({
      port: 0,
      mutualTls: {
        ...MTLS,
        certificateRevocationLists: [
          '-----BEGIN X509 CRL-----\ninvalid\n-----END X509 CRL-----',
        ],
      },
    }),
    ClusterControlHttpConfigurationError,
  );
  await assert.rejects(
    startClusterControlHttpSurface({ host: '0.0.0.0;bad', port: 0 }),
    ClusterControlHttpConfigurationError,
  );
  await assert.rejects(
    startClusterControlHttpSurface({
      port: 0,
      maxInFlightRequests: 1025,
    }),
    ClusterControlHttpConfigurationError,
  );
  await assert.rejects(
    startClusterControlHttpSurface({
      port: 0,
      authenticationRateMaxPeers: 65_537,
    }),
    ClusterControlHttpConfigurationError,
  );
});