feat(ql3): establish 3.0 incubation baseline

2026-09-20 16:07:11 +08:00 · 2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
@@ -0,0 +1,662 @@
+const assert = require('node:assert/strict');
+const { generateKeyPairSync, sign } = require('node:crypto');
+const { readFile, writeFile, chmod, mkdtemp, rm } = require('node:fs/promises');
+const { tmpdir } = require('node:os');
+const { join, resolve } = require('node:path');
+const { test } = require('node:test');
+
+const {
+  ClusterWorkerCredentialManagementProcessConfigError,
+  loadClusterWorkerCredentialManagementProcessConfig,
+  startClusterWorkerCredentialManagementProcess,
+} = require('@qinglong/cluster-admin/worker-credential-management-process');
+
+const SERVER_KEY = resolve(
+  __dirname,
+  '../../ql3-cluster-control/test/fixtures/mtls/server-key.pem',
+);
+const SERVER_CERT = resolve(
+  __dirname,
+  '../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem',
+);
+const CLIENT_CA = resolve(
+  __dirname,
+  '../../ql3-cluster-control/test/fixtures/mtls/ca-cert.pem',
+);
+const EMPTY_CRL = resolve(
+  __dirname,
+  '../../ql3-cluster-control/test/fixtures/mtls/empty-crl.pem',
+);
+const MANAGEMENT_SERVICE_CERT = resolve(
+  __dirname,
+  'fixtures/management-service-cert.pem',
+);
+const NOW_MS = Date.UTC(2030, 0, 1);
+const WORKER_ISSUER = 'https://identity.example.test/';
+const WORKER_AUDIENCE = 'qinglong3-worker-credential-management';
+
+function workerIdentityFixture() {
+  const kid = 'worker-identity-key-1';
+  const { privateKey, publicKey } = generateKeyPairSync('ed25519');
+  const keyset = {
+    schemaVersion: 1,
+    generation: 1,
+    issuer: WORKER_ISSUER,
+    audience: WORKER_AUDIENCE,
+    keys: [
+      {
+        ...publicKey.export({ format: 'jwk' }),
+        alg: 'EdDSA',
+        kid,
+        use: 'sig',
+      },
+    ],
+    revokedKids: [],
+    assuranceMappings: [
+      {
+        acr: 'urn:ql3:mfa',
+        assurance: 'multi_factor',
+        requiredAmr: ['pwd', 'otp'],
+      },
+    ],
+    constraints: {
+      maxAssertionBytes: 8 * 1024,
+      maxLifetimeMs: 5 * 60 * 1000,
+      maxAuthenticationAgeMs: 5 * 60 * 1000,
+      clockSkewMs: 5 * 1000,
+    },
+  };
+  const assertion = (type, purpose) => {
+    const header = Buffer.from(
+      JSON.stringify({ alg: 'EdDSA', kid, typ: type }),
+    ).toString('base64url');
+    const now = Math.floor(NOW_MS / 1_000);
+    const payload = Buffer.from(
+      JSON.stringify({
+        acr: 'urn:ql3:mfa',
+        amr: ['pwd', 'otp'],
+        aud: WORKER_AUDIENCE,
+        auth_time: now - 10,
+        exp: now + 120,
+        iat: now,
+        iss: WORKER_ISSUER,
+        jti: `worker-process-${purpose}`,
+        ql3_purpose: purpose,
+        sub: 'worker-operator-1',
+      }),
+    ).toString('base64url');
+    const signed = `${header}.${payload}`;
+    return `${signed}.${sign(
+      null,
+      Buffer.from(signed, 'ascii'),
+      privateKey,
+    ).toString('base64url')}`;
+  };
+  return {
+    keyset,
+    workerAssertion: assertion(
+      'ql3-worker-credential-management+jwt',
+      'worker-credential-management',
+    ),
+    pluginAssertion: assertion(
+      'ql3-plugin-package-management+jwt',
+      'plugin-package-management',
+    ),
+  };
+}
+
+function identityLedgerPool() {
+  let row;
+  const client = {
+    async query(statement, parameters = []) {
+      if (statement === 'BEGIN' || statement === 'COMMIT') return { rows: [] };
+      if (statement === 'ROLLBACK') return { rows: [] };
+      if (statement.includes('INSERT INTO')) {
+        row ??= {
+          generation: parameters[1],
+          digest: parameters[2],
+          issuer: parameters[3],
+          audience: parameters[4],
+          activeKeyIds: JSON.parse(parameters[5]),
+          revokedKeyIds: JSON.parse(parameters[6]),
+        };
+        return { rows: [] };
+      }
+      if (statement.includes('SELECT generation')) {
+        return { rows: row === undefined ? [] : [row] };
+      }
+      throw new Error('unexpected identity ledger query');
+    },
+    release() {},
+  };
+  return {
+    async query() {
+      throw new Error('construction must not query PostgreSQL');
+    },
+    async connect() {
+      return client;
+    },
+  };
+}
+
+function enabledEnvironment(paths, overrides = {}) {
+  return {
+    QL3_WORKER_CREDENTIAL_MANAGEMENT_ENABLED: 'true',
+    QL3_PROFILE: 'cluster-admin',
+    QL3_WORKER_CREDENTIAL_MANAGEMENT_HOST: '127.0.0.1',
+    QL3_WORKER_CREDENTIAL_MANAGEMENT_PORT: '8444',
+    QL3_WORKER_CREDENTIAL_MANAGEMENT_TLS_CERT_FILE: paths.certificateFile,
+    QL3_WORKER_CREDENTIAL_MANAGEMENT_TLS_KEY_FILE: paths.privateKeyFile,
+    QL3_WORKER_CREDENTIAL_MANAGEMENT_CLIENT_CA_FILE:
+      paths.clientCertificateAuthorityFile,
+    QL3_WORKER_CREDENTIAL_MANAGEMENT_CLIENT_CRL_FILE:
+      paths.clientCertificateRevocationListFile,
+    QL3_WORKER_CREDENTIAL_MANAGEMENT_IDENTITY_KEYSET_FILE:
+      paths.identityKeysetFile,
+    QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_URL:
+      'postgresql://ql3_worker_credential_manager:secret@postgres.example.test/ql3',
+    QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_TLS_MODE: 'disable',
+    QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_ALLOW_INSECURE: 'true',
+    ...overrides,
+  };
+}
+
+function readiness() {
+  return {
+    ready: true,
+    writablePrimary: true,
+    serverVersionNum: 180004,
+    serverMajor: 18,
+    currentUser: 'ql3_worker_credential_manager',
+    contractName: 'control-core',
+    contractVersion: 49,
+    migrationIds: ['pg-0050-worker-credential-management-boundary'],
+  };
+}
+
+function identities() {
+  let reloads = 0;
+  return {
+    get reloads() {
+      return reloads;
+    },
+    provider: {
+      async reload() {
+        reloads += 1;
+        return {
+          schemaVersion: 1,
+          generation: 7,
+          digest: 'worker-keyset-digest',
+          issuer: 'https://identity.example.test/',
+          audience: 'qinglong3-worker-credential-management',
+          activeKeyIds: ['identity-key-7'],
+          revokedKeyIds: ['identity-key-6'],
+        };
+      },
+      bind() {
+        throw new Error('HTTP stub must not authenticate');
+      },
+    },
+  };
+}
+
+async function tlsFixture(run) {
+  const directory = await mkdtemp(join(tmpdir(), 'ql3-worker-manager-'));
+  const paths = {
+    certificateFile: join(directory, 'tls.crt'),
+    privateKeyFile: join(directory, 'tls.key'),
+    clientCertificateAuthorityFile: join(directory, 'client-ca.crt'),
+    clientCertificateRevocationListFile: join(directory, 'client.crl'),
+    identityKeysetFile: join(directory, 'keyset.json'),
+  };
+  try {
+    await writeFile(paths.certificateFile, await readFile(SERVER_CERT), {
+      mode: 0o644,
+    });
+    await writeFile(paths.privateKeyFile, await readFile(SERVER_KEY), {
+      mode: 0o640,
+    });
+    await writeFile(
+      paths.clientCertificateAuthorityFile,
+      await readFile(CLIENT_CA),
+      { mode: 0o644 },
+    );
+    await writeFile(
+      paths.clientCertificateRevocationListFile,
+      await readFile(EMPTY_CRL),
+      { mode: 0o644 },
+    );
+    await writeFile(paths.identityKeysetFile, '{}\n', { mode: 0o644 });
+    return await run(paths);
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+}
+
+test('disabled Worker manager reads no profile, TLS or PostgreSQL authority', async () => {
+  const reads = [];
+  const environment = new Proxy(
+    { QL3_WORKER_CREDENTIAL_MANAGEMENT_ENABLED: 'false' },
+    {
+      get(target, property) {
+        reads.push(property);
+        if (property === 'QL3_WORKER_CREDENTIAL_MANAGEMENT_ENABLED') {
+          return target[property];
+        }
+        throw new Error(`disabled config read ${String(property)}`);
+      },
+    },
+  );
+  let opened = 0;
+  const runtime = await startClusterWorkerCredentialManagementProcess({
+    environment,
+    async openDatabase() {
+      opened += 1;
+      throw new Error('must not open');
+    },
+  });
+  assert.equal(runtime.status, 'disabled');
+  assert.equal(opened, 0);
+  assert.deepEqual(reads, ['QL3_WORKER_CREDENTIAL_MANAGEMENT_ENABLED']);
+  await runtime.close();
+});
+
+test('loads one explicit Worker manager-only HTTPS and database configuration', () => {
+  const config = loadClusterWorkerCredentialManagementProcessConfig(
+    enabledEnvironment({
+      certificateFile: '/run/ql3-worker-manager/tls.crt',
+      privateKeyFile: '/run/ql3-worker-manager/tls.key',
+      clientCertificateAuthorityFile:
+        '/run/ql3-worker-manager/client-ca.crt',
+      clientCertificateRevocationListFile:
+        '/run/ql3-worker-manager/client.crl',
+      identityKeysetFile: '/run/ql3-worker-manager/keyset.json',
+    }),
+  );
+  assert.equal(config.enabled, true);
+  assert.equal(config.profile, 'cluster-admin');
+  assert.equal(config.port, 8444);
+  assert.equal(
+    config.clientCertificateAuthorityFile,
+    '/run/ql3-worker-manager/client-ca.crt',
+  );
+  assert.equal(
+    config.clientCertificateRevocationListFile,
+    '/run/ql3-worker-manager/client.crl',
+  );
+  assert.equal(config.database.connection.tls.mode, 'disable');
+  assert.equal(config.database.pool.maxConnections, 2);
+  assert.equal(
+    config.database.pool.applicationName,
+    'qinglong3-worker-credential-manager',
+  );
+  assert.deepEqual(config.quota, {
+    windowMs: 60_000,
+    planLimit: 30,
+    proposeLimit: 30,
+    decideLimit: 60,
+    inspectLimit: 600,
+  });
+  assert.equal(config.planLifetimeMs, 15 * 60_000);
+  assert.equal(config.approvalLifetimeMs, 15 * 60_000);
+});
+
+test('rejects profile drift, implicit insecure PostgreSQL and unsafe bounds', () => {
+  const paths = {
+    certificateFile: '/run/ql3-worker-manager/tls.crt',
+    privateKeyFile: '/run/ql3-worker-manager/tls.key',
+    clientCertificateAuthorityFile:
+      '/run/ql3-worker-manager/client-ca.crt',
+    clientCertificateRevocationListFile:
+      '/run/ql3-worker-manager/client.crl',
+    identityKeysetFile: '/run/ql3-worker-manager/keyset.json',
+  };
+  for (const environment of [
+    enabledEnvironment(paths, { QL3_PROFILE: 'cluster-control' }),
+    enabledEnvironment(paths, {
+      QL3_POSTGRES_WORKER_CREDENTIAL_MANAGER_ALLOW_INSECURE: 'false',
+    }),
+    enabledEnvironment(paths, {
+      QL3_WORKER_CREDENTIAL_MANAGEMENT_TLS_KEY_FILE: 'relative.key',
+    }),
+    enabledEnvironment(paths, {
+      QL3_WORKER_CREDENTIAL_MANAGEMENT_CLIENT_CA_FILE: 'relative-ca.crt',
+    }),
+    enabledEnvironment(paths, {
+      QL3_WORKER_CREDENTIAL_MANAGEMENT_CLIENT_CRL_FILE: 'relative.crl',
+    }),
+    enabledEnvironment(paths, {
+      QL3_WORKER_CREDENTIAL_MANAGEMENT_INSPECT_QUOTA: '1001',
+    }),
+  ]) {
+    assert.throws(
+      () => loadClusterWorkerCredentialManagementProcessConfig(environment),
+      ClusterWorkerCredentialManagementProcessConfigError,
+    );
+  }
+});
+
+test('starts after manager readiness and identity validation then closes in order', async () => {
+  await tlsFixture(async (paths) => {
+    const order = [];
+    const identity = identities();
+    const pool = {
+      async query() {
+        throw new Error('construction must not query PostgreSQL');
+      },
+      async connect() {
+        throw new Error('construction must not acquire PostgreSQL');
+      },
+    };
+    let privateKey;
+    let httpOptions;
+    const runtime = await startClusterWorkerCredentialManagementProcess({
+      environment: enabledEnvironment(paths),
+      identities: identity.provider,
+      async openDatabase() {
+        order.push('database.open');
+        return {
+          pool,
+          async close() {
+            order.push('database.close');
+          },
+        };
+      },
+      async assertReady(observedPool) {
+        order.push('database.ready');
+        assert.equal(observedPool, pool);
+        return readiness();
+      },
+      async startHttp(options) {
+        order.push('http.start');
+        httpOptions = options;
+        privateKey = options.tls.privateKey;
+        assert.equal(
+          privateKey.some((value) => value !== 0),
+          true,
+        );
+        return {
+          status: 'active',
+          address: { host: '127.0.0.1', port: 9444 },
+          availabilityStatus: () => 'ready',
+          withdraw() {},
+          async close() {
+            order.push('http.close');
+          },
+        };
+      },
+      now: () => NOW_MS,
+    });
+
+    assert.equal(runtime.status, 'active');
+    assert.deepEqual(order, ['database.open', 'database.ready', 'http.start']);
+    assert.equal(identity.reloads, 1);
+    assert.equal(
+      privateKey.every((value) => value === 0),
+      true,
+    );
+    assert.equal(typeof httpOptions.transport.execute, 'function');
+    assert.equal(httpOptions.identities, identity.provider);
+    assert.deepEqual(runtime.identity.activeKeyIds, ['identity-key-7']);
+    assert.equal(runtime.database.contractVersion, 49);
+
+    await Promise.all([runtime.close(), runtime.close()]);
+    assert.deepEqual(order, [
+      'database.open',
+      'database.ready',
+      'http.start',
+      'http.close',
+      'database.close',
+    ]);
+  });
+});
+
+test('assembles the default Worker identity purpose without accepting Plugin assertions', async () => {
+  await tlsFixture(async (paths) => {
+    const identity = workerIdentityFixture();
+    await writeFile(
+      paths.identityKeysetFile,
+      `${JSON.stringify(identity.keyset)}\n`,
+      { mode: 0o644 },
+    );
+    const pool = identityLedgerPool();
+    let capturedIdentities;
+    const runtime = await startClusterWorkerCredentialManagementProcess({
+      environment: enabledEnvironment(paths),
+      async openDatabase() {
+        return { pool, async close() {} };
+      },
+      async assertReady() {
+        return readiness();
+      },
+      async startHttp(options) {
+        capturedIdentities = options.identities;
+        return {
+          status: 'active',
+          address: { host: '127.0.0.1', port: 9444 },
+          availabilityStatus: () => 'ready',
+          withdraw() {},
+          async close() {},
+        };
+      },
+      now: () => NOW_MS,
+    });
+
+    assert.ok(capturedIdentities);
+    assert.deepEqual(
+      (await capturedIdentities.bind(identity.workerAssertion).authenticate())
+        .subject,
+      { type: 'user', id: 'worker-operator-1' },
+    );
+    await assert.rejects(
+      capturedIdentities.bind(identity.pluginAssertion).authenticate(),
+      { code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
+    );
+    await runtime.close();
+  });
+});
+
+test('closes manager database when readiness or HTTP startup fails', async () => {
+  await tlsFixture(async (paths) => {
+    for (const failureAt of ['readiness', 'http']) {
+      let closes = 0;
+      await assert.rejects(
+        startClusterWorkerCredentialManagementProcess({
+          environment: enabledEnvironment(paths),
+          identities: identities().provider,
+          async openDatabase() {
+            return {
+              pool: {
+                async query() {
+                  throw new Error('must not query');
+                },
+                async connect() {
+                  throw new Error('must not connect');
+                },
+              },
+              async close() {
+                closes += 1;
+              },
+            };
+          },
+          async assertReady() {
+            if (failureAt === 'readiness') {
+              throw new Error('readiness failed');
+            }
+            return readiness();
+          },
+          async startHttp() {
+            throw new Error('HTTP failed');
+          },
+        }),
+        new RegExp(`${failureAt} failed`, 'i'),
+      );
+      assert.equal(closes, 1);
+    }
+  });
+});
+
+test('rejects publicly readable private TLS authority before listener start', async () => {
+  await tlsFixture(async (paths) => {
+    await chmod(paths.privateKeyFile, 0o644);
+    let starts = 0;
+    await assert.rejects(
+      startClusterWorkerCredentialManagementProcess({
+        environment: enabledEnvironment(paths),
+        identities: identities().provider,
+        async openDatabase() {
+          return {
+            pool: {
+              async query() {
+                throw new Error('must not query');
+              },
+              async connect() {
+                throw new Error('must not connect');
+              },
+            },
+            async close() {},
+          };
+        },
+        async assertReady() {
+          return readiness();
+        },
+        async startHttp() {
+          starts += 1;
+          throw new Error('must not start');
+        },
+      }),
+      ClusterWorkerCredentialManagementProcessConfigError,
+    );
+    assert.equal(starts, 0);
+  });
+});
+
+test('accepts a bounded client CA overlap bundle before listener start', async () => {
+  await tlsFixture(async (paths) => {
+    await writeFile(
+      paths.clientCertificateAuthorityFile,
+      Buffer.concat([
+        await readFile(CLIENT_CA),
+        await readFile(MANAGEMENT_SERVICE_CERT),
+      ]),
+      { mode: 0o644 },
+    );
+    let starts = 0;
+    const runtime = await startClusterWorkerCredentialManagementProcess({
+      environment: enabledEnvironment(paths),
+      identities: identities().provider,
+      async openDatabase() {
+        return {
+          pool: {
+            async query() {
+              throw new Error('must not query');
+            },
+            async connect() {
+              throw new Error('must not connect');
+            },
+          },
+          async close() {},
+        };
+      },
+      async assertReady() {
+        return readiness();
+      },
+      async startHttp() {
+        starts += 1;
+        return {
+          status: 'active',
+          address: { host: '127.0.0.1', port: 9444 },
+          availabilityStatus: () => 'ready',
+          withdraw() {},
+          async close() {},
+        };
+      },
+      now: () => NOW_MS,
+    });
+    assert.equal(starts, 1);
+    await runtime.close();
+  });
+});
+
+test('rejects malformed or unbounded client trust before listener start', async () => {
+  await tlsFixture(async (paths) => {
+    const authority = await readFile(CLIENT_CA);
+    const leaf = await readFile(SERVER_CERT);
+    const revocationList = await readFile(EMPTY_CRL);
+    const invalidConfigurations = [
+      {
+        authority: Buffer.concat([authority, authority]),
+        revocationList,
+      },
+      {
+        authority: Buffer.concat([authority, Buffer.from('unexpected\n')]),
+        revocationList,
+      },
+      { authority: leaf, revocationList },
+      {
+        authority: Buffer.concat(Array.from({ length: 17 }, () => authority)),
+        revocationList,
+      },
+      {
+        authority,
+        revocationList: Buffer.concat([revocationList, revocationList]),
+      },
+      {
+        authority,
+        revocationList: Buffer.from(
+          '-----BEGIN X509 CRL-----\ninvalid\n-----END X509 CRL-----\n',
+        ),
+      },
+      {
+        authority,
+        revocationList,
+        now: Date.UTC(2050, 0, 1),
+      },
+      { authority, revocationList, now: -1 },
+    ];
+    for (const invalid of invalidConfigurations) {
+      await writeFile(paths.clientCertificateAuthorityFile, invalid.authority, {
+        mode: 0o644,
+      });
+      await writeFile(
+        paths.clientCertificateRevocationListFile,
+        invalid.revocationList,
+        { mode: 0o644 },
+      );
+      let starts = 0;
+      let closes = 0;
+      await assert.rejects(
+        startClusterWorkerCredentialManagementProcess({
+          environment: enabledEnvironment(paths),
+          identities: identities().provider,
+          async openDatabase() {
+            return {
+              pool: {
+                async query() {
+                  throw new Error('must not query');
+                },
+                async connect() {
+                  throw new Error('must not connect');
+                },
+              },
+              async close() {
+                closes += 1;
+              },
+            };
+          },
+          async assertReady() {
+            return readiness();
+          },
+          async startHttp() {
+            starts += 1;
+            throw new Error('must not start');
+          },
+          now: () => invalid.now ?? NOW_MS,
+        }),
+        ClusterWorkerCredentialManagementProcessConfigError,
+      );
+      assert.equal(starts, 0);
+      assert.equal(closes, 1);
+    }
+  });
+});