feat(ql3): establish 3.0 incubation baseline

2026-09-20 16:07:11 +08:00 · 2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
@@ -0,0 +1,240 @@
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+const {
+  ClusterControlRecoveryEvidenceRegistry,
+  MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_PROVIDERS,
+} = require('../dist');
+
+function claim() {
+  return {
+    candidate: {
+      kind: 'attempt',
+      id: 'attempt-1',
+      runId: 'run-1',
+      status: 'running',
+      createdAtMs: 1,
+    },
+    observedAtMs: 100,
+    ownerId: 'replica-a',
+    token: '123e4567-e89b-42d3-a456-426614174000',
+    version: 1,
+    expiresAtMs: 30_100,
+  };
+}
+
+function target(overrides = {}) {
+  return {
+    runId: 'run-1',
+    attemptId: 'attempt-1',
+    attemptStatus: 'running',
+    executorType: 'remote_worker',
+    callbackSequence: 3,
+    workerId: 'worker-a',
+    executorHandle: 'offer-1',
+    leaseToken: 'lease-token-123456',
+    leaseExpiresAtMs: 99,
+    startedAtMs: 10,
+    ...overrides,
+  };
+}
+
+test('routes one exact executor identity without exposing the recovery claim', async () => {
+  const calls = [];
+  const provider = {
+    executorType: 'remote_worker',
+    requiredIdentity: ['workerId', 'executorHandle', 'leaseToken'],
+    async inspect(current, context) {
+      calls.push([current, context]);
+      assert.equal(Object.isFrozen(current), true);
+      assert.equal(Object.isFrozen(context), true);
+      assert.equal(context.timeoutMs, 100);
+      assert.equal(context.signal.aborted, false);
+      assert.equal('token' in current, false);
+      assert.equal('ownerId' in current, false);
+      return { status: 'running', ignored: 'not propagated' };
+    },
+  };
+  const registry = new ClusterControlRecoveryEvidenceRegistry([provider], {
+    timeoutMs: 100,
+  });
+  provider.inspect = async () => ({ status: 'not_running' });
+
+  assert.deepEqual(await registry.inspect(claim(), target()), {
+    status: 'running',
+  });
+  assert.equal(calls.length, 1);
+  registry.dispose();
+});
+
+test('fails closed for unknown executors and incomplete or malformed identities', async () => {
+  let calls = 0;
+  const registry = new ClusterControlRecoveryEvidenceRegistry([
+    {
+      executorType: 'remote_worker',
+      requiredIdentity: ['workerId', 'leaseToken'],
+      async inspect() {
+        calls += 1;
+        return { status: 'not_running' };
+      },
+    },
+  ]);
+
+  assert.deepEqual(
+    await registry.inspect(claim(), target({ executorType: 'kubernetes' })),
+    { status: 'unknown', reason: 'identity_unverifiable' },
+  );
+  assert.deepEqual(
+    await registry.inspect(claim(), target({ leaseToken: undefined })),
+    { status: 'unknown', reason: 'identity_unverifiable' },
+  );
+  assert.deepEqual(
+    await registry.inspect(claim(), target({ callbackSequence: -1 })),
+    { status: 'unknown', reason: 'identity_unverifiable' },
+  );
+  assert.equal(calls, 0);
+  registry.dispose();
+});
+
+test('maps provider failure and malformed evidence without leaking errors', async () => {
+  const failing = new ClusterControlRecoveryEvidenceRegistry([
+    {
+      executorType: 'remote_worker',
+      requiredIdentity: ['workerId'],
+      async inspect() {
+        throw new Error('sensitive transport failure');
+      },
+    },
+  ]);
+  assert.deepEqual(await failing.inspect(claim(), target()), {
+    status: 'unknown',
+    reason: 'provider_unavailable',
+  });
+  failing.dispose();
+
+  const malformed = new ClusterControlRecoveryEvidenceRegistry([
+    {
+      executorType: 'remote_worker',
+      requiredIdentity: ['workerId'],
+      async inspect() {
+        return { status: 'not_running', reason: 'untrusted-extra' };
+      },
+    },
+  ]);
+  assert.deepEqual(await malformed.inspect(claim(), target()), {
+    status: 'not_running',
+  });
+  malformed.dispose();
+});
+
+test('times out once per provider and prevents abandoned probe accumulation', async () => {
+  const keepAlive = setInterval(() => {}, 1_000);
+  let release;
+  let calls = 0;
+  let firstSignal;
+  const registry = new ClusterControlRecoveryEvidenceRegistry(
+    [
+      {
+        executorType: 'remote_worker',
+        requiredIdentity: ['workerId'],
+        inspect(_target, context) {
+          calls += 1;
+          firstSignal = context.signal;
+          return new Promise((resolve) => {
+            release = resolve;
+          });
+        },
+      },
+    ],
+    { timeoutMs: 5 },
+  );
+
+  try {
+    assert.deepEqual(await registry.inspect(claim(), target()), {
+      status: 'unknown',
+      reason: 'provider_unavailable',
+    });
+    assert.equal(firstSignal.aborted, true);
+    assert.deepEqual(await registry.inspect(claim(), target()), {
+      status: 'unknown',
+      reason: 'provider_unavailable',
+    });
+    assert.equal(calls, 1);
+
+    release({ status: 'running' });
+    await new Promise((resolve) => setImmediate(resolve));
+  } finally {
+    clearInterval(keepAlive);
+    registry.dispose();
+  }
+});
+
+test('dispose aborts an active provider and permanently fails closed', async () => {
+  let signal;
+  const registry = new ClusterControlRecoveryEvidenceRegistry(
+    [
+      {
+        executorType: 'remote_worker',
+        requiredIdentity: ['workerId'],
+        inspect(_target, context) {
+          signal = context.signal;
+          return new Promise(() => {});
+        },
+      },
+    ],
+    { timeoutMs: 1_000 },
+  );
+
+  const inspection = registry.inspect(claim(), target());
+  await new Promise((resolve) => setImmediate(resolve));
+  registry.dispose();
+  assert.equal(signal.aborted, true);
+  assert.deepEqual(await inspection, {
+    status: 'unknown',
+    reason: 'provider_unavailable',
+  });
+  assert.deepEqual(await registry.inspect(claim(), target()), {
+    status: 'unknown',
+    reason: 'provider_unavailable',
+  });
+});
+
+test('rejects duplicate, wildcard, identity-free and unbounded registrations', () => {
+  const provider = {
+    executorType: 'remote_worker',
+    requiredIdentity: ['workerId'],
+    async inspect() {
+      return { status: 'running' };
+    },
+  };
+  assert.throws(
+    () => new ClusterControlRecoveryEvidenceRegistry([provider, provider]),
+    /Duplicate/,
+  );
+  assert.throws(
+    () =>
+      new ClusterControlRecoveryEvidenceRegistry([
+        { ...provider, executorType: '*' },
+      ]),
+    /executorType/,
+  );
+  assert.throws(
+    () =>
+      new ClusterControlRecoveryEvidenceRegistry([
+        { ...provider, requiredIdentity: [] },
+      ]),
+    /requires an execution identity/,
+  );
+  assert.throws(
+    () =>
+      new ClusterControlRecoveryEvidenceRegistry(
+        Array.from(
+          { length: MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_PROVIDERS + 1 },
+          (_, index) => ({
+            ...provider,
+            executorType: `worker_${index}`,
+          }),
+        ),
+      ),
+    /cannot exceed/,
+  );
+});