qinglong/test/back/legacyShadowTerminalDifference.test.cjs

require('ts-node/register/transpile-only');

const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const { afterEach, test } = require('node:test');
const { DataTypes, Sequelize } = require('sequelize');
const {
  defineSchemaMigrationModel,
} = require('../../back/data/schemaMigration');
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
const {
  runningInstanceRunReferenceMigration,
} = require('../../back/migrations/0003-running-instance-run-reference');
const {
  runCancellationRequestMigration,
} = require('../../back/migrations/0004-run-cancellation-request');
const {
  runAttemptDeadlineMigration,
} = require('../../back/migrations/0006-run-attempt-deadline');
const { runMigrations } = require('../../back/migrations/runner');
const {
  LegacySequelizeRunRepository,
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
const {
  LegacySequelizeShadowTerminalDifferenceSource,
} = require('../../back/runtime/adapters/legacy-sequelize/legacyShadowTerminalDifferenceSource');
const {
  LegacyShadowRunWriter,
} = require('../../back/runtime/application/legacyShadowRunWriter');
const {
  LegacyShadowTerminalDifferenceAuditor,
} = require('../../back/runtime/application/legacyShadowTerminalDifferenceAuditor');
const {
  createLegacyLogArtifactId,
} = require('../../back/runtime/compatibility/legacyTaskRevision');
const {
  parseArguments,
} = require('../../scripts/ql3-legacy-shadow-terminal-audit.cjs');

const REPOSITORY_ROOT = path.resolve(__dirname, '../..');
const CLI_PATH = path.join(
  REPOSITORY_ROOT,
  'scripts',
  'ql3-legacy-shadow-terminal-audit.cjs',
);
const BASE_TIME = 1_750_100_000_000;
const databases = [];
const temporaryDirectories = [];
let idSequence = 4_000;
let timeSequence = BASE_TIME;

function nextId() {
  idSequence += 1;
  return `019f7300-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
}

function nextTime() {
  timeSequence += 10_000;
  return timeSequence;
}

async function createDatabase(storage = ':memory:') {
  const database = new Sequelize({
    dialect: 'sqlite',
    storage,
    logging: false,
  });
  await database.getQueryInterface().createTable('RunningInstances', {
    id: {
      type: DataTypes.INTEGER,
      primaryKey: true,
      autoIncrement: true,
    },
    cron_id: { type: DataTypes.INTEGER, allowNull: false },
    pid: { type: DataTypes.INTEGER, allowNull: true },
    log_path: { type: DataTypes.STRING, allowNull: true },
    started_at: { type: DataTypes.INTEGER, allowNull: false },
    finished_at: { type: DataTypes.INTEGER, allowNull: true },
    status: { type: DataTypes.INTEGER, allowNull: false },
    exit_code: { type: DataTypes.INTEGER, allowNull: true },
  });
  await runMigrations({
    database,
    migrationModel: defineSchemaMigrationModel(database),
    migrations: [
      runSchemaMigration,
      runningInstanceRunReferenceMigration,
      runCancellationRequestMigration,
      runAttemptDeadlineMigration,
    ],
    logger: { info() {} },
  });
  databases.push(database);
  return database;
}

async function createStack(storage) {
  const database = await createDatabase(storage);
  const repository = new LegacySequelizeRunRepository(database);
  const writer = new LegacyShadowRunWriter(repository, nextId);
  const source = new LegacySequelizeShadowTerminalDifferenceSource(
    database,
    createLegacyLogArtifactId,
  );
  const auditor = new LegacyShadowTerminalDifferenceAuditor(source);
  return { database, repository, writer, source, auditor };
}

async function createShadow(writer, overrides = {}) {
  const acceptedAtMs = overrides.acceptedAtMs ?? nextTime();
  const legacyCronId = overrides.legacyCronId ?? 31;
  const pid = overrides.pid ?? 5_101;
  const logPath = overrides.logPath ?? `cron/${acceptedAtMs}.log`;
  const origin = overrides.origin ?? 'manual';
  const reference = await writer.accept({
    origin,
    projectId: 'default',
    taskId: `legacy-cron:${legacyCronId}`,
    taskRevision: `sha256:terminal-audit-${legacyCronId}`,
    legacyCronId,
    triggerType: origin,
    acceptedAtMs,
  });
  if (overrides.acceptedOnly) {
    return { acceptedAtMs, legacyCronId, pid, logPath, origin, reference };
  }
  await writer.spawned(reference, {
    atMs: acceptedAtMs + 100,
    pid,
    logArtifactId: createLegacyLogArtifactId(logPath),
  });
  await writer.running(reference, acceptedAtMs + 200);
  if (!overrides.runningOnly) {
    await writer.exited(reference, {
      atMs: acceptedAtMs + 500,
      exitCode: overrides.shadowExitCode ?? 0,
    });
  }
  return { acceptedAtMs, legacyCronId, pid, logPath, origin, reference };
}

async function insertInstance(database, shadow, overrides = {}) {
  await database.getQueryInterface().bulkInsert('RunningInstances', [
    {
      cron_id: shadow.legacyCronId,
      run_id:
        overrides.direct === false
          ? null
          : overrides.runId ?? shadow.reference.runId,
      attempt_id:
        overrides.direct === false
          ? null
          : overrides.attemptId ?? shadow.reference.attemptId,
      pid: overrides.pid ?? shadow.pid,
      log_path: overrides.logPath ?? shadow.logPath,
      started_at: Math.floor((shadow.acceptedAtMs + 100) / 1_000),
      finished_at: Math.floor(
        (shadow.acceptedAtMs + (overrides.finishedOffsetMs ?? 500)) / 1_000,
      ),
      status: overrides.status ?? 1,
      exit_code: overrides.exitCode ?? 0,
    },
  ]);
}

function auditOptions(shadows, overrides = {}) {
  const times = shadows.map((shadow) => shadow.acceptedAtMs);
  const start = Math.min(...times) - 1;
  const end = Math.max(...times) + 1;
  return {
    profile: overrides.profile ?? 'edge',
    origins: overrides.origins ?? ['manual'],
    windowStartMs: start,
    windowEndMs: end,
    observedAtMs: overrides.observedAtMs ?? end + 5 * 60_000,
    minimumSettlingAgeMs: 5 * 60_000,
  };
}

afterEach(async () => {
  await Promise.all(databases.splice(0).map((database) => database.close()));
  for (const directory of temporaryDirectories.splice(0)) {
    fs.rmSync(directory, { recursive: true, force: true });
  }
});

test('reports a closed, fully comparable Shadow-to-Legacy terminal match', async () => {
  const { database, writer, auditor } = await createStack();
  const shadow = await createShadow(writer);
  await insertInstance(database, shadow);

  const report = await auditor.run(auditOptions([shadow]));

  assert.equal(report.assessment, 'matched');
  assert.equal(report.counts.matched, 1);
  assert.equal(report.terminalAgreementPermille, 1_000);
  assert.equal(report.fullyComparablePermille, 1_000);
  assert.equal(report.coverage.direction, 'shadow_to_legacy');
  assert.equal(report.coverage.legacyWithoutShadow, 'not_measured');
  assert.equal(JSON.stringify(report).includes(shadow.reference.runId), false);
  assert.equal(JSON.stringify(report).includes(shadow.logPath), false);
});

test('separates terminal status and field differences', async () => {
  const { database, writer, auditor } = await createStack();
  const statusShadow = await createShadow(writer, { legacyCronId: 41 });
  const fieldShadow = await createShadow(writer, { legacyCronId: 42 });
  await insertInstance(database, statusShadow, { status: 3, exitCode: 1 });
  await insertInstance(database, fieldShadow, { exitCode: 9 });

  const report = await auditor.run(auditOptions([statusShadow, fieldShadow]));

  assert.equal(report.assessment, 'differences_found');
  assert.equal(report.counts.status_mismatch, 1);
  assert.equal(report.counts.field_mismatch, 1);
  assert.equal(report.dimensions.status.mismatched, 1);
  assert.equal(report.dimensions.exitCode.mismatched, 1);
  assert.equal(report.terminalAgreementPermille, 0);
});

test('does not guess when Legacy evidence is missing or ambiguous', async () => {
  const { database, writer, auditor } = await createStack();
  const missing = await createShadow(writer, { legacyCronId: 51 });
  const ambiguous = await createShadow(writer, { legacyCronId: 52 });
  await insertInstance(database, ambiguous, { direct: false });
  await insertInstance(database, ambiguous, { direct: false });

  const report = await auditor.run(auditOptions([missing, ambiguous]));

  assert.equal(report.counts.legacy_evidence_missing, 1);
  assert.equal(report.counts.legacy_evidence_ambiguous, 1);
  assert.equal(report.assessment, 'differences_found');
});

test('rejects conflicting direct Run and Attempt references as ambiguous', async () => {
  const { database, writer, auditor } = await createStack();
  const shadow = await createShadow(writer, { legacyCronId: 53 });
  await insertInstance(database, shadow, {
    attemptId: '019f7300-0000-7000-8000-999999999999',
  });

  const report = await auditor.run(auditOptions([shadow]));

  assert.equal(report.counts.legacy_evidence_ambiguous, 1);
  assert.equal(report.assessment, 'differences_found');
});

test('classifies a settled cohort member with an active Shadow Run', async () => {
  const { writer, auditor } = await createStack();
  const shadow = await createShadow(writer, { runningOnly: true });

  const report = await auditor.run(auditOptions([shadow]));

  assert.equal(report.counts.shadow_not_terminal, 1);
  assert.equal(report.assessment, 'differences_found');
});

test('withholds ratios while the measurement window is still open', async () => {
  const { database, writer, auditor } = await createStack();
  const shadow = await createShadow(writer);
  await insertInstance(database, shadow);
  const options = auditOptions([shadow], {
    observedAtMs: shadow.acceptedAtMs + 1_000,
  });

  const report = await auditor.run(options);

  assert.equal(report.assessment, 'window_open');
  assert.equal(report.window.closed, false);
  assert.equal(report.terminalAgreementPermille, undefined);
  assert.equal(report.fullyComparablePermille, undefined);
});

test('withholds ratios when the edge candidate budget is exhausted', async () => {
  const { database, writer, auditor } = await createStack();
  const shadows = [];
  for (let index = 0; index < 9; index += 1) {
    const shadow = await createShadow(writer, {
      legacyCronId: 60 + index,
      pid: 6_000 + index,
    });
    shadows.push(shadow);
    await insertInstance(database, shadow);
  }

  const report = await auditor.run(auditOptions(shadows));

  assert.equal(report.scanned, 8);
  assert.equal(report.remaining, true);
  assert.equal(report.stopReason, 'page_limit');
  assert.equal(report.assessment, 'incomplete');
  assert.equal(report.terminalAgreementPermille, undefined);
});

test('marks evidence overflow incomplete instead of accepting a partial match set', async () => {
  const { database, writer, auditor } = await createStack();
  const shadow = await createShadow(writer, { legacyCronId: 71 });
  for (let index = 0; index < 9; index += 1) {
    await insertInstance(database, shadow, {
      direct: false,
      pid: 7_100 + index,
      logPath: `cron/overflow-${index}.log`,
    });
  }

  const report = await auditor.run(auditOptions([shadow]));

  assert.equal(report.evidenceComplete, false);
  assert.equal(report.evidenceOverflowPages, 1);
  assert.equal(report.counts.legacy_evidence_ambiguous, 1);
  assert.equal(report.assessment, 'incomplete');
});

test('keeps an exact, conservation-safe matrix for configured origins', async () => {
  const { database, writer, auditor } = await createStack();
  const manual = await createShadow(writer, { legacyCronId: 81 });
  const system = await createShadow(writer, {
    legacyCronId: 82,
    origin: 'scheduled_system',
  });
  await insertInstance(database, manual);
  await insertInstance(database, system);

  const report = await auditor.run(
    auditOptions([manual, system], {
      origins: ['manual', 'scheduled_system'],
    }),
  );

  assert.deepEqual(
    report.byOrigin.map(({ origin, scanned, matched }) => ({
      origin,
      scanned,
      matched,
    })),
    [
      { origin: 'manual', scanned: 1, matched: 1 },
      { origin: 'scheduled_system', scanned: 1, matched: 1 },
    ],
  );
  assert.equal(
    Object.values(report.counts).reduce((sum, count) => sum + count, 0),
    report.scanned,
  );
});

test('reports an empty closed cohort without claiming agreement', async () => {
  const { auditor } = await createStack();
  const report = await auditor.run({
    profile: 'edge',
    origins: ['manual'],
    windowStartMs: BASE_TIME - 10_000,
    windowEndMs: BASE_TIME - 5_000,
    observedAtMs: BASE_TIME + 300_000,
  });

  assert.equal(report.assessment, 'empty');
  assert.equal(report.scanned, 0);
  assert.equal(report.terminalAgreementPermille, undefined);
});

test('rejects malformed windows, excessive origins and tolerance', async () => {
  const { auditor } = await createStack();
  await assert.rejects(
    auditor.run({
      profile: 'edge',
      origins: ['manual'],
      windowStartMs: 10,
      windowEndMs: 10,
    }),
    /non-empty/,
  );
  await assert.rejects(
    auditor.run({
      profile: 'edge',
      origins: [
        'manual',
        'boot',
        'scheduled_node',
        'scheduled_system',
        'script',
        'subscription',
        'system',
        'grpc',
      ],
      windowStartMs: 1,
      windowEndMs: 2,
    }),
    /origin count/,
  );
  await assert.rejects(
    auditor.run({
      profile: 'edge',
      origins: ['manual'],
      windowStartMs: 1,
      windowEndMs: 2,
      correlationToleranceMs: 60_001,
    }),
    /hard limit/,
  );
});

test('rejects an adapter page that exceeds the evidence hard limit', async () => {
  const auditor = new LegacyShadowTerminalDifferenceAuditor({
    async listCandidates() {
      return {
        candidates: [],
        evidence: [
          {
            instanceId: 1,
            legacyCronId: 1,
            startedAtMs: 1,
            finishedAtMs: 2,
            outcome: 'succeeded',
          },
        ],
        evidenceTruncated: false,
        truncated: false,
      };
    },
  });

  await assert.rejects(
    auditor.run({
      profile: 'edge',
      origins: ['manual'],
      windowStartMs: 1,
      windowEndMs: 2,
      observedAtMs: 300_002,
    }),
    /exceeded evidence limit/,
  );
});

test('CLI arguments require an explicit cohort and reject unsupported origins', () => {
  assert.throws(
    () => parseArguments(['--origin=manual', '--window-start-ms=1']),
    /window-end-ms is required/,
  );
  assert.throws(
    () =>
      parseArguments([
        '--origin=grpc',
        '--window-start-ms=1',
        '--window-end-ms=2',
      ]),
    /unsupported Shadow origin/,
  );
  const options = parseArguments([
    '--profile=standalone',
    '--origin=manual,scheduled_system',
    '--window-start-ms=1',
    '--window-end-ms=2',
    '--json',
  ]);
  assert.equal(options.profile, 'standalone');
  assert.deepEqual(options.origins, ['manual', 'scheduled_system']);
  assert.equal(options.json, true);
});

test('read-only CLI emits the versioned redacted report', async () => {
  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-shadow-audit-'));
  temporaryDirectories.push(directory);
  const storage = path.join(directory, 'database.sqlite');
  const { database, writer } = await createStack(storage);
  const shadow = await createShadow(writer, { legacyCronId: 91 });
  await insertInstance(database, shadow);
  await database.close();
  databases.splice(databases.indexOf(database), 1);
  const options = auditOptions([shadow]);

  const result = spawnSync(
    process.execPath,
    [
      CLI_PATH,
      `--database=${storage}`,
      '--profile=edge',
      '--origin=manual',
      `--window-start-ms=${options.windowStartMs}`,
      `--window-end-ms=${options.windowEndMs}`,
      `--observed-at-ms=${options.observedAtMs}`,
      '--json',
      '--fail-on-difference',
    ],
    { cwd: REPOSITORY_ROOT, encoding: 'utf8' },
  );

  assert.equal(result.status, 0, result.stderr);
  const report = JSON.parse(result.stdout);
  assert.equal(
    report.schema,
    'qinglong/legacy-shadow-terminal-difference-report@v1',
  );
  assert.equal(report.assessment, 'matched');
  assert.equal(result.stdout.includes(shadow.reference.runId), false);
  assert.equal(result.stdout.includes(shadow.logPath), false);
});