refactor: rebuild desktop app with Tauri

benchmarks/semble/src/case.rs

@@ -0,0 +1,115 @@
+//! Version-pinned benchmark suites and hand-labelled implementation locations.
+
+use std::{fs, path::Path};
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Suite {
+    pub name: String,
+    pub repository: String,
+    pub commit: String,
+    pub queries: Vec<QueryCase>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct QueryCase {
+    pub id: String,
+    pub query: String,
+    pub literal_query: String,
+    pub symbol_query: String,
+    pub expected: Vec<ExpectedLocation>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ExpectedLocation {
+    pub path: String,
+    pub line: usize,
+}
+
+pub fn load_suites(directory: &Path) -> Result<Vec<Suite>, String> {
+    let mut paths = fs::read_dir(directory)
+        .map_err(|error| format!("read cases directory {}: {error}", directory.display()))?
+        .filter_map(|entry| entry.ok().map(|entry| entry.path()))
+        .filter(|path| {
+            path.extension()
+                .is_some_and(|extension| extension == "json")
+        })
+        .collect::<Vec<_>>();
+    paths.sort();
+    let mut suites = Vec::with_capacity(paths.len());
+    for path in paths {
+        let bytes = fs::read(&path)
+            .map_err(|error| format!("read benchmark suite {}: {error}", path.display()))?;
+        let suite = serde_json::from_slice::<Suite>(&bytes)
+            .map_err(|error| format!("parse benchmark suite {}: {error}", path.display()))?;
+        validate_suite(&suite)?;
+        suites.push(suite);
+    }
+    if suites.is_empty() {
+        return Err(format!(
+            "no benchmark suites found in {}",
+            directory.display()
+        ));
+    }
+    Ok(suites)
+}
+
+fn validate_suite(suite: &Suite) -> Result<(), String> {
+    if suite.name.trim().is_empty()
+        || suite.repository.trim().is_empty()
+        || suite.commit.trim().is_empty()
+        || suite.queries.is_empty()
+    {
+        return Err(format!("suite {:?} is missing required fields", suite.name));
+    }
+    for query in &suite.queries {
+        if query.id.trim().is_empty()
+            || query.query.trim().is_empty()
+            || query.literal_query.trim().is_empty()
+            || query.symbol_query.trim().is_empty()
+            || query.expected.is_empty()
+        {
+            return Err(format!(
+                "suite {} contains an incomplete query case",
+                suite.name
+            ));
+        }
+        if query
+            .expected
+            .iter()
+            .any(|location| location.path.trim().is_empty() || location.line == 0)
+        {
+            return Err(format!(
+                "suite {} query {} contains an invalid expected location",
+                suite.name, query.id
+            ));
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn rejects_locations_without_a_line_number() {
+        let suite = Suite {
+            name: "fixture".into(),
+            repository: "https://example.test/repo".into(),
+            commit: "abc".into(),
+            queries: vec![QueryCase {
+                id: "q1".into(),
+                query: "find it".into(),
+                literal_query: "find_it result value".into(),
+                symbol_query: "findIt".into(),
+                expected: vec![ExpectedLocation {
+                    path: "src/lib.rs".into(),
+                    line: 0,
+                }],
+            }],
+        };
+        assert!(validate_suite(&suite).is_err());
+    }
+}

benchmarks/semble/src/codegraph.rs

@@ -0,0 +1,258 @@
+//! Adapter lifecycle for benchmarking an installed CodeGraph distribution.
+
+use std::{
+    fs,
+    io::Write,
+    path::{Path, PathBuf},
+    process::{Command, Stdio},
+    time::Instant,
+};
+
+use serde::{Deserialize, Serialize};
+
+use crate::case::Suite;
+
+#[derive(Debug)]
+pub struct CodeGraphRun {
+    pub version: String,
+    pub cold_index_ms: f64,
+    pub persisted_index_bytes: u64,
+    pub adapter: AdapterOutput,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AdapterOutput {
+    pub load_ms: f64,
+    pub stats: AdapterStats,
+    pub tracks: Vec<AdapterTrack>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AdapterStats {
+    pub node_count: usize,
+    pub edge_count: usize,
+    pub file_count: usize,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct AdapterTrack {
+    pub name: String,
+    pub queries: Vec<AdapterQuery>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AdapterQuery {
+    pub id: String,
+    pub query: String,
+    pub durations_ms: Vec<f64>,
+    pub results: Vec<AdapterResult>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AdapterResult {
+    pub path: String,
+    pub start_line: usize,
+    pub end_line: usize,
+    pub score: f32,
+    pub lines: Option<Vec<usize>>,
+}
+
+#[derive(Serialize)]
+struct AdapterInput<'a> {
+    tracks: [AdapterInputTrack<'a>; 3],
+}
+
+#[derive(Serialize)]
+struct AdapterInputTrack<'a> {
+    name: &'static str,
+    queries: Vec<AdapterInputQuery<'a>>,
+}
+
+#[derive(Serialize)]
+struct AdapterInputQuery<'a> {
+    id: &'a str,
+    query: &'a str,
+}
+
+pub fn benchmark(
+    executable: &str,
+    adapter_path: &Path,
+    source: &Path,
+    suite: &Suite,
+    repetitions: usize,
+    top_k: usize,
+) -> Result<CodeGraphRun, String> {
+    let executable = locate_executable(executable)?;
+    let distribution = distribution_root(&executable)?;
+    let node = distribution.join("node");
+    let library = distribution.join("lib");
+    if !node.is_file() || !library.is_dir() {
+        return Err(format!(
+            "unsupported CodeGraph installation layout below {}",
+            distribution.display()
+        ));
+    }
+    if source.join(".codegraph").exists() {
+        command_status(
+            Command::new(&executable)
+                .args(["uninit", "--force"])
+                .arg(source),
+            "reset CodeGraph index",
+        )?;
+    }
+    let started = Instant::now();
+    command_status(
+        Command::new(&executable).arg("init").arg(source),
+        "build CodeGraph index",
+    )?;
+    let cold_index_ms = started.elapsed().as_secs_f64() * 1_000.0;
+    let persisted_index_bytes = directory_size(&source.join(".codegraph"))?;
+    let version = command_output(
+        Command::new(&executable).arg("version"),
+        "read CodeGraph version",
+    )?;
+    let input = AdapterInput {
+        tracks: [
+            AdapterInputTrack {
+                name: "natural_language",
+                queries: suite
+                    .queries
+                    .iter()
+                    .map(|query| AdapterInputQuery {
+                        id: &query.id,
+                        query: &query.query,
+                    })
+                    .collect(),
+            },
+            AdapterInputTrack {
+                name: "literal",
+                queries: suite
+                    .queries
+                    .iter()
+                    .map(|query| AdapterInputQuery {
+                        id: &query.id,
+                        query: &query.literal_query,
+                    })
+                    .collect(),
+            },
+            AdapterInputTrack {
+                name: "symbol",
+                queries: suite
+                    .queries
+                    .iter()
+                    .map(|query| AdapterInputQuery {
+                        id: &query.id,
+                        query: &query.symbol_query,
+                    })
+                    .collect(),
+            },
+        ],
+    };
+    let encoded = serde_json::to_vec(&input)
+        .map_err(|error| format!("serialize CodeGraph benchmark input: {error}"))?;
+    let mut child = Command::new(node)
+        .arg("--liftoff-only")
+        .arg(adapter_path)
+        .arg(library)
+        .arg(source)
+        .arg(repetitions.to_string())
+        .arg(top_k.to_string())
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .map_err(|error| format!("start CodeGraph benchmark adapter: {error}"))?;
+    child
+        .stdin
+        .take()
+        .ok_or_else(|| "CodeGraph adapter stdin is unavailable".to_owned())?
+        .write_all(&encoded)
+        .map_err(|error| format!("write CodeGraph benchmark input: {error}"))?;
+    let output = child
+        .wait_with_output()
+        .map_err(|error| format!("wait for CodeGraph benchmark adapter: {error}"))?;
+    if !output.status.success() {
+        return Err(format!(
+            "CodeGraph benchmark adapter failed: {}",
+            String::from_utf8_lossy(&output.stderr).trim()
+        ));
+    }
+    let adapter = serde_json::from_slice(&output.stdout)
+        .map_err(|error| format!("parse CodeGraph benchmark output: {error}"))?;
+    Ok(CodeGraphRun {
+        version,
+        cold_index_ms,
+        persisted_index_bytes,
+        adapter,
+    })
+}
+
+fn locate_executable(name: &str) -> Result<PathBuf, String> {
+    let output = command_output(
+        Command::new("which").arg(name),
+        "locate CodeGraph executable",
+    )?;
+    fs::canonicalize(output)
+        .map_err(|error| format!("resolve CodeGraph executable symlink: {error}"))
+}
+
+fn distribution_root(executable: &Path) -> Result<PathBuf, String> {
+    executable
+        .parent()
+        .and_then(Path::parent)
+        .map(Path::to_path_buf)
+        .ok_or_else(|| {
+            format!(
+                "cannot locate distribution root for {}",
+                executable.display()
+            )
+        })
+}
+
+fn command_status(command: &mut Command, action: &str) -> Result<(), String> {
+    let status = command
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .map_err(|error| format!("{action}: {error}"))?;
+    if status.success() {
+        Ok(())
+    } else {
+        Err(format!("{action} failed with {status}"))
+    }
+}
+
+fn command_output(command: &mut Command, action: &str) -> Result<String, String> {
+    let output = command
+        .output()
+        .map_err(|error| format!("{action}: {error}"))?;
+    if !output.status.success() {
+        return Err(format!("{action} failed with {}", output.status));
+    }
+    Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned())
+}
+
+fn directory_size(path: &Path) -> Result<u64, String> {
+    let mut total = 0_u64;
+    let mut pending = vec![path.to_path_buf()];
+    while let Some(directory) = pending.pop() {
+        for entry in fs::read_dir(&directory)
+            .map_err(|error| format!("read {}: {error}", directory.display()))?
+        {
+            let entry = entry.map_err(|error| format!("read directory entry: {error}"))?;
+            let metadata = entry
+                .metadata()
+                .map_err(|error| format!("inspect {}: {error}", entry.path().display()))?;
+            if metadata.is_dir() {
+                pending.push(entry.path());
+            } else if metadata.is_file() {
+                total = total.saturating_add(metadata.len());
+            }
+        }
+    }
+    Ok(total)
+}

benchmarks/semble/src/gates.rs

@@ -0,0 +1,324 @@
+//! Explicit quality floors for detecting retrieval regressions in CI.
+
+use std::{collections::BTreeMap, fs, path::Path};
+
+use serde::Deserialize;
+
+use crate::report::BenchmarkReport;
+
+const REQUIRED_TRACKS: [&str; 3] = ["natural_language", "literal", "symbol"];
+
+#[derive(Debug, Deserialize)]
+struct QualityGates {
+    schema_version: u32,
+    comparisons: ComparisonGates,
+    tracks: BTreeMap<String, QualityGate>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ComparisonGates {
+    cached_load: RatioGate,
+    symbol_latency: LatencyRatioGate,
+}
+
+#[derive(Debug, Deserialize)]
+struct RatioGate {
+    max_ratio: f64,
+}
+
+#[derive(Debug, Deserialize)]
+struct LatencyRatioGate {
+    max_p50_ratio: f64,
+    max_p95_ratio: f64,
+}
+
+#[derive(Debug, Deserialize)]
+struct QualityGate {
+    recall_at_1: f64,
+    recall_at_5: f64,
+    recall_at_10: f64,
+    mrr_at_10: f64,
+    ndcg_at_10: f64,
+}
+
+pub fn check(path: &Path, report: &BenchmarkReport) -> Result<(), String> {
+    let bytes = fs::read(path)
+        .map_err(|error| format!("read quality gates {}: {error}", path.display()))?;
+    let gates = serde_json::from_slice::<QualityGates>(&bytes)
+        .map_err(|error| format!("parse quality gates {}: {error}", path.display()))?;
+    if gates.schema_version != 1 {
+        return Err(format!(
+            "unsupported quality gate schema {} in {}",
+            gates.schema_version,
+            path.display()
+        ));
+    }
+    validate(&gates)?;
+    let mut failures = Vec::new();
+    for (track, gate) in &gates.tracks {
+        let Some(result) = report
+            .overall
+            .iter()
+            .find(|result| result.system == "Semble" && result.track == *track)
+        else {
+            failures.push(format!("missing Semble track {track}"));
+            continue;
+        };
+        let metrics = &result.effect;
+        compare(
+            &mut failures,
+            track,
+            "Recall@1",
+            metrics.recall_at_1,
+            gate.recall_at_1,
+        );
+        compare(
+            &mut failures,
+            track,
+            "Recall@5",
+            metrics.recall_at_5,
+            gate.recall_at_5,
+        );
+        compare(
+            &mut failures,
+            track,
+            "Recall@10",
+            metrics.recall_at_10,
+            gate.recall_at_10,
+        );
+        compare(
+            &mut failures,
+            track,
+            "MRR@10",
+            metrics.mrr_at_10,
+            gate.mrr_at_10,
+        );
+        compare(
+            &mut failures,
+            track,
+            "nDCG@10",
+            metrics.ndcg_at_10,
+            gate.ndcg_at_10,
+        );
+    }
+    for suite in &report.suites {
+        let Some(semble) = suite
+            .systems
+            .iter()
+            .find(|system| system.system == "Semble")
+        else {
+            failures.push(format!("{} is missing Semble results", suite.name));
+            continue;
+        };
+        let Some(codegraph) = suite
+            .systems
+            .iter()
+            .find(|system| system.system == "CodeGraph")
+        else {
+            continue;
+        };
+        compare_ratio(
+            &mut failures,
+            &suite.name,
+            "cached load",
+            semble.index.cached_load_ms,
+            codegraph.index.cached_load_ms,
+            gates.comparisons.cached_load.max_ratio,
+        );
+        let Some(semble_symbol) = semble.tracks.iter().find(|track| track.name == "symbol") else {
+            failures.push(format!("{} is missing Semble symbol results", suite.name));
+            continue;
+        };
+        let Some(codegraph_symbol) = codegraph.tracks.iter().find(|track| track.name == "symbol")
+        else {
+            failures.push(format!(
+                "{} is missing CodeGraph symbol results",
+                suite.name
+            ));
+            continue;
+        };
+        compare_latency(
+            &mut failures,
+            &suite.name,
+            "P50",
+            semble_symbol.query_latency.p50_ms,
+            codegraph_symbol.query_latency.p50_ms,
+            gates.comparisons.symbol_latency.max_p50_ratio,
+        );
+        compare_latency(
+            &mut failures,
+            &suite.name,
+            "P95",
+            semble_symbol.query_latency.p95_ms,
+            codegraph_symbol.query_latency.p95_ms,
+            gates.comparisons.symbol_latency.max_p95_ratio,
+        );
+    }
+    if failures.is_empty() {
+        Ok(())
+    } else {
+        Err(format!(
+            "quality gates failed:\n- {}",
+            failures.join("\n- ")
+        ))
+    }
+}
+
+fn validate(gates: &QualityGates) -> Result<(), String> {
+    for track in REQUIRED_TRACKS {
+        if !gates.tracks.contains_key(track) {
+            return Err(format!("quality gates are missing required track {track}"));
+        }
+    }
+    for (track, gate) in &gates.tracks {
+        for (metric, value) in [
+            ("Recall@1", gate.recall_at_1),
+            ("Recall@5", gate.recall_at_5),
+            ("Recall@10", gate.recall_at_10),
+            ("MRR@10", gate.mrr_at_10),
+            ("nDCG@10", gate.ndcg_at_10),
+        ] {
+            if !value.is_finite() || !(0.0..=1.0).contains(&value) {
+                return Err(format!(
+                    "quality gate {track} {metric} must be between 0 and 1"
+                ));
+            }
+        }
+    }
+    for (metric, value) in [
+        ("cached load ratio", gates.comparisons.cached_load.max_ratio),
+        (
+            "symbol latency P50 ratio",
+            gates.comparisons.symbol_latency.max_p50_ratio,
+        ),
+        (
+            "symbol latency P95 ratio",
+            gates.comparisons.symbol_latency.max_p95_ratio,
+        ),
+    ] {
+        if !value.is_finite() || value <= 0.0 || value > 1.0 {
+            return Err(format!(
+                "quality gate {metric} must be greater than 0 and at most 1"
+            ));
+        }
+    }
+    Ok(())
+}
+
+fn compare(failures: &mut Vec<String>, track: &str, metric: &str, actual: f64, minimum: f64) {
+    if actual + f64::EPSILON < minimum {
+        failures.push(format!(
+            "{track} {metric} was {actual:.3}, minimum is {minimum:.3}"
+        ));
+    }
+}
+
+fn compare_latency(
+    failures: &mut Vec<String>,
+    suite: &str,
+    percentile: &str,
+    semble_ms: f64,
+    codegraph_ms: f64,
+    max_ratio: f64,
+) {
+    compare_ratio(
+        failures,
+        suite,
+        &format!("symbol {percentile}"),
+        semble_ms,
+        codegraph_ms,
+        max_ratio,
+    );
+}
+
+fn compare_ratio(
+    failures: &mut Vec<String>,
+    suite: &str,
+    metric: &str,
+    semble_ms: f64,
+    codegraph_ms: f64,
+    max_ratio: f64,
+) {
+    let limit = codegraph_ms * max_ratio;
+    if semble_ms > limit {
+        failures.push(format!(
+            "{suite} {metric} was {semble_ms:.3} ms; limit is {limit:.3} ms ({max_ratio:.0}% of CodeGraph {codegraph_ms:.3} ms)"
+        ));
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn gate(value: f64) -> QualityGate {
+        QualityGate {
+            recall_at_1: value,
+            recall_at_5: value,
+            recall_at_10: value,
+            mrr_at_10: value,
+            ndcg_at_10: value,
+        }
+    }
+
+    fn comparisons() -> ComparisonGates {
+        ComparisonGates {
+            cached_load: RatioGate { max_ratio: 0.9 },
+            symbol_latency: LatencyRatioGate {
+                max_p50_ratio: 0.9,
+                max_p95_ratio: 0.9,
+            },
+        }
+    }
+
+    #[test]
+    fn rejects_incomplete_and_out_of_range_configuration() {
+        let mut tracks = BTreeMap::new();
+        tracks.insert("natural_language".into(), gate(0.5));
+        let gates = QualityGates {
+            schema_version: 1,
+            comparisons: comparisons(),
+            tracks,
+        };
+        assert!(validate(&gates).unwrap_err().contains("literal"));
+
+        let mut tracks = REQUIRED_TRACKS
+            .into_iter()
+            .map(|track| (track.to_owned(), gate(0.5)))
+            .collect::<BTreeMap<_, _>>();
+        tracks.insert("literal".into(), gate(1.1));
+        let gates = QualityGates {
+            schema_version: 1,
+            comparisons: comparisons(),
+            tracks,
+        };
+        assert!(validate(&gates).unwrap_err().contains("between 0 and 1"));
+    }
+
+    #[test]
+    fn comparison_only_records_values_below_the_floor() {
+        let mut failures = Vec::new();
+        compare(&mut failures, "literal", "Recall@1", 0.6, 0.6);
+        assert!(failures.is_empty());
+        compare(&mut failures, "literal", "Recall@1", 0.59, 0.6);
+        assert_eq!(failures.len(), 1);
+    }
+
+    #[test]
+    fn latency_comparison_enforces_the_configured_margin() {
+        let mut failures = Vec::new();
+        compare_latency(&mut failures, "react", "P50", 0.8, 1.0, 0.9);
+        assert!(failures.is_empty());
+        compare_latency(&mut failures, "react", "P95", 0.95, 1.0, 0.9);
+        assert_eq!(failures.len(), 1);
+    }
+
+    #[test]
+    fn cached_load_comparison_enforces_the_configured_margin() {
+        let mut failures = Vec::new();
+        compare_ratio(&mut failures, "react", "cached load", 80.0, 100.0, 0.9);
+        assert!(failures.is_empty());
+        compare_ratio(&mut failures, "react", "cached load", 95.0, 100.0, 0.9);
+        assert_eq!(failures.len(), 1);
+    }
+}

benchmarks/semble/src/main.rs

@@ -0,0 +1,710 @@
+//! Reproducible Semble and CodeGraph comparison on React and Vue.
+
+mod case;
+mod codegraph;
+mod gates;
+mod metrics;
+mod report;
+mod repository;
+
+use std::{
+    collections::{BTreeSet, HashMap},
+    env, fs,
+    path::{Path, PathBuf},
+    process::Command,
+    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
+};
+
+use case::{ExpectedLocation, QueryCase, Suite};
+use codegraph::{AdapterResult, AdapterTrack};
+use report::{
+    BenchmarkReport, Configuration, Environment, IndexMetrics, OverallReport, QueryReport,
+    ResultSummary, SuiteReport, SystemReport, TrackReport,
+};
+use semble_core::{
+    embedding::ModelAssets, ContentType, SearchEngine, SearchRequest, SearchResult, SembleConfig,
+};
+
+const TOP_K: usize = 10;
+const TRACKS: [&str; 3] = ["natural_language", "literal", "symbol"];
+
+fn main() {
+    if let Err(error) = run() {
+        eprintln!("semble-benchmark: {error}");
+        std::process::exit(1);
+    }
+}
+
+fn run() -> Result<(), String> {
+    let args = Args::parse()?;
+    if args.help {
+        print_help();
+        return Ok(());
+    }
+    let suites = case::load_suites(&args.cases_dir)?;
+    let sources = repository::resolve_sources(&suites, &args.sources_dir, &args.sources)?;
+    let codegraph_sources = if args.semble_only {
+        HashMap::new()
+    } else {
+        repository::prepare_competitor_sources(&suites, &sources, &args.codegraph_sources_dir)?
+    };
+    fs::create_dir_all(&args.cache_dir).map_err(|error| {
+        format!(
+            "create cache directory {}: {error}",
+            args.cache_dir.display()
+        )
+    })?;
+    if args.reset_indexes {
+        remove_indexes(&args.cache_dir)?;
+    }
+    ModelAssets::ensure(&args.cache_dir).map_err(|error| error.to_string())?;
+
+    let mut suite_reports = Vec::with_capacity(suites.len());
+    for suite in &suites {
+        println!("benchmarking {} at {}", suite.name, suite.commit);
+        let source = sources
+            .get(&suite.name)
+            .ok_or_else(|| format!("missing source path for {}", suite.name))?;
+        let codegraph_source = codegraph_sources.get(&suite.name).map(PathBuf::as_path);
+        suite_reports.push(benchmark_suite(suite, source, codegraph_source, &args)?);
+    }
+
+    let report = BenchmarkReport {
+        schema_version: 4,
+        generated_at_unix_seconds: SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs(),
+        environment: Environment {
+            os: env::consts::OS.into(),
+            architecture: env::consts::ARCH.into(),
+            rustc: rustc_version(),
+        },
+        configuration: Configuration {
+            top_k: TOP_K,
+            repetitions: args.repetitions,
+            tracks: TRACKS.iter().map(|track| (*track).to_owned()).collect(),
+            relevance:
+                "result path matches and result line range covers a labelled implementation line"
+                    .into(),
+        },
+        overall: overall(&suite_reports),
+        suites: suite_reports,
+    };
+    write_report(&args.json, &args.markdown, &report)?;
+    println!("wrote {}", args.json.display());
+    println!("wrote {}", args.markdown.display());
+    if args.check {
+        gates::check(&args.quality_gates, &report)?;
+        println!("quality gates passed");
+    }
+    Ok(())
+}
+
+fn benchmark_suite(
+    suite: &Suite,
+    source: &Path,
+    codegraph_source: Option<&Path>,
+    args: &Args,
+) -> Result<SuiteReport, String> {
+    validate_ground_truth(suite, source)?;
+    let semble = benchmark_semble(suite, source, &args.cache_dir, args.repetitions)?;
+    let mut systems = vec![semble];
+    if let Some(codegraph_source) = codegraph_source {
+        let codegraph = codegraph::benchmark(
+            &args.codegraph,
+            &args.codegraph_adapter,
+            codegraph_source,
+            suite,
+            args.repetitions,
+            TOP_K,
+        )?;
+        let cold_seconds = (codegraph.cold_index_ms / 1_000.0).max(f64::EPSILON);
+        systems.push(SystemReport {
+            system: "CodeGraph".into(),
+            version: codegraph.version,
+            index: IndexMetrics {
+                cold_ready_ms: codegraph.cold_index_ms,
+                cold_index_ms: codegraph.cold_index_ms,
+                cached_load_ms: codegraph.adapter.load_ms,
+                in_memory_prepare_ms: None,
+                indexed_files: codegraph.adapter.stats.file_count,
+                indexed_units: codegraph.adapter.stats.node_count,
+                indexed_unit: "nodes".into(),
+                indexed_edges: Some(codegraph.adapter.stats.edge_count),
+                persisted_index_bytes: codegraph.persisted_index_bytes,
+                source_bytes: None,
+                units_per_second: codegraph.adapter.stats.node_count as f64 / cold_seconds,
+                source_mib_per_second: None,
+            },
+            refresh_symbol_latency: None,
+            tracks: codegraph
+                .adapter
+                .tracks
+                .iter()
+                .map(|track| codegraph_track(track, suite))
+                .collect::<Result<Vec<_>, _>>()?,
+        });
+    }
+    Ok(SuiteReport {
+        name: suite.name.clone(),
+        repository: suite.repository.clone(),
+        commit: suite.commit.clone(),
+        systems,
+    })
+}
+
+fn benchmark_semble(
+    suite: &Suite,
+    source: &Path,
+    cache_dir: &Path,
+    repetitions: usize,
+) -> Result<SystemReport, String> {
+    let index_bytes_before = directory_size(&cache_dir.join("indexes"))?;
+    let model_started = Instant::now();
+    let engine = SearchEngine::load_default(SembleConfig::new(cache_dir))
+        .map_err(|error| error.to_string())?;
+    let model_load_ms = elapsed_ms(model_started.elapsed());
+    let cold_started = Instant::now();
+    let stats = engine
+        .prepare(source, &[ContentType::Code])
+        .map_err(|error| error.to_string())?;
+    let cold_duration = cold_started.elapsed();
+    let memory_started = Instant::now();
+    let memory_stats = engine
+        .prepare(source, &[ContentType::Code])
+        .map_err(|error| error.to_string())?;
+    let in_memory_prepare_ms = elapsed_ms(memory_started.elapsed());
+    if memory_stats != stats {
+        return Err(format!("{} in-memory index stats changed", suite.name));
+    }
+    drop(engine);
+
+    let reload_started = Instant::now();
+    let engine = SearchEngine::load_default(SembleConfig::new(cache_dir))
+        .map_err(|error| error.to_string())?;
+    let model_reload_ms = elapsed_ms(reload_started.elapsed());
+    let disk_started = Instant::now();
+    let disk_stats = engine
+        .prepare(source, &[ContentType::Code])
+        .map_err(|error| error.to_string())?;
+    let disk_index_load_ms = elapsed_ms(disk_started.elapsed());
+    if disk_stats != stats {
+        return Err(format!("{} persisted index stats changed", suite.name));
+    }
+    let tracks = vec![
+        semble_track(
+            &engine,
+            source,
+            suite,
+            "natural_language",
+            |query| &query.query,
+            repetitions,
+        )?,
+        semble_track(
+            &engine,
+            source,
+            suite,
+            "literal",
+            |query| &query.literal_query,
+            repetitions,
+        )?,
+        semble_track(
+            &engine,
+            source,
+            suite,
+            "symbol",
+            |query| &query.symbol_query,
+            repetitions,
+        )?,
+    ];
+    let refresh_symbol_latency = benchmark_refresh_symbol(&engine, source, suite, repetitions)?;
+    let index_bytes_after = directory_size(&cache_dir.join("indexes"))?;
+    let cold_seconds = cold_duration.as_secs_f64().max(f64::EPSILON);
+    Ok(SystemReport {
+        system: "Semble".into(),
+        version: env!("CARGO_PKG_VERSION").into(),
+        index: IndexMetrics {
+            cold_ready_ms: model_load_ms + elapsed_ms(cold_duration),
+            cold_index_ms: elapsed_ms(cold_duration),
+            cached_load_ms: model_reload_ms + disk_index_load_ms,
+            in_memory_prepare_ms: Some(in_memory_prepare_ms),
+            indexed_files: stats.file_count,
+            indexed_units: stats.chunk_count,
+            indexed_unit: "chunks".into(),
+            indexed_edges: None,
+            persisted_index_bytes: index_bytes_after.saturating_sub(index_bytes_before),
+            source_bytes: Some(stats.source_bytes),
+            units_per_second: stats.chunk_count as f64 / cold_seconds,
+            source_mib_per_second: Some(stats.source_bytes as f64 / 1_048_576.0 / cold_seconds),
+        },
+        refresh_symbol_latency: Some(refresh_symbol_latency),
+        tracks,
+    })
+}
+
+fn benchmark_refresh_symbol(
+    engine: &SearchEngine,
+    source: &Path,
+    suite: &Suite,
+    repetitions: usize,
+) -> Result<metrics::LatencyMetrics, String> {
+    let mut samples = Vec::with_capacity(suite.queries.len() * repetitions);
+    for case in &suite.queries {
+        for _ in 0..repetitions {
+            let started = Instant::now();
+            engine
+                .refresh(source, &[ContentType::Code])
+                .map_err(|error| format!("refresh before symbol query {}: {error}", case.id))?;
+            engine
+                .search(SearchRequest {
+                    query: case.symbol_query.clone(),
+                    repo: source.to_path_buf(),
+                    top_k: TOP_K,
+                    max_snippet_lines: Some(0),
+                    content: vec![ContentType::Code],
+                })
+                .map_err(|error| format!("symbol query after refresh {}: {error}", case.id))?;
+            samples.push(elapsed_ms(started.elapsed()));
+        }
+    }
+    Ok(metrics::latency(&samples))
+}
+
+fn validate_ground_truth(suite: &Suite, source: &Path) -> Result<(), String> {
+    for case in &suite.queries {
+        for expected in &case.expected {
+            let path = source.join(&expected.path);
+            let text = fs::read_to_string(&path).map_err(|error| {
+                format!(
+                    "ground truth {} / {} cannot read {}: {error}",
+                    suite.name,
+                    case.id,
+                    path.display()
+                )
+            })?;
+            let lines = text.lines().count();
+            if expected.line > lines {
+                return Err(format!(
+                    "ground truth {} / {} points past {} lines in {}",
+                    suite.name, case.id, lines, expected.path
+                ));
+            }
+        }
+    }
+    Ok(())
+}
+
+fn semble_track(
+    engine: &SearchEngine,
+    source: &Path,
+    suite: &Suite,
+    name: &str,
+    query_text: fn(&QueryCase) -> &str,
+    repetitions: usize,
+) -> Result<TrackReport, String> {
+    let mut queries = Vec::with_capacity(suite.queries.len());
+    let mut samples = Vec::with_capacity(suite.queries.len() * repetitions);
+    for case in &suite.queries {
+        let (report, durations) =
+            benchmark_semble_query(engine, source, case, query_text(case), repetitions)?;
+        queries.push(report);
+        samples.extend(durations);
+    }
+    Ok(track_report(name, queries, samples))
+}
+
+fn benchmark_semble_query(
+    engine: &SearchEngine,
+    source: &Path,
+    case: &QueryCase,
+    query: &str,
+    repetitions: usize,
+) -> Result<(QueryReport, Vec<f64>), String> {
+    let mut samples = Vec::with_capacity(repetitions);
+    let first_results = engine
+        .search(SearchRequest {
+            query: query.to_owned(),
+            repo: source.to_path_buf(),
+            top_k: TOP_K,
+            max_snippet_lines: Some(0),
+            content: vec![ContentType::Code],
+        })
+        .map_err(|error| format!("warm query {}: {error}", case.id))?
+        .results;
+    for _ in 0..repetitions {
+        let started = Instant::now();
+        engine
+            .search(SearchRequest {
+                query: query.to_owned(),
+                repo: source.to_path_buf(),
+                top_k: TOP_K,
+                max_snippet_lines: Some(0),
+                content: vec![ContentType::Code],
+            })
+            .map_err(|error| format!("query {}: {error}", case.id))?;
+        samples.push(elapsed_ms(started.elapsed()));
+    }
+    let summaries = first_results
+        .iter()
+        .enumerate()
+        .map(|(index, result)| summarize_semble(index + 1, result, &case.expected))
+        .collect::<Vec<_>>();
+    Ok((query_report(case, query, summaries, &samples), samples))
+}
+
+fn codegraph_track(track: &AdapterTrack, suite: &Suite) -> Result<TrackReport, String> {
+    let mut samples = Vec::new();
+    let mut queries = Vec::with_capacity(track.queries.len());
+    for query in &track.queries {
+        let case = suite
+            .queries
+            .iter()
+            .find(|case| case.id == query.id)
+            .ok_or_else(|| format!("CodeGraph returned unknown query id {}", query.id))?;
+        let summaries = query
+            .results
+            .iter()
+            .enumerate()
+            .map(|(index, result)| summarize_codegraph(index + 1, result, &case.expected))
+            .collect();
+        queries.push(query_report(
+            case,
+            &query.query,
+            summaries,
+            &query.durations_ms,
+        ));
+        samples.extend(query.durations_ms.iter().copied());
+    }
+    Ok(track_report(&track.name, queries, samples))
+}
+
+fn track_report(name: &str, queries: Vec<QueryReport>, samples: Vec<f64>) -> TrackReport {
+    let ranks = queries
+        .iter()
+        .map(|query| query.first_relevant_rank)
+        .collect::<Vec<_>>();
+    TrackReport {
+        name: name.into(),
+        effect: metrics::effect(&ranks),
+        query_latency: metrics::latency(&samples),
+        queries,
+    }
+}
+
+fn query_report(
+    case: &QueryCase,
+    query: &str,
+    results: Vec<ResultSummary>,
+    samples: &[f64],
+) -> QueryReport {
+    QueryReport {
+        id: case.id.clone(),
+        query: query.into(),
+        first_relevant_rank: results
+            .iter()
+            .find(|result| result.relevant)
+            .map(|result| result.rank),
+        latency: metrics::latency(samples),
+        expected: case
+            .expected
+            .iter()
+            .map(|location| format!("{}:{}", location.path, location.line))
+            .collect(),
+        results,
+    }
+}
+
+fn summarize_semble(
+    rank: usize,
+    result: &SearchResult,
+    expected: &[ExpectedLocation],
+) -> ResultSummary {
+    summary(
+        rank,
+        &result.file_path,
+        result.start_line,
+        result.end_line,
+        result.score,
+        expected,
+    )
+}
+
+fn summarize_codegraph(
+    rank: usize,
+    result: &AdapterResult,
+    expected: &[ExpectedLocation],
+) -> ResultSummary {
+    let mut output = summary(
+        rank,
+        &result.path,
+        result.start_line,
+        result.end_line,
+        result.score,
+        expected,
+    );
+    if let Some(lines) = result.lines.as_ref() {
+        output.relevant = expected
+            .iter()
+            .any(|location| location.path == result.path && lines.contains(&location.line));
+    }
+    output
+}
+
+fn summary(
+    rank: usize,
+    path: &str,
+    start_line: usize,
+    end_line: usize,
+    score: f32,
+    expected: &[ExpectedLocation],
+) -> ResultSummary {
+    ResultSummary {
+        rank,
+        path: path.into(),
+        start_line,
+        end_line,
+        score,
+        relevant: expected.iter().any(|location| {
+            location.path == path && start_line <= location.line && location.line <= end_line
+        }),
+    }
+}
+
+fn overall(suites: &[SuiteReport]) -> Vec<OverallReport> {
+    let systems = suites
+        .iter()
+        .flat_map(|suite| suite.systems.iter().map(|system| system.system.as_str()))
+        .collect::<BTreeSet<_>>();
+    systems
+        .into_iter()
+        .flat_map(|system_name| {
+            TRACKS.into_iter().map(move |track_name| {
+                let ranks = suites
+                    .iter()
+                    .filter_map(|suite| {
+                        suite
+                            .systems
+                            .iter()
+                            .find(|system| system.system == system_name)
+                    })
+                    .filter_map(|system| {
+                        system.tracks.iter().find(|track| track.name == track_name)
+                    })
+                    .flat_map(|track| track.queries.iter().map(|query| query.first_relevant_rank))
+                    .collect::<Vec<_>>();
+                OverallReport {
+                    system: system_name.into(),
+                    track: track_name.into(),
+                    effect: metrics::effect(&ranks),
+                }
+            })
+        })
+        .collect()
+}
+
+fn elapsed_ms(duration: Duration) -> f64 {
+    duration.as_secs_f64() * 1_000.0
+}
+
+fn directory_size(path: &Path) -> Result<u64, String> {
+    if !path.exists() {
+        return Ok(0);
+    }
+    let mut total = 0_u64;
+    let mut pending = vec![path.to_path_buf()];
+    while let Some(directory) = pending.pop() {
+        for entry in fs::read_dir(&directory)
+            .map_err(|error| format!("read {}: {error}", directory.display()))?
+        {
+            let entry = entry.map_err(|error| format!("read directory entry: {error}"))?;
+            let metadata = entry
+                .metadata()
+                .map_err(|error| format!("inspect {}: {error}", entry.path().display()))?;
+            if metadata.is_dir() {
+                pending.push(entry.path());
+            } else if metadata.is_file() {
+                total = total.saturating_add(metadata.len());
+            }
+        }
+    }
+    Ok(total)
+}
+
+fn remove_indexes(cache_dir: &Path) -> Result<(), String> {
+    let indexes = cache_dir.join("indexes");
+    if indexes.file_name().and_then(|name| name.to_str()) != Some("indexes") {
+        return Err(format!(
+            "refusing to remove unexpected path {}",
+            indexes.display()
+        ));
+    }
+    if indexes.exists() {
+        fs::remove_dir_all(&indexes)
+            .map_err(|error| format!("reset benchmark indexes {}: {error}", indexes.display()))?;
+    }
+    Ok(())
+}
+
+fn write_report(json: &Path, markdown: &Path, report: &BenchmarkReport) -> Result<(), String> {
+    for path in [json, markdown] {
+        if let Some(parent) = path
+            .parent()
+            .filter(|parent| !parent.as_os_str().is_empty())
+        {
+            fs::create_dir_all(parent).map_err(|error| {
+                format!("create report directory {}: {error}", parent.display())
+            })?;
+        }
+    }
+    let encoded = serde_json::to_vec_pretty(report)
+        .map_err(|error| format!("serialize benchmark report: {error}"))?;
+    fs::write(json, encoded).map_err(|error| format!("write {}: {error}", json.display()))?;
+    fs::write(markdown, report::markdown(report))
+        .map_err(|error| format!("write {}: {error}", markdown.display()))
+}
+
+fn rustc_version() -> String {
+    Command::new("rustc")
+        .arg("--version")
+        .output()
+        .ok()
+        .filter(|output| output.status.success())
+        .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned())
+        .unwrap_or_else(|| "unknown".into())
+}
+
+struct Args {
+    cases_dir: PathBuf,
+    sources_dir: PathBuf,
+    codegraph_sources_dir: PathBuf,
+    cache_dir: PathBuf,
+    json: PathBuf,
+    markdown: PathBuf,
+    codegraph_adapter: PathBuf,
+    codegraph: String,
+    quality_gates: PathBuf,
+    sources: HashMap<String, PathBuf>,
+    repetitions: usize,
+    reset_indexes: bool,
+    check: bool,
+    semble_only: bool,
+    help: bool,
+}
+
+impl Args {
+    fn parse() -> Result<Self, String> {
+        let root =
+            env::current_dir().map_err(|error| format!("read current directory: {error}"))?;
+        let mut parsed = Self {
+            cases_dir: root.join("benchmarks/semble/cases"),
+            sources_dir: root.join("target/semble-benchmark-sources"),
+            codegraph_sources_dir: root.join("target/semble-benchmark-codegraph-sources"),
+            cache_dir: root.join("target/semble-benchmark-cache"),
+            json: root.join("benchmarks/semble/results/latest.json"),
+            markdown: root.join("benchmarks/semble/results/latest.md"),
+            codegraph_adapter: root.join("benchmarks/semble/codegraph_adapter.cjs"),
+            codegraph: "codegraph".into(),
+            quality_gates: root.join("benchmarks/semble/quality-gates.json"),
+            sources: HashMap::new(),
+            repetitions: 5,
+            reset_indexes: true,
+            check: false,
+            semble_only: false,
+            help: false,
+        };
+        let mut arguments = env::args().skip(1);
+        while let Some(argument) = arguments.next() {
+            match argument.as_str() {
+                "--cases-dir" => parsed.cases_dir = value(&mut arguments, &argument)?.into(),
+                "--sources-dir" => parsed.sources_dir = value(&mut arguments, &argument)?.into(),
+                "--codegraph-sources-dir" => {
+                    parsed.codegraph_sources_dir = value(&mut arguments, &argument)?.into()
+                }
+                "--cache-dir" => parsed.cache_dir = value(&mut arguments, &argument)?.into(),
+                "--json" => parsed.json = value(&mut arguments, &argument)?.into(),
+                "--markdown" => parsed.markdown = value(&mut arguments, &argument)?.into(),
+                "--codegraph" => parsed.codegraph = value(&mut arguments, &argument)?,
+                "--codegraph-adapter" => {
+                    parsed.codegraph_adapter = value(&mut arguments, &argument)?.into()
+                }
+                "--quality-gates" => {
+                    parsed.quality_gates = value(&mut arguments, &argument)?.into()
+                }
+                "--source" => {
+                    let source = value(&mut arguments, &argument)?;
+                    let (name, path) = source
+                        .split_once('=')
+                        .ok_or_else(|| "--source must be NAME=PATH".to_owned())?;
+                    parsed.sources.insert(name.into(), path.into());
+                }
+                "--repetitions" => {
+                    parsed.repetitions = value(&mut arguments, &argument)?
+                        .parse()
+                        .map_err(|_| "--repetitions must be a positive integer".to_owned())?;
+                }
+                "--keep-indexes" => parsed.reset_indexes = false,
+                "--check" => parsed.check = true,
+                "--semble-only" => parsed.semble_only = true,
+                "-h" | "--help" => parsed.help = true,
+                _ => return Err(format!("unknown argument {argument}")),
+            }
+        }
+        if parsed.repetitions == 0 {
+            return Err("--repetitions must be greater than zero".into());
+        }
+        Ok(parsed)
+    }
+}
+
+fn value(arguments: &mut impl Iterator<Item = String>, option: &str) -> Result<String, String> {
+    arguments
+        .next()
+        .ok_or_else(|| format!("{option} requires a value"))
+}
+
+fn print_help() {
+    println!(
+        "Semble and CodeGraph React/Vue benchmark\n\n\
+         Options:\n\
+           --cases-dir PATH       benchmark suite JSON directory\n\
+           --sources-dir PATH     managed pinned Git checkouts\n\
+           --codegraph-sources-dir PATH  isolated CodeGraph Git worktrees\n\
+           --cache-dir PATH       isolated Semble model and index cache\n\
+           --codegraph PATH       CodeGraph executable (default: codegraph)\n\
+           --codegraph-adapter PATH  CodeGraph JSON adapter script\n\
+           --quality-gates PATH   retrieval quality floor configuration\n\
+           --source NAME=PATH     use an existing exact-commit checkout\n\
+           --repetitions N        warm search repetitions per query (default 5)\n\
+           --json PATH            JSON report path\n\
+           --markdown PATH        Markdown report path\n\
+           --keep-indexes         keep the existing Semble indexes\n\
+           --check                fail when Semble misses a quality floor\n\
+           --semble-only          skip CodeGraph for fast Semble iteration"
+    );
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn relevance_requires_path_and_line_overlap() {
+        let expected = [ExpectedLocation {
+            path: "src/lib.rs".into(),
+            line: 15,
+        }];
+        assert!(summary(1, "src/lib.rs", 10, 20, 1.0, &expected).relevant);
+        assert!(!summary(1, "src/lib.rs", 16, 20, 1.0, &expected).relevant);
+        assert!(!summary(1, "src/other.rs", 10, 20, 1.0, &expected).relevant);
+    }
+
+    #[test]
+    fn directory_size_sums_nested_files() {
+        let directory = tempfile::tempdir().unwrap();
+        fs::create_dir(directory.path().join("nested")).unwrap();
+        fs::write(directory.path().join("one"), [1_u8, 2]).unwrap();
+        fs::write(directory.path().join("nested/two"), [3_u8, 4, 5]).unwrap();
+        assert_eq!(directory_size(directory.path()).unwrap(), 5);
+    }
+}

benchmarks/semble/src/metrics.rs

@@ -0,0 +1,125 @@
+//! Retrieval metrics and latency summaries used by the benchmark report.
+
+use serde::Serialize;
+
+#[derive(Clone, Debug, Serialize)]
+pub struct EffectMetrics {
+    pub query_count: usize,
+    pub recall_at_1: f64,
+    pub recall_at_3: f64,
+    pub recall_at_5: f64,
+    pub recall_at_10: f64,
+    pub mrr_at_10: f64,
+    pub ndcg_at_10: f64,
+}
+
+#[derive(Clone, Debug, Serialize)]
+pub struct LatencyMetrics {
+    pub samples: usize,
+    pub min_ms: f64,
+    pub max_ms: f64,
+    pub mean_ms: f64,
+    pub stddev_ms: f64,
+    pub p50_ms: f64,
+    pub p95_ms: f64,
+    pub p99_ms: f64,
+}
+
+pub fn effect(ranks: &[Option<usize>]) -> EffectMetrics {
+    let count = ranks.len();
+    let denominator = count.max(1) as f64;
+    let recall = |limit: usize| {
+        ranks
+            .iter()
+            .filter(|rank| rank.is_some_and(|rank| rank <= limit))
+            .count() as f64
+            / denominator
+    };
+    let mrr = positive_zero(
+        ranks
+            .iter()
+            .filter_map(|rank| rank.filter(|rank| *rank <= 10))
+            .map(|rank| 1.0 / rank as f64)
+            .sum::<f64>()
+            / denominator,
+    );
+    let ndcg = positive_zero(
+        ranks
+            .iter()
+            .filter_map(|rank| rank.filter(|rank| *rank <= 10))
+            .map(|rank| 1.0 / (rank as f64 + 1.0).log2())
+            .sum::<f64>()
+            / denominator,
+    );
+    EffectMetrics {
+        query_count: count,
+        recall_at_1: recall(1),
+        recall_at_3: recall(3),
+        recall_at_5: recall(5),
+        recall_at_10: recall(10),
+        mrr_at_10: mrr,
+        ndcg_at_10: ndcg,
+    }
+}
+
+fn positive_zero(value: f64) -> f64 {
+    if value == 0.0 {
+        0.0
+    } else {
+        value
+    }
+}
+
+pub fn latency(samples_ms: &[f64]) -> LatencyMetrics {
+    let mut sorted = samples_ms.to_vec();
+    sorted.sort_by(f64::total_cmp);
+    let mean = sorted.iter().sum::<f64>() / sorted.len().max(1) as f64;
+    let variance = sorted
+        .iter()
+        .map(|sample| (sample - mean).powi(2))
+        .sum::<f64>()
+        / sorted.len().max(1) as f64;
+    LatencyMetrics {
+        samples: sorted.len(),
+        min_ms: sorted.first().copied().unwrap_or_default(),
+        max_ms: sorted.last().copied().unwrap_or_default(),
+        mean_ms: mean,
+        stddev_ms: variance.sqrt(),
+        p50_ms: percentile(&sorted, 0.50),
+        p95_ms: percentile(&sorted, 0.95),
+        p99_ms: percentile(&sorted, 0.99),
+    }
+}
+
+fn percentile(sorted: &[f64], fraction: f64) -> f64 {
+    if sorted.is_empty() {
+        return 0.0;
+    }
+    let index = ((sorted.len() - 1) as f64 * fraction).ceil() as usize;
+    sorted[index]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn computes_binary_relevance_metrics() {
+        let metrics = effect(&[Some(1), Some(4), None]);
+        assert!((metrics.recall_at_1 - 1.0 / 3.0).abs() < 1e-9);
+        assert!((metrics.recall_at_5 - 2.0 / 3.0).abs() < 1e-9);
+        assert!((metrics.mrr_at_10 - (1.0 + 0.25) / 3.0).abs() < 1e-9);
+    }
+
+    #[test]
+    fn reports_nearest_rank_percentiles() {
+        let metrics = latency(&[5.0, 1.0, 3.0, 2.0, 4.0]);
+        assert_eq!(metrics.mean_ms, 3.0);
+        assert_eq!(metrics.min_ms, 1.0);
+        assert_eq!(metrics.max_ms, 5.0);
+        assert_eq!(metrics.p50_ms, 3.0);
+        assert_eq!(metrics.p95_ms, 5.0);
+        assert_eq!(metrics.p99_ms, 5.0);
+        assert!((metrics.stddev_ms - 2.0_f64.sqrt()).abs() < 1e-9);
+    }
+}

benchmarks/semble/src/report.rs

@@ -0,0 +1,342 @@
+//! Machine-readable comparison report structures and Markdown rendering.
+
+use serde::Serialize;
+
+use crate::metrics::{EffectMetrics, LatencyMetrics};
+
+#[derive(Debug, Serialize)]
+pub struct BenchmarkReport {
+    pub schema_version: u32,
+    pub generated_at_unix_seconds: u64,
+    pub environment: Environment,
+    pub configuration: Configuration,
+    pub overall: Vec<OverallReport>,
+    pub suites: Vec<SuiteReport>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct Environment {
+    pub os: String,
+    pub architecture: String,
+    pub rustc: String,
+}
+
+#[derive(Debug, Serialize)]
+pub struct Configuration {
+    pub top_k: usize,
+    pub repetitions: usize,
+    pub tracks: Vec<String>,
+    pub relevance: String,
+}
+
+#[derive(Debug, Serialize)]
+pub struct OverallReport {
+    pub system: String,
+    pub track: String,
+    pub effect: EffectMetrics,
+}
+
+#[derive(Debug, Serialize)]
+pub struct SuiteReport {
+    pub name: String,
+    pub repository: String,
+    pub commit: String,
+    pub systems: Vec<SystemReport>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct SystemReport {
+    pub system: String,
+    pub version: String,
+    pub index: IndexMetrics,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub refresh_symbol_latency: Option<LatencyMetrics>,
+    pub tracks: Vec<TrackReport>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct IndexMetrics {
+    pub cold_ready_ms: f64,
+    pub cold_index_ms: f64,
+    pub cached_load_ms: f64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub in_memory_prepare_ms: Option<f64>,
+    pub indexed_files: usize,
+    pub indexed_units: usize,
+    pub indexed_unit: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub indexed_edges: Option<usize>,
+    pub persisted_index_bytes: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub source_bytes: Option<u64>,
+    pub units_per_second: f64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub source_mib_per_second: Option<f64>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct TrackReport {
+    pub name: String,
+    pub effect: EffectMetrics,
+    pub query_latency: LatencyMetrics,
+    pub queries: Vec<QueryReport>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct QueryReport {
+    pub id: String,
+    pub query: String,
+    pub first_relevant_rank: Option<usize>,
+    pub latency: LatencyMetrics,
+    pub expected: Vec<String>,
+    pub results: Vec<ResultSummary>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct ResultSummary {
+    pub rank: usize,
+    pub path: String,
+    pub start_line: usize,
+    pub end_line: usize,
+    pub score: f32,
+    pub relevant: bool,
+}
+
+pub fn markdown(report: &BenchmarkReport) -> String {
+    let mut output = String::new();
+    output.push_str("# Semble 与 CodeGraph React/Vue 对比基准\n\n");
+    output.push_str(&format!(
+        "- 环境：{} {}，{}\n- 查询：Top {}，每条重复 {} 次\n- natural_language：英文行为描述；literal：多词代码/错误字面片段；symbol：精确符号名\n- 对比：natural_language、literal 使用 CodeGraph explore；symbol 使用 CodeGraph searchNodes\n- 判定：返回代码范围必须覆盖人工标注的实现行\n\n",
+        report.environment.os,
+        report.environment.architecture,
+        report.environment.rustc,
+        report.configuration.top_k,
+        report.configuration.repetitions
+    ));
+    output.push_str("## 整体效果\n\n");
+    output.push_str("| 系统 | 查询轨道 | Recall@1 | Recall@5 | Recall@10 | MRR@10 | nDCG@10 |\n");
+    output.push_str("| --- | --- | ---: | ---: | ---: | ---: | ---: |\n");
+    for item in &report.overall {
+        output.push_str(&format!(
+            "| {} | {} | {:.1}% | {:.1}% | {:.1}% | {:.3} | {:.3} |\n",
+            item.system,
+            item.track,
+            item.effect.recall_at_1 * 100.0,
+            item.effect.recall_at_5 * 100.0,
+            item.effect.recall_at_10 * 100.0,
+            item.effect.mrr_at_10,
+            item.effect.ndcg_at_10,
+        ));
+    }
+    output.push_str("\n## 性能对比\n\n");
+    output.push_str("冷启动就绪包含运行时加载和首次索引；Semble 查询在一秒刷新窗口内直接复用已检查索引，窗口到期或显式 refresh 时重新校验源码指纹。缓存查询与 refresh＋symbol 分开计时；查询耗时均为持久进程中的实际工具处理耗时，不含 CLI 进程启动。每条查询先预热一次。\n\n");
+    output.push_str("| 数据集 | 系统 | 冷启动就绪 | 缓存加载 | 自然语言 P50 / P95 / σ | 字面 P50 / P95 / σ | 缓存符号 P50 / P95 / σ | 刷新＋符号 P50 / P95 | 文件 / 索引单元 | 索引体积 |\n");
+    output.push_str("| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n");
+    for suite in &report.suites {
+        for system in &suite.systems {
+            let natural = track(system, "natural_language");
+            let literal = track(system, "literal");
+            let symbol = track(system, "symbol");
+            output.push_str(&format!(
+                "| {} | {} {} | {:.1} ms | {:.1} ms | {:.2} / {:.2} / {:.2} ms | {:.2} / {:.2} / {:.2} ms | {:.2} / {:.2} / {:.2} ms | {} | {} / {} {}{} | {:.2} MiB |\n",
+                suite.name,
+                system.system,
+                system.version,
+                system.index.cold_ready_ms,
+                system.index.cached_load_ms,
+                natural.query_latency.p50_ms,
+                natural.query_latency.p95_ms,
+                natural.query_latency.stddev_ms,
+                literal.query_latency.p50_ms,
+                literal.query_latency.p95_ms,
+                literal.query_latency.stddev_ms,
+                symbol.query_latency.p50_ms,
+                symbol.query_latency.p95_ms,
+                symbol.query_latency.stddev_ms,
+                system
+                    .refresh_symbol_latency
+                    .as_ref()
+                    .map(|latency| format!("{:.2} / {:.2} ms", latency.p50_ms, latency.p95_ms))
+                    .unwrap_or_else(|| "—".into()),
+                system.index.indexed_files,
+                system.index.indexed_units,
+                system.index.indexed_unit,
+                system
+                    .index
+                    .indexed_edges
+                    .map(|edges| format!(" / {edges} edges"))
+                    .unwrap_or_default(),
+                as_mib(system.index.persisted_index_bytes),
+            ));
+        }
+    }
+    output.push_str("\n## 分数据集效果\n\n");
+    output.push_str("| 数据集 | 系统 | 轨道 | Recall@1 / @5 / @10 | MRR@10 | nDCG@10 |\n");
+    output.push_str("| --- | --- | --- | ---: | ---: | ---: |\n");
+    for suite in &report.suites {
+        for system in &suite.systems {
+            for track in &system.tracks {
+                output.push_str(&format!(
+                    "| {} | {} | {} | {:.1}% / {:.1}% / {:.1}% | {:.3} | {:.3} |\n",
+                    suite.name,
+                    system.system,
+                    track.name,
+                    track.effect.recall_at_1 * 100.0,
+                    track.effect.recall_at_5 * 100.0,
+                    track.effect.recall_at_10 * 100.0,
+                    track.effect.mrr_at_10,
+                    track.effect.ndcg_at_10,
+                ));
+            }
+        }
+    }
+    for suite in &report.suites {
+        for system in &suite.systems {
+            for track in &system.tracks {
+                output.push_str(&format!(
+                    "\n## {} · {} · {} 明细\n\n",
+                    suite.name, system.system, track.name
+                ));
+                output.push_str("| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |\n");
+                output.push_str("| --- | ---: | ---: | --- |\n");
+                for query in &track.queries {
+                    let rank = query
+                        .first_relevant_rank
+                        .map(|rank| rank.to_string())
+                        .unwrap_or_else(|| "未命中".into());
+                    let top = query
+                        .results
+                        .first()
+                        .map(|result| {
+                            format!("{}:{}-{}", result.path, result.start_line, result.end_line)
+                        })
+                        .unwrap_or_else(|| "—".into());
+                    output.push_str(&format!(
+                        "| `{}` | {} | {:.2} / {:.2} / {:.2} ms | `{}` |\n",
+                        query.id,
+                        rank,
+                        query.latency.p50_ms,
+                        query.latency.p95_ms,
+                        query.latency.stddev_ms,
+                        top
+                    ));
+                }
+            }
+        }
+    }
+    output.push_str("\n## 公平性说明\n\n");
+    output.push_str("自然语言和多词字面轨道调用 CodeGraph 官方推荐的 codegraph_explore，包含图扩展和最终源码读取；符号轨道调用 searchNodes。Semble 三条轨道都调用同一个混合搜索接口。所有查询共享同一组人工标注实现位置，未针对系统选择不同真值。两者输出粒度不同，因此本报告适合比较达到同一代码位置的效果和实际工具延迟，不代表各内部算法的微基准。CodeGraph 的独立 callers、callees 和 impact 能力不属于本次范围。\n");
+    output
+}
+
+fn track<'a>(system: &'a SystemReport, name: &str) -> &'a TrackReport {
+    system
+        .tracks
+        .iter()
+        .find(|track| track.name == name)
+        .expect("every system report contains all benchmark tracks")
+}
+
+fn as_mib(bytes: u64) -> f64 {
+    bytes as f64 / 1_048_576.0
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn effect() -> EffectMetrics {
+        EffectMetrics {
+            query_count: 1,
+            recall_at_1: 1.0,
+            recall_at_3: 1.0,
+            recall_at_5: 1.0,
+            recall_at_10: 1.0,
+            mrr_at_10: 1.0,
+            ndcg_at_10: 1.0,
+        }
+    }
+
+    fn track_report(name: &str) -> TrackReport {
+        TrackReport {
+            name: name.into(),
+            effect: effect(),
+            query_latency: LatencyMetrics {
+                samples: 1,
+                min_ms: 1.0,
+                max_ms: 1.0,
+                mean_ms: 1.0,
+                stddev_ms: 0.0,
+                p50_ms: 1.0,
+                p95_ms: 1.0,
+                p99_ms: 1.0,
+            },
+            queries: Vec::new(),
+        }
+    }
+
+    #[test]
+    fn markdown_contains_both_systems_and_tracks() {
+        let report = BenchmarkReport {
+            schema_version: 4,
+            generated_at_unix_seconds: 0,
+            environment: Environment {
+                os: "test".into(),
+                architecture: "test".into(),
+                rustc: "rustc test".into(),
+            },
+            configuration: Configuration {
+                top_k: 10,
+                repetitions: 3,
+                tracks: vec!["natural_language".into(), "literal".into(), "symbol".into()],
+                relevance: "location overlap".into(),
+            },
+            overall: vec![OverallReport {
+                system: "Semble".into(),
+                track: "natural_language".into(),
+                effect: effect(),
+            }],
+            suites: vec![SuiteReport {
+                name: "fixture".into(),
+                repository: "repo".into(),
+                commit: "commit".into(),
+                systems: vec![SystemReport {
+                    system: "Semble".into(),
+                    version: "0.1".into(),
+                    index: IndexMetrics {
+                        cold_ready_ms: 1.0,
+                        cold_index_ms: 1.0,
+                        cached_load_ms: 1.0,
+                        in_memory_prepare_ms: Some(1.0),
+                        indexed_files: 1,
+                        indexed_units: 2,
+                        indexed_unit: "chunks".into(),
+                        indexed_edges: None,
+                        persisted_index_bytes: 3,
+                        source_bytes: Some(4),
+                        units_per_second: 5.0,
+                        source_mib_per_second: Some(6.0),
+                    },
+                    refresh_symbol_latency: Some(LatencyMetrics {
+                        samples: 1,
+                        min_ms: 2.0,
+                        max_ms: 2.0,
+                        mean_ms: 2.0,
+                        stddev_ms: 0.0,
+                        p50_ms: 2.0,
+                        p95_ms: 2.0,
+                        p99_ms: 2.0,
+                    }),
+                    tracks: vec![
+                        track_report("natural_language"),
+                        track_report("literal"),
+                        track_report("symbol"),
+                    ],
+                }],
+            }],
+        };
+        let rendered = markdown(&report);
+        assert!(rendered.contains("Semble"));
+        assert!(rendered.contains("natural_language"));
+    }
+}

benchmarks/semble/src/repository.rs

@@ -0,0 +1,145 @@
+//! Deterministic Git checkout management for version-pinned benchmark sources.
+
+use std::{
+    collections::HashMap,
+    fs,
+    path::{Path, PathBuf},
+    process::Command,
+};
+
+use crate::case::Suite;
+
+pub fn resolve_sources(
+    suites: &[Suite],
+    root: &Path,
+    overrides: &HashMap<String, PathBuf>,
+) -> Result<HashMap<String, PathBuf>, String> {
+    fs::create_dir_all(root)
+        .map_err(|error| format!("create source directory {}: {error}", root.display()))?;
+    suites
+        .iter()
+        .map(|suite| {
+            let path = overrides
+                .get(&suite.name)
+                .cloned()
+                .unwrap_or_else(|| root.join(&suite.name));
+            if overrides.contains_key(&suite.name) {
+                verify_checkout(&path, &suite.commit)?;
+            } else {
+                ensure_checkout(&path, suite)?;
+            }
+            Ok((suite.name.clone(), path))
+        })
+        .collect()
+}
+
+pub fn prepare_competitor_sources(
+    suites: &[Suite],
+    sources: &HashMap<String, PathBuf>,
+    root: &Path,
+) -> Result<HashMap<String, PathBuf>, String> {
+    fs::create_dir_all(root).map_err(|error| {
+        format!(
+            "create competitor source directory {}: {error}",
+            root.display()
+        )
+    })?;
+    suites
+        .iter()
+        .map(|suite| {
+            let source = sources
+                .get(&suite.name)
+                .ok_or_else(|| format!("missing source path for {}", suite.name))?;
+            let target = root.join(&suite.name);
+            if target.exists() {
+                fs::remove_dir_all(&target).map_err(|error| {
+                    format!("reset competitor checkout {}: {error}", target.display())
+                })?;
+            }
+            run(
+                Command::new("git")
+                    .args(["clone", "--shared", "--no-checkout"])
+                    .arg(source)
+                    .arg(&target),
+                "clone isolated competitor checkout",
+            )?;
+            run(
+                Command::new("git").arg("-C").arg(&target).args([
+                    "checkout",
+                    "--detach",
+                    &suite.commit,
+                ]),
+                "checkout competitor commit",
+            )?;
+            Ok((suite.name.clone(), target))
+        })
+        .collect()
+}
+
+fn ensure_checkout(path: &Path, suite: &Suite) -> Result<(), String> {
+    if !path.join(".git").exists() {
+        if path.exists() {
+            fs::remove_dir_all(path).map_err(|error| {
+                format!("remove incomplete checkout {}: {error}", path.display())
+            })?;
+        }
+        run(
+            Command::new("git")
+                .args(["clone", "--filter=blob:none", "--no-checkout"])
+                .arg(&suite.repository)
+                .arg(path),
+            "clone repository",
+        )?;
+    }
+    if verify_checkout(path, &suite.commit).is_ok() {
+        return Ok(());
+    }
+    run(
+        Command::new("git").arg("-C").arg(path).args([
+            "fetch",
+            "--depth=1",
+            "origin",
+            &suite.commit,
+        ]),
+        "fetch pinned commit",
+    )?;
+    run(
+        Command::new("git")
+            .arg("-C")
+            .arg(path)
+            .args(["checkout", "--detach", &suite.commit]),
+        "checkout pinned commit",
+    )?;
+    verify_checkout(path, &suite.commit)
+}
+
+fn verify_checkout(path: &Path, expected: &str) -> Result<(), String> {
+    let output = Command::new("git")
+        .arg("-C")
+        .arg(path)
+        .args(["rev-parse", "HEAD"])
+        .output()
+        .map_err(|error| format!("inspect checkout {}: {error}", path.display()))?;
+    if !output.status.success() {
+        return Err(format!("{} is not a readable Git checkout", path.display()));
+    }
+    let actual = String::from_utf8_lossy(&output.stdout).trim().to_owned();
+    if actual != expected {
+        return Err(format!(
+            "checkout {} is at {actual}, expected {expected}",
+            path.display()
+        ));
+    }
+    Ok(())
+}
+
+fn run(command: &mut Command, action: &str) -> Result<(), String> {
+    let status = command
+        .status()
+        .map_err(|error| format!("{action}: {error}"))?;
+    if status.success() {
+        Ok(())
+    } else {
+        Err(format!("{action} failed with {status}"))
+    }
+}