mirror of
https://wget.la/https://github.com/leookun/cursor-byok
synced 2026-10-05 12:13:05 +08:00
refactor: rebuild desktop app with Tauri
This commit is contained in:
leookun
@@ -0,0 +1,14 @@
|
||||
# Reproducible performance and retrieval-quality benchmark for Semble.
|
||||
[package]
|
||||
name = "semble-benchmark"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
semble-core = { path = "../../crates/semble-core" }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
@@ -0,0 +1,64 @@
|
||||
<!-- Reproducible benchmark protocol for Semble's React and Vue evaluation. -->
|
||||
# Semble / CodeGraph React/Vue 对比基准
|
||||
|
||||
这个基准在相同的 React、Vue 提交上比较 Semble 和本机安装的 CodeGraph。每条查询都人工标注到具体实现行，只有返回代码范围覆盖该行才算命中。
|
||||
|
||||
为了兼顾不同检索方式，效果分为三条轨道：
|
||||
|
||||
- `natural_language`：Semble `search` 对比 CodeGraph 官方推荐的 `codegraph_explore`；
|
||||
- `literal`：多词代码片段、错误文本或相邻标识符组合，两边均通过各自的产品级搜索入口检索；
|
||||
- `symbol`：Semble `search` 对比 CodeGraph `searchNodes`，输入相同精确符号名。
|
||||
|
||||
三条轨道复用相同的人工标注实现位置。每次运行还会检查标注文件存在且行号没有越界，避免失效真值产生虚假分数。
|
||||
|
||||
## 指标
|
||||
|
||||
性能包含：
|
||||
|
||||
- 冷启动到可查询、冷索引、索引单元吞吐；
|
||||
- 持久索引加载时间；
|
||||
- 缓存热查询 min、max、mean、标准差、P50、P95、P99；
|
||||
- Semble 显式刷新索引后执行符号查询的独立延迟；
|
||||
- 索引文件、代码块或图节点、图边、持久化索引体积。
|
||||
|
||||
效果包含：
|
||||
|
||||
- Recall@1、Recall@3、Recall@5、Recall@10；
|
||||
- MRR@10；
|
||||
- nDCG@10；
|
||||
- 每条查询的首个命中排名和 Top 10 结果明细。
|
||||
|
||||
## 运行
|
||||
|
||||
需要本机 `PATH` 中存在 `codegraph`。从工作区根目录运行：
|
||||
|
||||
```sh
|
||||
cargo run --release -p semble-benchmark
|
||||
```
|
||||
|
||||
需要用于 CI 或本地回归检查时增加 `--check`。跑分报告仍会正常生成，但任一 Semble 轨道低于 [`quality-gates.json`](./quality-gates.json) 中的公开质量下限时，命令会返回失败。完整双系统报告还要求 Semble 的缓存加载，以及符号查询的 P50、P95，均不超过 CodeGraph 的 90%：
|
||||
|
||||
```sh
|
||||
cargo run --release -p semble-benchmark -- --check
|
||||
```
|
||||
|
||||
只验证 Semble 查询效果和质量门槛、跳过耗时较长的 CodeGraph 重建时使用：
|
||||
|
||||
```sh
|
||||
cargo run --release -p semble-benchmark -- --semble-only --check
|
||||
```
|
||||
|
||||
工具默认自行拉取固定提交，并清空隔离的基准索引，模型缓存保留。CodeGraph 使用位于 `target` 下的独立 Git 工作副本，不会创建、覆盖或删除传入代码库中的 `.codegraph`。已有精确提交的工作区可以复用：
|
||||
|
||||
```sh
|
||||
cargo run --release -p semble-benchmark -- \
|
||||
--source react=/path/to/react \
|
||||
--source vue=/path/to/vue \
|
||||
--repetitions 5
|
||||
```
|
||||
|
||||
结果写入 `results/latest.json` 和 `results/latest.md`。JSON 用于后续回归比较，Markdown 用于人工审阅。运行时环境、提交、参数和逐查询结果都会写进报告。
|
||||
|
||||
## 解读限制
|
||||
|
||||
这是代码定位基准，不评价生成答案本身。`codegraph_explore` 的图扩展和源码读取计入实际工具耗时，因此这是产品路径对比，不是内部算法微基准。Semble 同时持久化增量构建快照与可直接反序列化的运行时索引；加载后在固定一秒刷新窗口内复用已检查的内存索引，窗口到期后的首次查询或显式 `refresh` 会并行扫描文件元数据，并且只重新处理变化文件。报告将缓存查询与刷新加符号查询分开记录。CodeGraph 的加载指标打开已准备图数据库。CodeGraph 的独立 callers、callees、impact 能力不在本次范围内。标注集规模较小，适合防止检索质量回退，不应被解释为覆盖所有 React/Vue 开发问题的总体准确率。每条查询先执行一次不计时预热，再记录五次；报告中的标准差用于识别抖动，但跨版本性能比较仍应在同一机器、同一电源状态下至少重复三轮。
|
||||
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"_comment": "React benchmark ground truth pinned to one immutable upstream commit.",
|
||||
"name": "react",
|
||||
"repository": "https://github.com/react/react.git",
|
||||
"commit": "eafeac097ba51e1eab809c07102126bd5f8e5425",
|
||||
"queries": [
|
||||
{
|
||||
"id": "state-hook-api",
|
||||
"query": "where is the public useState hook implemented",
|
||||
"literal_query": "export function useState",
|
||||
"symbol_query": "useState",
|
||||
"expected": [{ "path": "packages/react/src/ReactHooks.js", "line": 66 }]
|
||||
},
|
||||
{
|
||||
"id": "create-context",
|
||||
"query": "create a React context object from a default value",
|
||||
"literal_query": "REACT_CONTEXT_TYPE defaultValue ReactContext",
|
||||
"symbol_query": "createContext",
|
||||
"expected": [{ "path": "packages/react/src/ReactContext.js", "line": 14 }]
|
||||
},
|
||||
{
|
||||
"id": "schedule-fiber-update",
|
||||
"query": "schedule an update on a Fiber root",
|
||||
"literal_query": "scheduleUpdateOnFiber FiberRoot fiber lane",
|
||||
"symbol_query": "scheduleUpdateOnFiber",
|
||||
"expected": [{ "path": "packages/react-reconciler/src/ReactFiberWorkLoop.js", "line": 987 }]
|
||||
},
|
||||
{
|
||||
"id": "commit-root",
|
||||
"query": "commit the finished React root and flush its effects",
|
||||
"literal_query": "commitRoot root finishedWork lanes",
|
||||
"symbol_query": "commitRoot",
|
||||
"expected": [{ "path": "packages/react-reconciler/src/ReactFiberWorkLoop.js", "line": 3725 }]
|
||||
},
|
||||
{
|
||||
"id": "suspense-begin-work",
|
||||
"query": "update a Suspense component during Fiber begin work",
|
||||
"literal_query": "updateSuspenseComponent workInProgress renderLanes",
|
||||
"symbol_query": "updateSuspenseComponent",
|
||||
"expected": [{ "path": "packages/react-reconciler/src/ReactFiberBeginWork.js", "line": 2383 }]
|
||||
},
|
||||
{
|
||||
"id": "delegated-dom-events",
|
||||
"query": "register delegated listeners for every supported DOM event",
|
||||
"literal_query": "listeningMarker rootContainerElement EventTarget",
|
||||
"symbol_query": "listenToAllSupportedEvents",
|
||||
"expected": [{ "path": "packages/react-dom-bindings/src/events/DOMPluginEventSystem.js", "line": 432 }]
|
||||
},
|
||||
{
|
||||
"id": "create-dom-root",
|
||||
"query": "create a concurrent React DOM root",
|
||||
"literal_query": "createRoot CreateRootOptions RootType container",
|
||||
"symbol_query": "createRoot",
|
||||
"expected": [
|
||||
{ "path": "packages/react-dom/src/client/ReactDOMRoot.js", "line": 171 },
|
||||
{ "path": "packages/react-dom/src/client/ReactDOMRootFB.js", "line": 131 }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "reconcile-keyed-array",
|
||||
"query": "reconcile an array of keyed React children",
|
||||
"literal_query": "reconcileChildrenArray currentFirstChild newChildren lanes",
|
||||
"symbol_query": "reconcileChildrenArray",
|
||||
"expected": [{ "path": "packages/react-reconciler/src/ReactChildFiber.js", "line": 1177 }]
|
||||
},
|
||||
{
|
||||
"id": "dispatch-state-update",
|
||||
"query": "dispatch a useState hook update",
|
||||
"literal_query": "dispatchSetState fiber queue action",
|
||||
"symbol_query": "dispatchSetState",
|
||||
"expected": [{ "path": "packages/react-reconciler/src/ReactFiberHooks.js", "line": 3626 }]
|
||||
},
|
||||
{
|
||||
"id": "hydrate-dom-root",
|
||||
"query": "hydrate a server-rendered React DOM root",
|
||||
"literal_query": "hydrateRoot initialChildren HydrateRootOptions",
|
||||
"symbol_query": "hydrateRoot",
|
||||
"expected": [
|
||||
{ "path": "packages/react-dom/src/client/ReactDOMRoot.js", "line": 276 },
|
||||
{ "path": "packages/react-dom/src/client/ReactDOMRootFB.js", "line": 148 }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"_comment": "Vue benchmark ground truth pinned to one immutable upstream commit.",
|
||||
"name": "vue",
|
||||
"repository": "https://github.com/vuejs/core.git",
|
||||
"commit": "a2b40db9a83b36ed9da3a16403cf8f040262d73f",
|
||||
"queries": [
|
||||
{
|
||||
"id": "reactive-proxy",
|
||||
"query": "turn an object into a deeply reactive Vue proxy",
|
||||
"literal_query": "createReactiveObject mutableHandlers mutableCollectionHandlers",
|
||||
"symbol_query": "reactive",
|
||||
"expected": [{ "path": "packages/reactivity/src/reactive.ts", "line": 87 }]
|
||||
},
|
||||
{
|
||||
"id": "create-ref",
|
||||
"query": "create a Vue ref around a value",
|
||||
"literal_query": "IfAny UnwrapRef createRef",
|
||||
"symbol_query": "ref",
|
||||
"expected": [{ "path": "packages/reactivity/src/ref.ts", "line": 64 }]
|
||||
},
|
||||
{
|
||||
"id": "create-renderer",
|
||||
"query": "create a custom Vue renderer from host operations",
|
||||
"literal_query": "baseCreateRenderer RendererOptions HostElement",
|
||||
"symbol_query": "createRenderer",
|
||||
"expected": [{ "path": "packages/runtime-core/src/renderer.ts", "line": 318 }]
|
||||
},
|
||||
{
|
||||
"id": "patch-keyed-children",
|
||||
"query": "diff keyed children using the longest increasing subsequence",
|
||||
"literal_query": "patchKeyedChildren VNodeArrayChildren parentAnchor",
|
||||
"symbol_query": "patchKeyedChildren",
|
||||
"expected": [{ "path": "packages/runtime-core/src/renderer.ts", "line": 1804 }]
|
||||
},
|
||||
{
|
||||
"id": "compile-template",
|
||||
"query": "compile a Vue template AST into render function code",
|
||||
"literal_query": "baseCompile source options TransformOptions",
|
||||
"symbol_query": "baseCompile",
|
||||
"expected": [{ "path": "packages/compiler-core/src/compile.ts", "line": 67 }]
|
||||
},
|
||||
{
|
||||
"id": "parse-template",
|
||||
"query": "parse a Vue template string into a root AST",
|
||||
"literal_query": "baseParse currentInput defaultParserOptions",
|
||||
"symbol_query": "baseParse",
|
||||
"expected": [{ "path": "packages/compiler-core/src/parser.ts", "line": 1028 }]
|
||||
},
|
||||
{
|
||||
"id": "reactivity-watch",
|
||||
"query": "watch reactive sources and run a callback when values change",
|
||||
"literal_query": "watch source cb options WatchOptions",
|
||||
"symbol_query": "watch",
|
||||
"expected": [{ "path": "packages/reactivity/src/watch.ts", "line": 120 }]
|
||||
},
|
||||
{
|
||||
"id": "create-app-api",
|
||||
"query": "create a Vue application instance and mount its root component",
|
||||
"literal_query": "createApp rootComponent rootProps",
|
||||
"symbol_query": "createAppAPI",
|
||||
"expected": [{ "path": "packages/runtime-core/src/apiCreateApp.ts", "line": 253 }]
|
||||
},
|
||||
{
|
||||
"id": "define-component",
|
||||
"query": "define a Vue component while preserving type inference",
|
||||
"literal_query": "isFunction options extraOptions setup",
|
||||
"symbol_query": "defineComponent",
|
||||
"expected": [{ "path": "packages/runtime-core/src/apiDefineComponent.ts", "line": 305 }]
|
||||
},
|
||||
{
|
||||
"id": "hydrate-node",
|
||||
"query": "hydrate server-rendered DOM nodes into Vue virtual nodes",
|
||||
"literal_query": "hydrateNode container firstChild vnode",
|
||||
"symbol_query": "hydrateNode",
|
||||
"expected": [{ "path": "packages/runtime-core/src/hydration.ts", "line": 137 }]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
// Persistent-process adapter for measuring CodeGraph library search without CLI startup noise.
|
||||
|
||||
'use strict'
|
||||
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
|
||||
async function main() {
|
||||
const [libraryRoot, projectRoot, repetitionsText, limitText] = process.argv.slice(2)
|
||||
const repetitions = Number(repetitionsText)
|
||||
const limit = Number(limitText)
|
||||
const input = JSON.parse(fs.readFileSync(0, 'utf8'))
|
||||
const loadStarted = process.hrtime.bigint()
|
||||
const codegraphModule = require(path.join(libraryRoot, 'dist/index.js'))
|
||||
const generatedModule = require(path.join(
|
||||
libraryRoot,
|
||||
'dist/extraction/generated-detection.js',
|
||||
))
|
||||
const { ToolHandler } = require(path.join(libraryRoot, 'dist/mcp/tools.js'))
|
||||
const CodeGraph = codegraphModule.CodeGraph || codegraphModule.default
|
||||
const graph = await CodeGraph.open(projectRoot, { sync: false, readOnly: true })
|
||||
const loadMs = millisecondsSince(loadStarted)
|
||||
const stats = graph.getStats()
|
||||
const handler = new ToolHandler(graph)
|
||||
const tracks = []
|
||||
for (const track of input.tracks) {
|
||||
const queries = []
|
||||
for (const query of track.queries) {
|
||||
queries.push(
|
||||
track.name !== 'symbol'
|
||||
? await runExplore(handler, query, repetitions, limit)
|
||||
: runSearch(graph, generatedModule, query, repetitions, limit),
|
||||
)
|
||||
}
|
||||
tracks.push({ name: track.name, queries })
|
||||
}
|
||||
graph.close()
|
||||
process.stdout.write(JSON.stringify({ loadMs, stats, tracks }))
|
||||
}
|
||||
|
||||
async function runExplore(handler, query, repetitions, limit) {
|
||||
const first = await explore(handler, query.query, limit)
|
||||
const durationsMs = []
|
||||
for (let index = 0; index < repetitions; index += 1) {
|
||||
const started = process.hrtime.bigint()
|
||||
await explore(handler, query.query, limit)
|
||||
durationsMs.push(millisecondsSince(started))
|
||||
}
|
||||
return { id: query.id, query: query.query, durationsMs, results: first }
|
||||
}
|
||||
|
||||
async function explore(handler, query, limit) {
|
||||
const response = await handler.execute('codegraph_explore', { query, maxFiles: limit })
|
||||
if (response.isError) throw new Error(response.content?.[0]?.text || 'CodeGraph explore failed')
|
||||
return parseExplore(response.content?.[0]?.text || '')
|
||||
}
|
||||
|
||||
function parseExplore(text) {
|
||||
const results = []
|
||||
let current = null
|
||||
let inCode = false
|
||||
for (const line of text.split('\n')) {
|
||||
const heading = line.match(/^####\s+(.+?)(?:\s+—|$)/)
|
||||
if (heading) {
|
||||
current = { path: heading[1].trim(), startLine: 0, endLine: 0, score: 0, lines: [] }
|
||||
results.push(current)
|
||||
inCode = false
|
||||
continue
|
||||
}
|
||||
if (line.startsWith('```')) {
|
||||
inCode = !inCode
|
||||
continue
|
||||
}
|
||||
if (!inCode || !current) continue
|
||||
const numbered = line.match(/^(\d+)\t/)
|
||||
if (!numbered) continue
|
||||
const number = Number(numbered[1])
|
||||
current.lines.push(number)
|
||||
if (current.startLine === 0) current.startLine = number
|
||||
current.endLine = number
|
||||
}
|
||||
return results
|
||||
.filter((result) => result.lines.length > 0)
|
||||
.map((result, index) => ({ ...result, score: 1 / (index + 1) }))
|
||||
}
|
||||
|
||||
function runSearch(graph, generatedModule, query, repetitions, limit) {
|
||||
const durationsMs = []
|
||||
const first = sortedResults(graph, generatedModule, query.query, limit)
|
||||
for (let index = 0; index < repetitions; index += 1) {
|
||||
const started = process.hrtime.bigint()
|
||||
sortedResults(graph, generatedModule, query.query, limit)
|
||||
durationsMs.push(millisecondsSince(started))
|
||||
}
|
||||
return {
|
||||
id: query.id,
|
||||
query: query.query,
|
||||
durationsMs,
|
||||
results: first.map(({ node, score }) => ({
|
||||
path: node.filePath,
|
||||
startLine: node.startLine,
|
||||
endLine: node.endLine,
|
||||
score,
|
||||
lines: null,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
function sortedResults(graph, generatedModule, query, limit) {
|
||||
const raw = graph.searchNodes(query, { limit })
|
||||
return [...raw].sort((left, right) => {
|
||||
const leftGenerated = generatedModule.isGeneratedFile(left.node.filePath) ? 1 : 0
|
||||
const rightGenerated = generatedModule.isGeneratedFile(right.node.filePath) ? 1 : 0
|
||||
return leftGenerated - rightGenerated
|
||||
})
|
||||
}
|
||||
|
||||
function millisecondsSince(started) {
|
||||
return Number(process.hrtime.bigint() - started) / 1_000_000
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error.stack || error}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"comparisons": {
|
||||
"cached_load": {
|
||||
"max_ratio": 0.90
|
||||
},
|
||||
"symbol_latency": {
|
||||
"max_p50_ratio": 0.90,
|
||||
"max_p95_ratio": 0.90
|
||||
}
|
||||
},
|
||||
"tracks": {
|
||||
"natural_language": {
|
||||
"recall_at_1": 0.50,
|
||||
"recall_at_5": 0.90,
|
||||
"recall_at_10": 0.90,
|
||||
"mrr_at_10": 0.65,
|
||||
"ndcg_at_10": 0.70
|
||||
},
|
||||
"literal": {
|
||||
"recall_at_1": 0.60,
|
||||
"recall_at_5": 0.85,
|
||||
"recall_at_10": 0.90,
|
||||
"mrr_at_10": 0.70,
|
||||
"ndcg_at_10": 0.75
|
||||
},
|
||||
"symbol": {
|
||||
"recall_at_1": 1.0,
|
||||
"recall_at_5": 1.0,
|
||||
"recall_at_10": 1.0,
|
||||
"mrr_at_10": 1.0,
|
||||
"ndcg_at_10": 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,230 @@
|
||||
# Semble 与 CodeGraph React/Vue 对比基准
|
||||
|
||||
- 环境：macos aarch64，rustc 1.97.1 (8bab26f4f 2026-07-14)
|
||||
- 查询：Top 10，每条重复 5 次
|
||||
- natural_language：英文行为描述；literal：多词代码/错误字面片段；symbol：精确符号名
|
||||
- 对比：natural_language、literal 使用 CodeGraph explore；symbol 使用 CodeGraph searchNodes
|
||||
- 判定：返回代码范围必须覆盖人工标注的实现行
|
||||
|
||||
## 整体效果
|
||||
|
||||
| 系统 | 查询轨道 | Recall@1 | Recall@5 | Recall@10 | MRR@10 | nDCG@10 |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| CodeGraph | natural_language | 10.0% | 20.0% | 20.0% | 0.150 | 0.163 |
|
||||
| CodeGraph | literal | 25.0% | 25.0% | 25.0% | 0.250 | 0.250 |
|
||||
| CodeGraph | symbol | 50.0% | 50.0% | 50.0% | 0.500 | 0.500 |
|
||||
| Semble | natural_language | 55.0% | 95.0% | 95.0% | 0.729 | 0.786 |
|
||||
| Semble | literal | 60.0% | 100.0% | 100.0% | 0.750 | 0.813 |
|
||||
| Semble | symbol | 100.0% | 100.0% | 100.0% | 1.000 | 1.000 |
|
||||
|
||||
## 性能对比
|
||||
|
||||
冷启动就绪包含运行时加载和首次索引；Semble 查询在一秒刷新窗口内直接复用已检查索引，窗口到期或显式 refresh 时重新校验源码指纹。缓存查询与 refresh＋symbol 分开计时；查询耗时均为持久进程中的实际工具处理耗时，不含 CLI 进程启动。每条查询先预热一次。
|
||||
|
||||
| 数据集 | 系统 | 冷启动就绪 | 缓存加载 | 自然语言 P50 / P95 / σ | 字面 P50 / P95 / σ | 缓存符号 P50 / P95 / σ | 刷新＋符号 P50 / P95 | 文件 / 索引单元 | 索引体积 |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| react | Semble 0.1.0 | 5797.8 ms | 65.3 ms | 11.22 / 12.39 / 0.57 ms | 11.21 / 12.64 / 2.50 ms | 0.01 / 0.01 / 0.00 ms | 13.89 / 15.40 ms | 4760 / 45275 chunks | 57.35 MiB |
|
||||
| react | CodeGraph 1.0.1 | 36601.0 ms | 96.0 ms | 230.27 / 275.19 / 32.93 ms | 235.11 / 294.48 / 30.22 ms | 2.73 / 14.48 / 4.87 ms | — | 4584 / 53295 nodes / 163188 edges | 317.67 MiB |
|
||||
| vue | Semble 0.1.0 | 826.1 ms | 12.1 ms | 1.88 / 2.18 / 0.18 ms | 1.96 / 2.41 / 0.23 ms | 0.01 / 0.01 / 0.00 ms | 4.66 / 4.93 ms | 554 / 7933 chunks | 9.24 MiB |
|
||||
| vue | CodeGraph 1.0.1 | 5232.5 ms | 46.8 ms | 55.94 / 72.82 / 8.29 ms | 41.13 / 50.20 / 3.69 ms | 0.57 / 1.96 / 0.66 ms | — | 552 / 6381 nodes / 44277 edges | 33.33 MiB |
|
||||
|
||||
## 分数据集效果
|
||||
|
||||
| 数据集 | 系统 | 轨道 | Recall@1 / @5 / @10 | MRR@10 | nDCG@10 |
|
||||
| --- | --- | --- | ---: | ---: | ---: |
|
||||
| react | Semble | natural_language | 70.0% / 100.0% / 100.0% | 0.850 | 0.889 |
|
||||
| react | Semble | literal | 70.0% / 100.0% / 100.0% | 0.792 | 0.843 |
|
||||
| react | Semble | symbol | 100.0% / 100.0% / 100.0% | 1.000 | 1.000 |
|
||||
| react | CodeGraph | natural_language | 0.0% / 0.0% / 0.0% | 0.000 | 0.000 |
|
||||
| react | CodeGraph | literal | 10.0% / 10.0% / 10.0% | 0.100 | 0.100 |
|
||||
| react | CodeGraph | symbol | 20.0% / 20.0% / 20.0% | 0.200 | 0.200 |
|
||||
| vue | Semble | natural_language | 40.0% / 90.0% / 90.0% | 0.608 | 0.682 |
|
||||
| vue | Semble | literal | 50.0% / 100.0% / 100.0% | 0.708 | 0.782 |
|
||||
| vue | Semble | symbol | 100.0% / 100.0% / 100.0% | 1.000 | 1.000 |
|
||||
| vue | CodeGraph | natural_language | 20.0% / 40.0% / 40.0% | 0.300 | 0.326 |
|
||||
| vue | CodeGraph | literal | 40.0% / 40.0% / 40.0% | 0.400 | 0.400 |
|
||||
| vue | CodeGraph | symbol | 80.0% / 80.0% / 80.0% | 0.800 | 0.800 |
|
||||
|
||||
## react · Semble · natural_language 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `state-hook-api` | 1 | 11.16 / 11.40 / 0.10 ms | `packages/react/src/ReactHooks.js:44-66` |
|
||||
| `create-context` | 1 | 12.25 / 12.44 / 0.16 ms | `packages/react/src/ReactContext.js:1-18` |
|
||||
| `schedule-fiber-update` | 1 | 10.89 / 11.11 / 0.12 ms | `packages/react-reconciler/src/ReactFiberWorkLoop.js:972-1005` |
|
||||
| `commit-root` | 2 | 12.02 / 12.39 / 0.21 ms | `packages/react-reconciler/src/ReactFiberDevToolsHook.js:84-111` |
|
||||
| `suspense-begin-work` | 1 | 10.70 / 10.89 / 0.13 ms | `packages/react-reconciler/src/ReactFiberBeginWork.js:2369-2387` |
|
||||
| `delegated-dom-events` | 1 | 10.28 / 10.55 / 0.18 ms | `packages/react-dom-bindings/src/events/DOMPluginEventSystem.js:430-433` |
|
||||
| `create-dom-root` | 2 | 11.56 / 11.69 / 0.07 ms | `packages/react-dom/src/client/ReactDOMRoot.js:99-117` |
|
||||
| `reconcile-keyed-array` | 1 | 11.22 / 11.41 / 0.18 ms | `packages/react-reconciler/src/ReactChildFiber.js:1160-1182` |
|
||||
| `dispatch-state-update` | 2 | 11.09 / 11.23 / 0.08 ms | `packages/react-reconciler/src/ReactFiberHooks.js:3654-3679` |
|
||||
| `hydrate-dom-root` | 1 | 11.45 / 11.61 / 0.07 ms | `packages/react-dom/src/client/ReactDOMRootFB.js:134-169` |
|
||||
|
||||
## react · Semble · literal 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `state-hook-api` | 4 | 11.45 / 11.72 / 0.17 ms | `packages/react-server/src/ReactFizzHooks.js:406-428` |
|
||||
| `create-context` | 1 | 12.60 / 12.64 / 0.10 ms | `packages/react/src/ReactContext.js:1-18` |
|
||||
| `schedule-fiber-update` | 1 | 11.30 / 11.48 / 0.12 ms | `packages/react-reconciler/src/ReactFiberWorkLoop.js:972-1005` |
|
||||
| `commit-root` | 1 | 11.45 / 11.55 / 0.13 ms | `packages/react-reconciler/src/ReactFiberWorkLoop.js:3706-3735` |
|
||||
| `suspense-begin-work` | 3 | 12.51 / 27.95 / 6.14 ms | `packages/react-reconciler/src/ReactFiberBeginWork.js:4367-4389` |
|
||||
| `delegated-dom-events` | 1 | 10.61 / 10.85 / 0.18 ms | `packages/react-dom-bindings/src/events/DOMPluginEventSystem.js:430-433` |
|
||||
| `create-dom-root` | 3 | 11.03 / 11.12 / 0.20 ms | `packages/react-dom/src/client/ReactDOMRootFB.js:1-33` |
|
||||
| `reconcile-keyed-array` | 1 | 10.89 / 11.59 / 0.28 ms | `packages/react-reconciler/src/ReactChildFiber.js:1160-1182` |
|
||||
| `dispatch-state-update` | 1 | 10.33 / 10.54 / 0.15 ms | `packages/react-reconciler/src/ReactFiberHooks.js:3602-3630` |
|
||||
| `hydrate-dom-root` | 1 | 9.78 / 9.90 / 0.09 ms | `packages/react-dom/src/client/ReactDOMRootFB.js:134-169` |
|
||||
|
||||
## react · Semble · symbol 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `state-hook-api` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/react/src/ReactHooks.js:44-66` |
|
||||
| `create-context` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/react/src/ReactContext.js:1-18` |
|
||||
| `schedule-fiber-update` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/react-reconciler/src/ReactFiberWorkLoop.js:972-1005` |
|
||||
| `commit-root` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/react-reconciler/src/ReactFiberWorkLoop.js:3706-3735` |
|
||||
| `suspense-begin-work` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/react-reconciler/src/ReactFiberBeginWork.js:2369-2387` |
|
||||
| `delegated-dom-events` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/react-dom-bindings/src/events/DOMPluginEventSystem.js:430-433` |
|
||||
| `create-dom-root` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/react-dom/src/client/ReactDOMRoot.js:153-174` |
|
||||
| `reconcile-keyed-array` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/react-reconciler/src/ReactChildFiber.js:1160-1182` |
|
||||
| `dispatch-state-update` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/react-reconciler/src/ReactFiberHooks.js:3602-3630` |
|
||||
| `hydrate-dom-root` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/react-dom/src/client/ReactDOMRootFB.js:134-169` |
|
||||
|
||||
## react · CodeGraph · natural_language 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `state-hook-api` | 未命中 | 204.85 / 211.44 / 3.00 ms | `compiler/packages/babel-plugin-react-compiler/src/ReactiveScopes/PruneNonEscapingScopes.ts:202-417` |
|
||||
| `create-context` | 未命中 | 248.40 / 251.60 / 2.98 ms | `compiler/crates/react_compiler_hir/src/environment_config.rs:1-230` |
|
||||
| `schedule-fiber-update` | 未命中 | 177.86 / 180.33 / 1.97 ms | `compiler/packages/snap/src/reporter.ts:117-181` |
|
||||
| `commit-root` | 未命中 | 257.56 / 306.25 / 19.81 ms | `compiler/packages/babel-plugin-react-compiler/src/Utils/Result.ts:115-201` |
|
||||
| `suspense-begin-work` | 未命中 | 230.27 / 232.95 / 1.73 ms | `compiler/packages/babel-plugin-react-compiler/src/Flood/Types.ts:17-946` |
|
||||
| `delegated-dom-events` | 未命中 | 274.57 / 275.35 / 1.58 ms | `packages/internal-test-utils/ReactInternalTestUtils.js:349-379` |
|
||||
| `create-dom-root` | 未命中 | 211.90 / 214.24 / 1.10 ms | `compiler/packages/babel-plugin-react-compiler/src/Inference/InferMutationAliasingRanges.ts:599-619` |
|
||||
| `reconcile-keyed-array` | 未命中 | 226.78 / 242.64 / 7.47 ms | `compiler/packages/babel-plugin-react-compiler/src/Flood/Types.ts:17-946` |
|
||||
| `dispatch-state-update` | 未命中 | 183.96 / 203.61 / 9.03 ms | `compiler/packages/snap/src/reporter.ts:1-275` |
|
||||
| `hydrate-dom-root` | 未命中 | 268.40 / 272.53 / 2.65 ms | `packages/react-dom/src/client/ReactDOMRoot.js:149-357` |
|
||||
|
||||
## react · CodeGraph · literal 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `state-hook-api` | 未命中 | 202.23 / 224.65 / 9.12 ms | `compiler/packages/babel-plugin-react-compiler/src/Flood/Types.ts:148-865` |
|
||||
| `create-context` | 未命中 | 226.69 / 231.59 / 3.10 ms | `compiler/crates/react_compiler_hir/src/object_shape.rs:305-433` |
|
||||
| `schedule-fiber-update` | 1 | 233.96 / 235.11 / 1.69 ms | `packages/react-reconciler/src/ReactFiberWorkLoop.js:699-1132` |
|
||||
| `commit-root` | 未命中 | 200.31 / 202.11 / 1.09 ms | `packages/react-dom/src/client/ReactDOMRoot.js:149-357` |
|
||||
| `suspense-begin-work` | 未命中 | 246.94 / 250.10 / 2.24 ms | `packages/react-reconciler/src/ReactFiberRootScheduler.js:572-582` |
|
||||
| `delegated-dom-events` | 未命中 | 287.21 / 294.48 / 3.28 ms | `packages/react-dom/src/client/ReactDOMRoot.js:149-357` |
|
||||
| `create-dom-root` | 未命中 | 248.93 / 250.32 / 1.06 ms | `packages/react-dom/src/client/ReactDOMRoot.js:149-357` |
|
||||
| `reconcile-keyed-array` | 未命中 | 291.47 / 298.40 / 3.24 ms | `compiler/crates/react_compiler_hir/src/environment.rs:38-534` |
|
||||
| `dispatch-state-update` | 未命中 | 238.16 / 240.85 / 4.42 ms | `compiler/crates/react_compiler_inference/src/infer_mutation_aliasing_effects.rs:117-144` |
|
||||
| `hydrate-dom-root` | 未命中 | 211.90 / 216.60 / 2.41 ms | `packages/react-dom-bindings/src/shared/ReactFlightClientConfigDOM.js:1-139` |
|
||||
|
||||
## react · CodeGraph · symbol 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `state-hook-api` | 未命中 | 2.51 / 3.14 / 0.29 ms | `compiler/packages/babel-plugin-react-compiler/src/__tests__/fixtures/compiler/globals-dont-resolve-local-useState.js:4-7` |
|
||||
| `create-context` | 未命中 | 2.78 / 3.19 / 0.19 ms | `compiler/apps/playground/lib/createContext.ts:26-40` |
|
||||
| `schedule-fiber-update` | 1 | 2.14 / 2.57 / 0.23 ms | `packages/react-reconciler/src/ReactFiberWorkLoop.js:987-1113` |
|
||||
| `commit-root` | 未命中 | 2.23 / 2.84 / 0.29 ms | `packages/react-reconciler/src/ReactFiberCommitWork.js:220-238` |
|
||||
| `suspense-begin-work` | 未命中 | 12.34 / 12.72 / 0.27 ms | `—` |
|
||||
| `delegated-dom-events` | 1 | 2.34 / 2.78 / 0.27 ms | `packages/react-dom-bindings/src/events/DOMPluginEventSystem.js:432-459` |
|
||||
| `create-dom-root` | 未命中 | 2.68 / 2.94 / 0.26 ms | `packages/use-sync-external-store/src/__tests__/useSyncExternalStoreShared-test.js:90-110` |
|
||||
| `reconcile-keyed-array` | 未命中 | 14.48 / 15.25 / 0.65 ms | `—` |
|
||||
| `dispatch-state-update` | 未命中 | 12.23 / 13.05 / 0.40 ms | `—` |
|
||||
| `hydrate-dom-root` | 未命中 | 2.14 / 2.40 / 0.11 ms | `packages/react-dom/src/__tests__/ReactDOMFizzShellHydration-test.js:104-123` |
|
||||
|
||||
## vue · Semble · natural_language 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `reactive-proxy` | 1 | 1.85 / 1.92 / 0.08 ms | `packages/reactivity/src/reactive.ts:85-106` |
|
||||
| `create-ref` | 4 | 1.87 / 1.89 / 0.02 ms | `packages/reactivity/src/ref.ts:104-114` |
|
||||
| `create-renderer` | 1 | 1.90 / 1.92 / 0.03 ms | `packages/runtime-core/src/renderer.ts:303-323` |
|
||||
| `patch-keyed-children` | 3 | 1.62 / 1.76 / 0.06 ms | `packages/runtime-core/src/compat/instanceChildren.ts:1-17` |
|
||||
| `compile-template` | 未命中 | 1.91 / 1.95 / 0.04 ms | `packages/compiler-sfc/src/compileTemplate.ts:105-107` |
|
||||
| `parse-template` | 2 | 2.18 / 2.23 / 0.03 ms | `packages/compiler-sfc/src/parse.ts:95-110` |
|
||||
| `reactivity-watch` | 2 | 1.75 / 1.77 / 0.04 ms | `packages/reactivity/src/index.ts:81-97` |
|
||||
| `create-app-api` | 2 | 2.05 / 2.09 / 0.06 ms | `packages/runtime-core/src/component.ts:597-624` |
|
||||
| `define-component` | 1 | 1.92 / 2.01 / 0.04 ms | `packages/runtime-core/src/apiDefineComponent.ts:303-315` |
|
||||
| `hydrate-node` | 1 | 1.58 / 1.70 / 0.06 ms | `packages/runtime-core/src/hydration.ts:119-137` |
|
||||
|
||||
## vue · Semble · literal 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `reactive-proxy` | 1 | 1.75 / 1.87 / 0.05 ms | `packages/reactivity/src/reactive.ts:85-106` |
|
||||
| `create-ref` | 1 | 1.96 / 1.97 / 0.02 ms | `packages/reactivity/src/ref.ts:52-72` |
|
||||
| `create-renderer` | 2 | 1.98 / 2.00 / 0.03 ms | `packages/runtime-core/src/renderer.ts:325-346` |
|
||||
| `patch-keyed-children` | 3 | 2.14 / 2.25 / 0.07 ms | `packages/runtime-core/src/renderer.ts:1660-1675` |
|
||||
| `compile-template` | 2 | 1.82 / 1.97 / 0.07 ms | `packages/compiler-core/src/index.ts:1-30` |
|
||||
| `parse-template` | 2 | 1.77 / 1.85 / 0.04 ms | `packages/compiler-core/src/parser.ts:70-96` |
|
||||
| `reactivity-watch` | 4 | 1.58 / 1.62 / 0.03 ms | `packages/reactivity/src/watch.ts:49-78` |
|
||||
| `create-app-api` | 1 | 2.12 / 2.28 / 0.07 ms | `packages/runtime-core/src/apiCreateApp.ts:253-273` |
|
||||
| `define-component` | 1 | 1.91 / 2.00 / 0.05 ms | `packages/runtime-core/src/apiDefineComponent.ts:303-315` |
|
||||
| `hydrate-node` | 1 | 2.41 / 2.48 / 0.04 ms | `packages/runtime-core/src/hydration.ts:119-137` |
|
||||
|
||||
## vue · Semble · symbol 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `reactive-proxy` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/reactivity/src/reactive.ts:85-106` |
|
||||
| `create-ref` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/reactivity/src/ref.ts:52-72` |
|
||||
| `create-renderer` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/runtime-core/src/renderer.ts:303-323` |
|
||||
| `patch-keyed-children` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/runtime-core/src/renderer.ts:1777-1804` |
|
||||
| `compile-template` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/compiler-core/src/compile.ts:65-70` |
|
||||
| `parse-template` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/compiler-core/src/parser.ts:1009-1028` |
|
||||
| `reactivity-watch` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/reactivity/src/watch.ts:103-124` |
|
||||
| `create-app-api` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/runtime-core/src/apiCreateApp.ts:253-273` |
|
||||
| `define-component` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/runtime-core/src/apiDefineComponent.ts:303-315` |
|
||||
| `hydrate-node` | 1 | 0.01 / 0.01 / 0.00 ms | `packages/runtime-core/src/hydration.ts:119-137` |
|
||||
|
||||
## vue · CodeGraph · natural_language 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `reactive-proxy` | 1 | 48.76 / 51.18 / 1.03 ms | `packages/reactivity/src/reactive.ts:15-438` |
|
||||
| `create-ref` | 1 | 46.94 / 48.57 / 0.81 ms | `packages/reactivity/src/ref.ts:61-579` |
|
||||
| `create-renderer` | 2 | 72.82 / 72.94 / 0.32 ms | `packages/runtime-core/src/compat/global.ts:93-353` |
|
||||
| `patch-keyed-children` | 未命中 | 50.77 / 51.52 / 0.92 ms | `packages/runtime-core/src/renderer.ts:153-2640` |
|
||||
| `compile-template` | 未命中 | 56.67 / 58.12 / 0.74 ms | `packages/runtime-core/src/compat/global.ts:146-156` |
|
||||
| `parse-template` | 未命中 | 49.11 / 49.73 / 0.43 ms | `packages/compiler-sfc/src/parse.ts:22-354` |
|
||||
| `reactivity-watch` | 未命中 | 47.20 / 47.46 / 0.48 ms | `packages/reactivity/src/watch.ts:34-93` |
|
||||
| `create-app-api` | 未命中 | 62.82 / 63.58 / 0.45 ms | `packages/runtime-core/src/compat/global.ts:85-156` |
|
||||
| `define-component` | 2 | 61.96 / 62.26 / 0.18 ms | `packages/runtime-core/src/compat/global.ts:64-333` |
|
||||
| `hydrate-node` | 未命中 | 61.93 / 63.06 / 0.67 ms | `packages/runtime-core/src/compat/global.ts:146-156` |
|
||||
|
||||
## vue · CodeGraph · literal 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `reactive-proxy` | 1 | 43.32 / 44.13 / 0.45 ms | `packages/reactivity/src/reactive.ts:1-446` |
|
||||
| `create-ref` | 1 | 38.00 / 38.77 / 0.38 ms | `packages/reactivity/src/ref.ts:1-579` |
|
||||
| `create-renderer` | 未命中 | 50.20 / 50.30 / 0.26 ms | `packages/runtime-core/src/renderer.ts:93-304` |
|
||||
| `patch-keyed-children` | 未命中 | 41.23 / 41.83 / 0.32 ms | `packages-private/dts-test/ref.test-d.ts:345-351` |
|
||||
| `compile-template` | 1 | 39.40 / 40.28 / 0.38 ms | `packages/compiler-core/src/compile.ts:1-122` |
|
||||
| `parse-template` | 未命中 | 40.95 / 41.30 / 0.25 ms | `packages/compiler-core/src/parser.ts:1-288` |
|
||||
| `reactivity-watch` | 未命中 | 37.55 / 38.09 / 0.54 ms | `packages/runtime-core/src/apiWatch.ts:1-92` |
|
||||
| `create-app-api` | 1 | 40.47 / 40.94 / 0.46 ms | `packages/runtime-core/src/apiCreateApp.ts:30-494` |
|
||||
| `define-component` | 未命中 | 43.52 / 45.06 / 0.66 ms | `packages/shared/src/general.ts:1-219` |
|
||||
| `hydrate-node` | 未命中 | 45.72 / 47.40 / 0.78 ms | `packages/runtime-core/src/vnode.ts:71-256` |
|
||||
|
||||
## vue · CodeGraph · symbol 明细
|
||||
|
||||
| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |
|
||||
| --- | ---: | ---: | --- |
|
||||
| `reactive-proxy` | 1 | 0.88 / 0.91 / 0.02 ms | `packages/reactivity/src/reactive.ts:87-99` |
|
||||
| `create-ref` | 1 | 0.96 / 1.12 / 0.07 ms | `packages/reactivity/src/ref.ts:64-66` |
|
||||
| `create-renderer` | 1 | 0.30 / 0.30 / 0.01 ms | `packages/runtime-core/src/renderer.ts:318-323` |
|
||||
| `patch-keyed-children` | 未命中 | 1.96 / 4.02 / 0.82 ms | `—` |
|
||||
| `compile-template` | 1 | 0.33 / 0.35 / 0.01 ms | `packages/compiler-core/src/compile.ts:67-122` |
|
||||
| `parse-template` | 1 | 0.40 / 0.41 / 0.00 ms | `packages/compiler-core/src/parser.ts:1028-1079` |
|
||||
| `reactivity-watch` | 1 | 0.58 / 0.60 / 0.01 ms | `packages/reactivity/src/watch.ts:120-331` |
|
||||
| `create-app-api` | 1 | 0.29 / 0.30 / 0.00 ms | `packages/runtime-core/src/apiCreateApp.ts:253-487` |
|
||||
| `define-component` | 1 | 0.62 / 0.64 / 0.01 ms | `packages/runtime-core/src/apiDefineComponent.ts:305-315` |
|
||||
| `hydrate-node` | 未命中 | 0.27 / 0.37 / 0.05 ms | `packages/runtime-core/src/components/Suspense.ts:806-856` |
|
||||
|
||||
## 公平性说明
|
||||
|
||||
自然语言和多词字面轨道调用 CodeGraph 官方推荐的 codegraph_explore，包含图扩展和最终源码读取；符号轨道调用 searchNodes。Semble 三条轨道都调用同一个混合搜索接口。所有查询共享同一组人工标注实现位置，未针对系统选择不同真值。两者输出粒度不同，因此本报告适合比较达到同一代码位置的效果和实际工具延迟，不代表各内部算法的微基准。CodeGraph 的独立 callers、callees 和 impact 能力不属于本次范围。
|
||||
@@ -0,0 +1,115 @@
|
||||
//! Version-pinned benchmark suites and hand-labelled implementation locations.
|
||||
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub struct Suite {
|
||||
pub name: String,
|
||||
pub repository: String,
|
||||
pub commit: String,
|
||||
pub queries: Vec<QueryCase>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub struct QueryCase {
|
||||
pub id: String,
|
||||
pub query: String,
|
||||
pub literal_query: String,
|
||||
pub symbol_query: String,
|
||||
pub expected: Vec<ExpectedLocation>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
pub struct ExpectedLocation {
|
||||
pub path: String,
|
||||
pub line: usize,
|
||||
}
|
||||
|
||||
pub fn load_suites(directory: &Path) -> Result<Vec<Suite>, String> {
|
||||
let mut paths = fs::read_dir(directory)
|
||||
.map_err(|error| format!("read cases directory {}: {error}", directory.display()))?
|
||||
.filter_map(|entry| entry.ok().map(|entry| entry.path()))
|
||||
.filter(|path| {
|
||||
path.extension()
|
||||
.is_some_and(|extension| extension == "json")
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
paths.sort();
|
||||
let mut suites = Vec::with_capacity(paths.len());
|
||||
for path in paths {
|
||||
let bytes = fs::read(&path)
|
||||
.map_err(|error| format!("read benchmark suite {}: {error}", path.display()))?;
|
||||
let suite = serde_json::from_slice::<Suite>(&bytes)
|
||||
.map_err(|error| format!("parse benchmark suite {}: {error}", path.display()))?;
|
||||
validate_suite(&suite)?;
|
||||
suites.push(suite);
|
||||
}
|
||||
if suites.is_empty() {
|
||||
return Err(format!(
|
||||
"no benchmark suites found in {}",
|
||||
directory.display()
|
||||
));
|
||||
}
|
||||
Ok(suites)
|
||||
}
|
||||
|
||||
fn validate_suite(suite: &Suite) -> Result<(), String> {
|
||||
if suite.name.trim().is_empty()
|
||||
|| suite.repository.trim().is_empty()
|
||||
|| suite.commit.trim().is_empty()
|
||||
|| suite.queries.is_empty()
|
||||
{
|
||||
return Err(format!("suite {:?} is missing required fields", suite.name));
|
||||
}
|
||||
for query in &suite.queries {
|
||||
if query.id.trim().is_empty()
|
||||
|| query.query.trim().is_empty()
|
||||
|| query.literal_query.trim().is_empty()
|
||||
|| query.symbol_query.trim().is_empty()
|
||||
|| query.expected.is_empty()
|
||||
{
|
||||
return Err(format!(
|
||||
"suite {} contains an incomplete query case",
|
||||
suite.name
|
||||
));
|
||||
}
|
||||
if query
|
||||
.expected
|
||||
.iter()
|
||||
.any(|location| location.path.trim().is_empty() || location.line == 0)
|
||||
{
|
||||
return Err(format!(
|
||||
"suite {} query {} contains an invalid expected location",
|
||||
suite.name, query.id
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rejects_locations_without_a_line_number() {
|
||||
let suite = Suite {
|
||||
name: "fixture".into(),
|
||||
repository: "https://example.test/repo".into(),
|
||||
commit: "abc".into(),
|
||||
queries: vec![QueryCase {
|
||||
id: "q1".into(),
|
||||
query: "find it".into(),
|
||||
literal_query: "find_it result value".into(),
|
||||
symbol_query: "findIt".into(),
|
||||
expected: vec![ExpectedLocation {
|
||||
path: "src/lib.rs".into(),
|
||||
line: 0,
|
||||
}],
|
||||
}],
|
||||
};
|
||||
assert!(validate_suite(&suite).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
//! Adapter lifecycle for benchmarking an installed CodeGraph distribution.
|
||||
|
||||
use std::{
|
||||
fs,
|
||||
io::Write,
|
||||
path::{Path, PathBuf},
|
||||
process::{Command, Stdio},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::case::Suite;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct CodeGraphRun {
|
||||
pub version: String,
|
||||
pub cold_index_ms: f64,
|
||||
pub persisted_index_bytes: u64,
|
||||
pub adapter: AdapterOutput,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AdapterOutput {
|
||||
pub load_ms: f64,
|
||||
pub stats: AdapterStats,
|
||||
pub tracks: Vec<AdapterTrack>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AdapterStats {
|
||||
pub node_count: usize,
|
||||
pub edge_count: usize,
|
||||
pub file_count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct AdapterTrack {
|
||||
pub name: String,
|
||||
pub queries: Vec<AdapterQuery>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AdapterQuery {
|
||||
pub id: String,
|
||||
pub query: String,
|
||||
pub durations_ms: Vec<f64>,
|
||||
pub results: Vec<AdapterResult>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AdapterResult {
|
||||
pub path: String,
|
||||
pub start_line: usize,
|
||||
pub end_line: usize,
|
||||
pub score: f32,
|
||||
pub lines: Option<Vec<usize>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct AdapterInput<'a> {
|
||||
tracks: [AdapterInputTrack<'a>; 3],
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct AdapterInputTrack<'a> {
|
||||
name: &'static str,
|
||||
queries: Vec<AdapterInputQuery<'a>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct AdapterInputQuery<'a> {
|
||||
id: &'a str,
|
||||
query: &'a str,
|
||||
}
|
||||
|
||||
pub fn benchmark(
|
||||
executable: &str,
|
||||
adapter_path: &Path,
|
||||
source: &Path,
|
||||
suite: &Suite,
|
||||
repetitions: usize,
|
||||
top_k: usize,
|
||||
) -> Result<CodeGraphRun, String> {
|
||||
let executable = locate_executable(executable)?;
|
||||
let distribution = distribution_root(&executable)?;
|
||||
let node = distribution.join("node");
|
||||
let library = distribution.join("lib");
|
||||
if !node.is_file() || !library.is_dir() {
|
||||
return Err(format!(
|
||||
"unsupported CodeGraph installation layout below {}",
|
||||
distribution.display()
|
||||
));
|
||||
}
|
||||
if source.join(".codegraph").exists() {
|
||||
command_status(
|
||||
Command::new(&executable)
|
||||
.args(["uninit", "--force"])
|
||||
.arg(source),
|
||||
"reset CodeGraph index",
|
||||
)?;
|
||||
}
|
||||
let started = Instant::now();
|
||||
command_status(
|
||||
Command::new(&executable).arg("init").arg(source),
|
||||
"build CodeGraph index",
|
||||
)?;
|
||||
let cold_index_ms = started.elapsed().as_secs_f64() * 1_000.0;
|
||||
let persisted_index_bytes = directory_size(&source.join(".codegraph"))?;
|
||||
let version = command_output(
|
||||
Command::new(&executable).arg("version"),
|
||||
"read CodeGraph version",
|
||||
)?;
|
||||
let input = AdapterInput {
|
||||
tracks: [
|
||||
AdapterInputTrack {
|
||||
name: "natural_language",
|
||||
queries: suite
|
||||
.queries
|
||||
.iter()
|
||||
.map(|query| AdapterInputQuery {
|
||||
id: &query.id,
|
||||
query: &query.query,
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
AdapterInputTrack {
|
||||
name: "literal",
|
||||
queries: suite
|
||||
.queries
|
||||
.iter()
|
||||
.map(|query| AdapterInputQuery {
|
||||
id: &query.id,
|
||||
query: &query.literal_query,
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
AdapterInputTrack {
|
||||
name: "symbol",
|
||||
queries: suite
|
||||
.queries
|
||||
.iter()
|
||||
.map(|query| AdapterInputQuery {
|
||||
id: &query.id,
|
||||
query: &query.symbol_query,
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
],
|
||||
};
|
||||
let encoded = serde_json::to_vec(&input)
|
||||
.map_err(|error| format!("serialize CodeGraph benchmark input: {error}"))?;
|
||||
let mut child = Command::new(node)
|
||||
.arg("--liftoff-only")
|
||||
.arg(adapter_path)
|
||||
.arg(library)
|
||||
.arg(source)
|
||||
.arg(repetitions.to_string())
|
||||
.arg(top_k.to_string())
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.map_err(|error| format!("start CodeGraph benchmark adapter: {error}"))?;
|
||||
child
|
||||
.stdin
|
||||
.take()
|
||||
.ok_or_else(|| "CodeGraph adapter stdin is unavailable".to_owned())?
|
||||
.write_all(&encoded)
|
||||
.map_err(|error| format!("write CodeGraph benchmark input: {error}"))?;
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.map_err(|error| format!("wait for CodeGraph benchmark adapter: {error}"))?;
|
||||
if !output.status.success() {
|
||||
return Err(format!(
|
||||
"CodeGraph benchmark adapter failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr).trim()
|
||||
));
|
||||
}
|
||||
let adapter = serde_json::from_slice(&output.stdout)
|
||||
.map_err(|error| format!("parse CodeGraph benchmark output: {error}"))?;
|
||||
Ok(CodeGraphRun {
|
||||
version,
|
||||
cold_index_ms,
|
||||
persisted_index_bytes,
|
||||
adapter,
|
||||
})
|
||||
}
|
||||
|
||||
fn locate_executable(name: &str) -> Result<PathBuf, String> {
|
||||
let output = command_output(
|
||||
Command::new("which").arg(name),
|
||||
"locate CodeGraph executable",
|
||||
)?;
|
||||
fs::canonicalize(output)
|
||||
.map_err(|error| format!("resolve CodeGraph executable symlink: {error}"))
|
||||
}
|
||||
|
||||
fn distribution_root(executable: &Path) -> Result<PathBuf, String> {
|
||||
executable
|
||||
.parent()
|
||||
.and_then(Path::parent)
|
||||
.map(Path::to_path_buf)
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"cannot locate distribution root for {}",
|
||||
executable.display()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn command_status(command: &mut Command, action: &str) -> Result<(), String> {
|
||||
let status = command
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.map_err(|error| format!("{action}: {error}"))?;
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("{action} failed with {status}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn command_output(command: &mut Command, action: &str) -> Result<String, String> {
|
||||
let output = command
|
||||
.output()
|
||||
.map_err(|error| format!("{action}: {error}"))?;
|
||||
if !output.status.success() {
|
||||
return Err(format!("{action} failed with {}", output.status));
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned())
|
||||
}
|
||||
|
||||
fn directory_size(path: &Path) -> Result<u64, String> {
|
||||
let mut total = 0_u64;
|
||||
let mut pending = vec![path.to_path_buf()];
|
||||
while let Some(directory) = pending.pop() {
|
||||
for entry in fs::read_dir(&directory)
|
||||
.map_err(|error| format!("read {}: {error}", directory.display()))?
|
||||
{
|
||||
let entry = entry.map_err(|error| format!("read directory entry: {error}"))?;
|
||||
let metadata = entry
|
||||
.metadata()
|
||||
.map_err(|error| format!("inspect {}: {error}", entry.path().display()))?;
|
||||
if metadata.is_dir() {
|
||||
pending.push(entry.path());
|
||||
} else if metadata.is_file() {
|
||||
total = total.saturating_add(metadata.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
//! Explicit quality floors for detecting retrieval regressions in CI.
|
||||
|
||||
use std::{collections::BTreeMap, fs, path::Path};
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::report::BenchmarkReport;
|
||||
|
||||
const REQUIRED_TRACKS: [&str; 3] = ["natural_language", "literal", "symbol"];
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct QualityGates {
|
||||
schema_version: u32,
|
||||
comparisons: ComparisonGates,
|
||||
tracks: BTreeMap<String, QualityGate>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ComparisonGates {
|
||||
cached_load: RatioGate,
|
||||
symbol_latency: LatencyRatioGate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RatioGate {
|
||||
max_ratio: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LatencyRatioGate {
|
||||
max_p50_ratio: f64,
|
||||
max_p95_ratio: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct QualityGate {
|
||||
recall_at_1: f64,
|
||||
recall_at_5: f64,
|
||||
recall_at_10: f64,
|
||||
mrr_at_10: f64,
|
||||
ndcg_at_10: f64,
|
||||
}
|
||||
|
||||
pub fn check(path: &Path, report: &BenchmarkReport) -> Result<(), String> {
|
||||
let bytes = fs::read(path)
|
||||
.map_err(|error| format!("read quality gates {}: {error}", path.display()))?;
|
||||
let gates = serde_json::from_slice::<QualityGates>(&bytes)
|
||||
.map_err(|error| format!("parse quality gates {}: {error}", path.display()))?;
|
||||
if gates.schema_version != 1 {
|
||||
return Err(format!(
|
||||
"unsupported quality gate schema {} in {}",
|
||||
gates.schema_version,
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
validate(&gates)?;
|
||||
let mut failures = Vec::new();
|
||||
for (track, gate) in &gates.tracks {
|
||||
let Some(result) = report
|
||||
.overall
|
||||
.iter()
|
||||
.find(|result| result.system == "Semble" && result.track == *track)
|
||||
else {
|
||||
failures.push(format!("missing Semble track {track}"));
|
||||
continue;
|
||||
};
|
||||
let metrics = &result.effect;
|
||||
compare(
|
||||
&mut failures,
|
||||
track,
|
||||
"Recall@1",
|
||||
metrics.recall_at_1,
|
||||
gate.recall_at_1,
|
||||
);
|
||||
compare(
|
||||
&mut failures,
|
||||
track,
|
||||
"Recall@5",
|
||||
metrics.recall_at_5,
|
||||
gate.recall_at_5,
|
||||
);
|
||||
compare(
|
||||
&mut failures,
|
||||
track,
|
||||
"Recall@10",
|
||||
metrics.recall_at_10,
|
||||
gate.recall_at_10,
|
||||
);
|
||||
compare(
|
||||
&mut failures,
|
||||
track,
|
||||
"MRR@10",
|
||||
metrics.mrr_at_10,
|
||||
gate.mrr_at_10,
|
||||
);
|
||||
compare(
|
||||
&mut failures,
|
||||
track,
|
||||
"nDCG@10",
|
||||
metrics.ndcg_at_10,
|
||||
gate.ndcg_at_10,
|
||||
);
|
||||
}
|
||||
for suite in &report.suites {
|
||||
let Some(semble) = suite
|
||||
.systems
|
||||
.iter()
|
||||
.find(|system| system.system == "Semble")
|
||||
else {
|
||||
failures.push(format!("{} is missing Semble results", suite.name));
|
||||
continue;
|
||||
};
|
||||
let Some(codegraph) = suite
|
||||
.systems
|
||||
.iter()
|
||||
.find(|system| system.system == "CodeGraph")
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
compare_ratio(
|
||||
&mut failures,
|
||||
&suite.name,
|
||||
"cached load",
|
||||
semble.index.cached_load_ms,
|
||||
codegraph.index.cached_load_ms,
|
||||
gates.comparisons.cached_load.max_ratio,
|
||||
);
|
||||
let Some(semble_symbol) = semble.tracks.iter().find(|track| track.name == "symbol") else {
|
||||
failures.push(format!("{} is missing Semble symbol results", suite.name));
|
||||
continue;
|
||||
};
|
||||
let Some(codegraph_symbol) = codegraph.tracks.iter().find(|track| track.name == "symbol")
|
||||
else {
|
||||
failures.push(format!(
|
||||
"{} is missing CodeGraph symbol results",
|
||||
suite.name
|
||||
));
|
||||
continue;
|
||||
};
|
||||
compare_latency(
|
||||
&mut failures,
|
||||
&suite.name,
|
||||
"P50",
|
||||
semble_symbol.query_latency.p50_ms,
|
||||
codegraph_symbol.query_latency.p50_ms,
|
||||
gates.comparisons.symbol_latency.max_p50_ratio,
|
||||
);
|
||||
compare_latency(
|
||||
&mut failures,
|
||||
&suite.name,
|
||||
"P95",
|
||||
semble_symbol.query_latency.p95_ms,
|
||||
codegraph_symbol.query_latency.p95_ms,
|
||||
gates.comparisons.symbol_latency.max_p95_ratio,
|
||||
);
|
||||
}
|
||||
if failures.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"quality gates failed:\n- {}",
|
||||
failures.join("\n- ")
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn validate(gates: &QualityGates) -> Result<(), String> {
|
||||
for track in REQUIRED_TRACKS {
|
||||
if !gates.tracks.contains_key(track) {
|
||||
return Err(format!("quality gates are missing required track {track}"));
|
||||
}
|
||||
}
|
||||
for (track, gate) in &gates.tracks {
|
||||
for (metric, value) in [
|
||||
("Recall@1", gate.recall_at_1),
|
||||
("Recall@5", gate.recall_at_5),
|
||||
("Recall@10", gate.recall_at_10),
|
||||
("MRR@10", gate.mrr_at_10),
|
||||
("nDCG@10", gate.ndcg_at_10),
|
||||
] {
|
||||
if !value.is_finite() || !(0.0..=1.0).contains(&value) {
|
||||
return Err(format!(
|
||||
"quality gate {track} {metric} must be between 0 and 1"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (metric, value) in [
|
||||
("cached load ratio", gates.comparisons.cached_load.max_ratio),
|
||||
(
|
||||
"symbol latency P50 ratio",
|
||||
gates.comparisons.symbol_latency.max_p50_ratio,
|
||||
),
|
||||
(
|
||||
"symbol latency P95 ratio",
|
||||
gates.comparisons.symbol_latency.max_p95_ratio,
|
||||
),
|
||||
] {
|
||||
if !value.is_finite() || value <= 0.0 || value > 1.0 {
|
||||
return Err(format!(
|
||||
"quality gate {metric} must be greater than 0 and at most 1"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compare(failures: &mut Vec<String>, track: &str, metric: &str, actual: f64, minimum: f64) {
|
||||
if actual + f64::EPSILON < minimum {
|
||||
failures.push(format!(
|
||||
"{track} {metric} was {actual:.3}, minimum is {minimum:.3}"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
fn compare_latency(
|
||||
failures: &mut Vec<String>,
|
||||
suite: &str,
|
||||
percentile: &str,
|
||||
semble_ms: f64,
|
||||
codegraph_ms: f64,
|
||||
max_ratio: f64,
|
||||
) {
|
||||
compare_ratio(
|
||||
failures,
|
||||
suite,
|
||||
&format!("symbol {percentile}"),
|
||||
semble_ms,
|
||||
codegraph_ms,
|
||||
max_ratio,
|
||||
);
|
||||
}
|
||||
|
||||
fn compare_ratio(
|
||||
failures: &mut Vec<String>,
|
||||
suite: &str,
|
||||
metric: &str,
|
||||
semble_ms: f64,
|
||||
codegraph_ms: f64,
|
||||
max_ratio: f64,
|
||||
) {
|
||||
let limit = codegraph_ms * max_ratio;
|
||||
if semble_ms > limit {
|
||||
failures.push(format!(
|
||||
"{suite} {metric} was {semble_ms:.3} ms; limit is {limit:.3} ms ({max_ratio:.0}% of CodeGraph {codegraph_ms:.3} ms)"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn gate(value: f64) -> QualityGate {
|
||||
QualityGate {
|
||||
recall_at_1: value,
|
||||
recall_at_5: value,
|
||||
recall_at_10: value,
|
||||
mrr_at_10: value,
|
||||
ndcg_at_10: value,
|
||||
}
|
||||
}
|
||||
|
||||
fn comparisons() -> ComparisonGates {
|
||||
ComparisonGates {
|
||||
cached_load: RatioGate { max_ratio: 0.9 },
|
||||
symbol_latency: LatencyRatioGate {
|
||||
max_p50_ratio: 0.9,
|
||||
max_p95_ratio: 0.9,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_incomplete_and_out_of_range_configuration() {
|
||||
let mut tracks = BTreeMap::new();
|
||||
tracks.insert("natural_language".into(), gate(0.5));
|
||||
let gates = QualityGates {
|
||||
schema_version: 1,
|
||||
comparisons: comparisons(),
|
||||
tracks,
|
||||
};
|
||||
assert!(validate(&gates).unwrap_err().contains("literal"));
|
||||
|
||||
let mut tracks = REQUIRED_TRACKS
|
||||
.into_iter()
|
||||
.map(|track| (track.to_owned(), gate(0.5)))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
tracks.insert("literal".into(), gate(1.1));
|
||||
let gates = QualityGates {
|
||||
schema_version: 1,
|
||||
comparisons: comparisons(),
|
||||
tracks,
|
||||
};
|
||||
assert!(validate(&gates).unwrap_err().contains("between 0 and 1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comparison_only_records_values_below_the_floor() {
|
||||
let mut failures = Vec::new();
|
||||
compare(&mut failures, "literal", "Recall@1", 0.6, 0.6);
|
||||
assert!(failures.is_empty());
|
||||
compare(&mut failures, "literal", "Recall@1", 0.59, 0.6);
|
||||
assert_eq!(failures.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn latency_comparison_enforces_the_configured_margin() {
|
||||
let mut failures = Vec::new();
|
||||
compare_latency(&mut failures, "react", "P50", 0.8, 1.0, 0.9);
|
||||
assert!(failures.is_empty());
|
||||
compare_latency(&mut failures, "react", "P95", 0.95, 1.0, 0.9);
|
||||
assert_eq!(failures.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_load_comparison_enforces_the_configured_margin() {
|
||||
let mut failures = Vec::new();
|
||||
compare_ratio(&mut failures, "react", "cached load", 80.0, 100.0, 0.9);
|
||||
assert!(failures.is_empty());
|
||||
compare_ratio(&mut failures, "react", "cached load", 95.0, 100.0, 0.9);
|
||||
assert_eq!(failures.len(), 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,710 @@
|
||||
//! Reproducible Semble and CodeGraph comparison on React and Vue.
|
||||
|
||||
mod case;
|
||||
mod codegraph;
|
||||
mod gates;
|
||||
mod metrics;
|
||||
mod report;
|
||||
mod repository;
|
||||
|
||||
use std::{
|
||||
collections::{BTreeSet, HashMap},
|
||||
env, fs,
|
||||
path::{Path, PathBuf},
|
||||
process::Command,
|
||||
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use case::{ExpectedLocation, QueryCase, Suite};
|
||||
use codegraph::{AdapterResult, AdapterTrack};
|
||||
use report::{
|
||||
BenchmarkReport, Configuration, Environment, IndexMetrics, OverallReport, QueryReport,
|
||||
ResultSummary, SuiteReport, SystemReport, TrackReport,
|
||||
};
|
||||
use semble_core::{
|
||||
embedding::ModelAssets, ContentType, SearchEngine, SearchRequest, SearchResult, SembleConfig,
|
||||
};
|
||||
|
||||
const TOP_K: usize = 10;
|
||||
const TRACKS: [&str; 3] = ["natural_language", "literal", "symbol"];
|
||||
|
||||
fn main() {
|
||||
if let Err(error) = run() {
|
||||
eprintln!("semble-benchmark: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> Result<(), String> {
|
||||
let args = Args::parse()?;
|
||||
if args.help {
|
||||
print_help();
|
||||
return Ok(());
|
||||
}
|
||||
let suites = case::load_suites(&args.cases_dir)?;
|
||||
let sources = repository::resolve_sources(&suites, &args.sources_dir, &args.sources)?;
|
||||
let codegraph_sources = if args.semble_only {
|
||||
HashMap::new()
|
||||
} else {
|
||||
repository::prepare_competitor_sources(&suites, &sources, &args.codegraph_sources_dir)?
|
||||
};
|
||||
fs::create_dir_all(&args.cache_dir).map_err(|error| {
|
||||
format!(
|
||||
"create cache directory {}: {error}",
|
||||
args.cache_dir.display()
|
||||
)
|
||||
})?;
|
||||
if args.reset_indexes {
|
||||
remove_indexes(&args.cache_dir)?;
|
||||
}
|
||||
ModelAssets::ensure(&args.cache_dir).map_err(|error| error.to_string())?;
|
||||
|
||||
let mut suite_reports = Vec::with_capacity(suites.len());
|
||||
for suite in &suites {
|
||||
println!("benchmarking {} at {}", suite.name, suite.commit);
|
||||
let source = sources
|
||||
.get(&suite.name)
|
||||
.ok_or_else(|| format!("missing source path for {}", suite.name))?;
|
||||
let codegraph_source = codegraph_sources.get(&suite.name).map(PathBuf::as_path);
|
||||
suite_reports.push(benchmark_suite(suite, source, codegraph_source, &args)?);
|
||||
}
|
||||
|
||||
let report = BenchmarkReport {
|
||||
schema_version: 4,
|
||||
generated_at_unix_seconds: SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs(),
|
||||
environment: Environment {
|
||||
os: env::consts::OS.into(),
|
||||
architecture: env::consts::ARCH.into(),
|
||||
rustc: rustc_version(),
|
||||
},
|
||||
configuration: Configuration {
|
||||
top_k: TOP_K,
|
||||
repetitions: args.repetitions,
|
||||
tracks: TRACKS.iter().map(|track| (*track).to_owned()).collect(),
|
||||
relevance:
|
||||
"result path matches and result line range covers a labelled implementation line"
|
||||
.into(),
|
||||
},
|
||||
overall: overall(&suite_reports),
|
||||
suites: suite_reports,
|
||||
};
|
||||
write_report(&args.json, &args.markdown, &report)?;
|
||||
println!("wrote {}", args.json.display());
|
||||
println!("wrote {}", args.markdown.display());
|
||||
if args.check {
|
||||
gates::check(&args.quality_gates, &report)?;
|
||||
println!("quality gates passed");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn benchmark_suite(
|
||||
suite: &Suite,
|
||||
source: &Path,
|
||||
codegraph_source: Option<&Path>,
|
||||
args: &Args,
|
||||
) -> Result<SuiteReport, String> {
|
||||
validate_ground_truth(suite, source)?;
|
||||
let semble = benchmark_semble(suite, source, &args.cache_dir, args.repetitions)?;
|
||||
let mut systems = vec![semble];
|
||||
if let Some(codegraph_source) = codegraph_source {
|
||||
let codegraph = codegraph::benchmark(
|
||||
&args.codegraph,
|
||||
&args.codegraph_adapter,
|
||||
codegraph_source,
|
||||
suite,
|
||||
args.repetitions,
|
||||
TOP_K,
|
||||
)?;
|
||||
let cold_seconds = (codegraph.cold_index_ms / 1_000.0).max(f64::EPSILON);
|
||||
systems.push(SystemReport {
|
||||
system: "CodeGraph".into(),
|
||||
version: codegraph.version,
|
||||
index: IndexMetrics {
|
||||
cold_ready_ms: codegraph.cold_index_ms,
|
||||
cold_index_ms: codegraph.cold_index_ms,
|
||||
cached_load_ms: codegraph.adapter.load_ms,
|
||||
in_memory_prepare_ms: None,
|
||||
indexed_files: codegraph.adapter.stats.file_count,
|
||||
indexed_units: codegraph.adapter.stats.node_count,
|
||||
indexed_unit: "nodes".into(),
|
||||
indexed_edges: Some(codegraph.adapter.stats.edge_count),
|
||||
persisted_index_bytes: codegraph.persisted_index_bytes,
|
||||
source_bytes: None,
|
||||
units_per_second: codegraph.adapter.stats.node_count as f64 / cold_seconds,
|
||||
source_mib_per_second: None,
|
||||
},
|
||||
refresh_symbol_latency: None,
|
||||
tracks: codegraph
|
||||
.adapter
|
||||
.tracks
|
||||
.iter()
|
||||
.map(|track| codegraph_track(track, suite))
|
||||
.collect::<Result<Vec<_>, _>>()?,
|
||||
});
|
||||
}
|
||||
Ok(SuiteReport {
|
||||
name: suite.name.clone(),
|
||||
repository: suite.repository.clone(),
|
||||
commit: suite.commit.clone(),
|
||||
systems,
|
||||
})
|
||||
}
|
||||
|
||||
fn benchmark_semble(
|
||||
suite: &Suite,
|
||||
source: &Path,
|
||||
cache_dir: &Path,
|
||||
repetitions: usize,
|
||||
) -> Result<SystemReport, String> {
|
||||
let index_bytes_before = directory_size(&cache_dir.join("indexes"))?;
|
||||
let model_started = Instant::now();
|
||||
let engine = SearchEngine::load_default(SembleConfig::new(cache_dir))
|
||||
.map_err(|error| error.to_string())?;
|
||||
let model_load_ms = elapsed_ms(model_started.elapsed());
|
||||
let cold_started = Instant::now();
|
||||
let stats = engine
|
||||
.prepare(source, &[ContentType::Code])
|
||||
.map_err(|error| error.to_string())?;
|
||||
let cold_duration = cold_started.elapsed();
|
||||
let memory_started = Instant::now();
|
||||
let memory_stats = engine
|
||||
.prepare(source, &[ContentType::Code])
|
||||
.map_err(|error| error.to_string())?;
|
||||
let in_memory_prepare_ms = elapsed_ms(memory_started.elapsed());
|
||||
if memory_stats != stats {
|
||||
return Err(format!("{} in-memory index stats changed", suite.name));
|
||||
}
|
||||
drop(engine);
|
||||
|
||||
let reload_started = Instant::now();
|
||||
let engine = SearchEngine::load_default(SembleConfig::new(cache_dir))
|
||||
.map_err(|error| error.to_string())?;
|
||||
let model_reload_ms = elapsed_ms(reload_started.elapsed());
|
||||
let disk_started = Instant::now();
|
||||
let disk_stats = engine
|
||||
.prepare(source, &[ContentType::Code])
|
||||
.map_err(|error| error.to_string())?;
|
||||
let disk_index_load_ms = elapsed_ms(disk_started.elapsed());
|
||||
if disk_stats != stats {
|
||||
return Err(format!("{} persisted index stats changed", suite.name));
|
||||
}
|
||||
let tracks = vec![
|
||||
semble_track(
|
||||
&engine,
|
||||
source,
|
||||
suite,
|
||||
"natural_language",
|
||||
|query| &query.query,
|
||||
repetitions,
|
||||
)?,
|
||||
semble_track(
|
||||
&engine,
|
||||
source,
|
||||
suite,
|
||||
"literal",
|
||||
|query| &query.literal_query,
|
||||
repetitions,
|
||||
)?,
|
||||
semble_track(
|
||||
&engine,
|
||||
source,
|
||||
suite,
|
||||
"symbol",
|
||||
|query| &query.symbol_query,
|
||||
repetitions,
|
||||
)?,
|
||||
];
|
||||
let refresh_symbol_latency = benchmark_refresh_symbol(&engine, source, suite, repetitions)?;
|
||||
let index_bytes_after = directory_size(&cache_dir.join("indexes"))?;
|
||||
let cold_seconds = cold_duration.as_secs_f64().max(f64::EPSILON);
|
||||
Ok(SystemReport {
|
||||
system: "Semble".into(),
|
||||
version: env!("CARGO_PKG_VERSION").into(),
|
||||
index: IndexMetrics {
|
||||
cold_ready_ms: model_load_ms + elapsed_ms(cold_duration),
|
||||
cold_index_ms: elapsed_ms(cold_duration),
|
||||
cached_load_ms: model_reload_ms + disk_index_load_ms,
|
||||
in_memory_prepare_ms: Some(in_memory_prepare_ms),
|
||||
indexed_files: stats.file_count,
|
||||
indexed_units: stats.chunk_count,
|
||||
indexed_unit: "chunks".into(),
|
||||
indexed_edges: None,
|
||||
persisted_index_bytes: index_bytes_after.saturating_sub(index_bytes_before),
|
||||
source_bytes: Some(stats.source_bytes),
|
||||
units_per_second: stats.chunk_count as f64 / cold_seconds,
|
||||
source_mib_per_second: Some(stats.source_bytes as f64 / 1_048_576.0 / cold_seconds),
|
||||
},
|
||||
refresh_symbol_latency: Some(refresh_symbol_latency),
|
||||
tracks,
|
||||
})
|
||||
}
|
||||
|
||||
fn benchmark_refresh_symbol(
|
||||
engine: &SearchEngine,
|
||||
source: &Path,
|
||||
suite: &Suite,
|
||||
repetitions: usize,
|
||||
) -> Result<metrics::LatencyMetrics, String> {
|
||||
let mut samples = Vec::with_capacity(suite.queries.len() * repetitions);
|
||||
for case in &suite.queries {
|
||||
for _ in 0..repetitions {
|
||||
let started = Instant::now();
|
||||
engine
|
||||
.refresh(source, &[ContentType::Code])
|
||||
.map_err(|error| format!("refresh before symbol query {}: {error}", case.id))?;
|
||||
engine
|
||||
.search(SearchRequest {
|
||||
query: case.symbol_query.clone(),
|
||||
repo: source.to_path_buf(),
|
||||
top_k: TOP_K,
|
||||
max_snippet_lines: Some(0),
|
||||
content: vec![ContentType::Code],
|
||||
})
|
||||
.map_err(|error| format!("symbol query after refresh {}: {error}", case.id))?;
|
||||
samples.push(elapsed_ms(started.elapsed()));
|
||||
}
|
||||
}
|
||||
Ok(metrics::latency(&samples))
|
||||
}
|
||||
|
||||
fn validate_ground_truth(suite: &Suite, source: &Path) -> Result<(), String> {
|
||||
for case in &suite.queries {
|
||||
for expected in &case.expected {
|
||||
let path = source.join(&expected.path);
|
||||
let text = fs::read_to_string(&path).map_err(|error| {
|
||||
format!(
|
||||
"ground truth {} / {} cannot read {}: {error}",
|
||||
suite.name,
|
||||
case.id,
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
let lines = text.lines().count();
|
||||
if expected.line > lines {
|
||||
return Err(format!(
|
||||
"ground truth {} / {} points past {} lines in {}",
|
||||
suite.name, case.id, lines, expected.path
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn semble_track(
|
||||
engine: &SearchEngine,
|
||||
source: &Path,
|
||||
suite: &Suite,
|
||||
name: &str,
|
||||
query_text: fn(&QueryCase) -> &str,
|
||||
repetitions: usize,
|
||||
) -> Result<TrackReport, String> {
|
||||
let mut queries = Vec::with_capacity(suite.queries.len());
|
||||
let mut samples = Vec::with_capacity(suite.queries.len() * repetitions);
|
||||
for case in &suite.queries {
|
||||
let (report, durations) =
|
||||
benchmark_semble_query(engine, source, case, query_text(case), repetitions)?;
|
||||
queries.push(report);
|
||||
samples.extend(durations);
|
||||
}
|
||||
Ok(track_report(name, queries, samples))
|
||||
}
|
||||
|
||||
fn benchmark_semble_query(
|
||||
engine: &SearchEngine,
|
||||
source: &Path,
|
||||
case: &QueryCase,
|
||||
query: &str,
|
||||
repetitions: usize,
|
||||
) -> Result<(QueryReport, Vec<f64>), String> {
|
||||
let mut samples = Vec::with_capacity(repetitions);
|
||||
let first_results = engine
|
||||
.search(SearchRequest {
|
||||
query: query.to_owned(),
|
||||
repo: source.to_path_buf(),
|
||||
top_k: TOP_K,
|
||||
max_snippet_lines: Some(0),
|
||||
content: vec![ContentType::Code],
|
||||
})
|
||||
.map_err(|error| format!("warm query {}: {error}", case.id))?
|
||||
.results;
|
||||
for _ in 0..repetitions {
|
||||
let started = Instant::now();
|
||||
engine
|
||||
.search(SearchRequest {
|
||||
query: query.to_owned(),
|
||||
repo: source.to_path_buf(),
|
||||
top_k: TOP_K,
|
||||
max_snippet_lines: Some(0),
|
||||
content: vec![ContentType::Code],
|
||||
})
|
||||
.map_err(|error| format!("query {}: {error}", case.id))?;
|
||||
samples.push(elapsed_ms(started.elapsed()));
|
||||
}
|
||||
let summaries = first_results
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, result)| summarize_semble(index + 1, result, &case.expected))
|
||||
.collect::<Vec<_>>();
|
||||
Ok((query_report(case, query, summaries, &samples), samples))
|
||||
}
|
||||
|
||||
fn codegraph_track(track: &AdapterTrack, suite: &Suite) -> Result<TrackReport, String> {
|
||||
let mut samples = Vec::new();
|
||||
let mut queries = Vec::with_capacity(track.queries.len());
|
||||
for query in &track.queries {
|
||||
let case = suite
|
||||
.queries
|
||||
.iter()
|
||||
.find(|case| case.id == query.id)
|
||||
.ok_or_else(|| format!("CodeGraph returned unknown query id {}", query.id))?;
|
||||
let summaries = query
|
||||
.results
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, result)| summarize_codegraph(index + 1, result, &case.expected))
|
||||
.collect();
|
||||
queries.push(query_report(
|
||||
case,
|
||||
&query.query,
|
||||
summaries,
|
||||
&query.durations_ms,
|
||||
));
|
||||
samples.extend(query.durations_ms.iter().copied());
|
||||
}
|
||||
Ok(track_report(&track.name, queries, samples))
|
||||
}
|
||||
|
||||
fn track_report(name: &str, queries: Vec<QueryReport>, samples: Vec<f64>) -> TrackReport {
|
||||
let ranks = queries
|
||||
.iter()
|
||||
.map(|query| query.first_relevant_rank)
|
||||
.collect::<Vec<_>>();
|
||||
TrackReport {
|
||||
name: name.into(),
|
||||
effect: metrics::effect(&ranks),
|
||||
query_latency: metrics::latency(&samples),
|
||||
queries,
|
||||
}
|
||||
}
|
||||
|
||||
fn query_report(
|
||||
case: &QueryCase,
|
||||
query: &str,
|
||||
results: Vec<ResultSummary>,
|
||||
samples: &[f64],
|
||||
) -> QueryReport {
|
||||
QueryReport {
|
||||
id: case.id.clone(),
|
||||
query: query.into(),
|
||||
first_relevant_rank: results
|
||||
.iter()
|
||||
.find(|result| result.relevant)
|
||||
.map(|result| result.rank),
|
||||
latency: metrics::latency(samples),
|
||||
expected: case
|
||||
.expected
|
||||
.iter()
|
||||
.map(|location| format!("{}:{}", location.path, location.line))
|
||||
.collect(),
|
||||
results,
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_semble(
|
||||
rank: usize,
|
||||
result: &SearchResult,
|
||||
expected: &[ExpectedLocation],
|
||||
) -> ResultSummary {
|
||||
summary(
|
||||
rank,
|
||||
&result.file_path,
|
||||
result.start_line,
|
||||
result.end_line,
|
||||
result.score,
|
||||
expected,
|
||||
)
|
||||
}
|
||||
|
||||
fn summarize_codegraph(
|
||||
rank: usize,
|
||||
result: &AdapterResult,
|
||||
expected: &[ExpectedLocation],
|
||||
) -> ResultSummary {
|
||||
let mut output = summary(
|
||||
rank,
|
||||
&result.path,
|
||||
result.start_line,
|
||||
result.end_line,
|
||||
result.score,
|
||||
expected,
|
||||
);
|
||||
if let Some(lines) = result.lines.as_ref() {
|
||||
output.relevant = expected
|
||||
.iter()
|
||||
.any(|location| location.path == result.path && lines.contains(&location.line));
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
fn summary(
|
||||
rank: usize,
|
||||
path: &str,
|
||||
start_line: usize,
|
||||
end_line: usize,
|
||||
score: f32,
|
||||
expected: &[ExpectedLocation],
|
||||
) -> ResultSummary {
|
||||
ResultSummary {
|
||||
rank,
|
||||
path: path.into(),
|
||||
start_line,
|
||||
end_line,
|
||||
score,
|
||||
relevant: expected.iter().any(|location| {
|
||||
location.path == path && start_line <= location.line && location.line <= end_line
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn overall(suites: &[SuiteReport]) -> Vec<OverallReport> {
|
||||
let systems = suites
|
||||
.iter()
|
||||
.flat_map(|suite| suite.systems.iter().map(|system| system.system.as_str()))
|
||||
.collect::<BTreeSet<_>>();
|
||||
systems
|
||||
.into_iter()
|
||||
.flat_map(|system_name| {
|
||||
TRACKS.into_iter().map(move |track_name| {
|
||||
let ranks = suites
|
||||
.iter()
|
||||
.filter_map(|suite| {
|
||||
suite
|
||||
.systems
|
||||
.iter()
|
||||
.find(|system| system.system == system_name)
|
||||
})
|
||||
.filter_map(|system| {
|
||||
system.tracks.iter().find(|track| track.name == track_name)
|
||||
})
|
||||
.flat_map(|track| track.queries.iter().map(|query| query.first_relevant_rank))
|
||||
.collect::<Vec<_>>();
|
||||
OverallReport {
|
||||
system: system_name.into(),
|
||||
track: track_name.into(),
|
||||
effect: metrics::effect(&ranks),
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn elapsed_ms(duration: Duration) -> f64 {
|
||||
duration.as_secs_f64() * 1_000.0
|
||||
}
|
||||
|
||||
fn directory_size(path: &Path) -> Result<u64, String> {
|
||||
if !path.exists() {
|
||||
return Ok(0);
|
||||
}
|
||||
let mut total = 0_u64;
|
||||
let mut pending = vec![path.to_path_buf()];
|
||||
while let Some(directory) = pending.pop() {
|
||||
for entry in fs::read_dir(&directory)
|
||||
.map_err(|error| format!("read {}: {error}", directory.display()))?
|
||||
{
|
||||
let entry = entry.map_err(|error| format!("read directory entry: {error}"))?;
|
||||
let metadata = entry
|
||||
.metadata()
|
||||
.map_err(|error| format!("inspect {}: {error}", entry.path().display()))?;
|
||||
if metadata.is_dir() {
|
||||
pending.push(entry.path());
|
||||
} else if metadata.is_file() {
|
||||
total = total.saturating_add(metadata.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
fn remove_indexes(cache_dir: &Path) -> Result<(), String> {
|
||||
let indexes = cache_dir.join("indexes");
|
||||
if indexes.file_name().and_then(|name| name.to_str()) != Some("indexes") {
|
||||
return Err(format!(
|
||||
"refusing to remove unexpected path {}",
|
||||
indexes.display()
|
||||
));
|
||||
}
|
||||
if indexes.exists() {
|
||||
fs::remove_dir_all(&indexes)
|
||||
.map_err(|error| format!("reset benchmark indexes {}: {error}", indexes.display()))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_report(json: &Path, markdown: &Path, report: &BenchmarkReport) -> Result<(), String> {
|
||||
for path in [json, markdown] {
|
||||
if let Some(parent) = path
|
||||
.parent()
|
||||
.filter(|parent| !parent.as_os_str().is_empty())
|
||||
{
|
||||
fs::create_dir_all(parent).map_err(|error| {
|
||||
format!("create report directory {}: {error}", parent.display())
|
||||
})?;
|
||||
}
|
||||
}
|
||||
let encoded = serde_json::to_vec_pretty(report)
|
||||
.map_err(|error| format!("serialize benchmark report: {error}"))?;
|
||||
fs::write(json, encoded).map_err(|error| format!("write {}: {error}", json.display()))?;
|
||||
fs::write(markdown, report::markdown(report))
|
||||
.map_err(|error| format!("write {}: {error}", markdown.display()))
|
||||
}
|
||||
|
||||
fn rustc_version() -> String {
|
||||
Command::new("rustc")
|
||||
.arg("--version")
|
||||
.output()
|
||||
.ok()
|
||||
.filter(|output| output.status.success())
|
||||
.map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned())
|
||||
.unwrap_or_else(|| "unknown".into())
|
||||
}
|
||||
|
||||
struct Args {
|
||||
cases_dir: PathBuf,
|
||||
sources_dir: PathBuf,
|
||||
codegraph_sources_dir: PathBuf,
|
||||
cache_dir: PathBuf,
|
||||
json: PathBuf,
|
||||
markdown: PathBuf,
|
||||
codegraph_adapter: PathBuf,
|
||||
codegraph: String,
|
||||
quality_gates: PathBuf,
|
||||
sources: HashMap<String, PathBuf>,
|
||||
repetitions: usize,
|
||||
reset_indexes: bool,
|
||||
check: bool,
|
||||
semble_only: bool,
|
||||
help: bool,
|
||||
}
|
||||
|
||||
impl Args {
|
||||
fn parse() -> Result<Self, String> {
|
||||
let root =
|
||||
env::current_dir().map_err(|error| format!("read current directory: {error}"))?;
|
||||
let mut parsed = Self {
|
||||
cases_dir: root.join("benchmarks/semble/cases"),
|
||||
sources_dir: root.join("target/semble-benchmark-sources"),
|
||||
codegraph_sources_dir: root.join("target/semble-benchmark-codegraph-sources"),
|
||||
cache_dir: root.join("target/semble-benchmark-cache"),
|
||||
json: root.join("benchmarks/semble/results/latest.json"),
|
||||
markdown: root.join("benchmarks/semble/results/latest.md"),
|
||||
codegraph_adapter: root.join("benchmarks/semble/codegraph_adapter.cjs"),
|
||||
codegraph: "codegraph".into(),
|
||||
quality_gates: root.join("benchmarks/semble/quality-gates.json"),
|
||||
sources: HashMap::new(),
|
||||
repetitions: 5,
|
||||
reset_indexes: true,
|
||||
check: false,
|
||||
semble_only: false,
|
||||
help: false,
|
||||
};
|
||||
let mut arguments = env::args().skip(1);
|
||||
while let Some(argument) = arguments.next() {
|
||||
match argument.as_str() {
|
||||
"--cases-dir" => parsed.cases_dir = value(&mut arguments, &argument)?.into(),
|
||||
"--sources-dir" => parsed.sources_dir = value(&mut arguments, &argument)?.into(),
|
||||
"--codegraph-sources-dir" => {
|
||||
parsed.codegraph_sources_dir = value(&mut arguments, &argument)?.into()
|
||||
}
|
||||
"--cache-dir" => parsed.cache_dir = value(&mut arguments, &argument)?.into(),
|
||||
"--json" => parsed.json = value(&mut arguments, &argument)?.into(),
|
||||
"--markdown" => parsed.markdown = value(&mut arguments, &argument)?.into(),
|
||||
"--codegraph" => parsed.codegraph = value(&mut arguments, &argument)?,
|
||||
"--codegraph-adapter" => {
|
||||
parsed.codegraph_adapter = value(&mut arguments, &argument)?.into()
|
||||
}
|
||||
"--quality-gates" => {
|
||||
parsed.quality_gates = value(&mut arguments, &argument)?.into()
|
||||
}
|
||||
"--source" => {
|
||||
let source = value(&mut arguments, &argument)?;
|
||||
let (name, path) = source
|
||||
.split_once('=')
|
||||
.ok_or_else(|| "--source must be NAME=PATH".to_owned())?;
|
||||
parsed.sources.insert(name.into(), path.into());
|
||||
}
|
||||
"--repetitions" => {
|
||||
parsed.repetitions = value(&mut arguments, &argument)?
|
||||
.parse()
|
||||
.map_err(|_| "--repetitions must be a positive integer".to_owned())?;
|
||||
}
|
||||
"--keep-indexes" => parsed.reset_indexes = false,
|
||||
"--check" => parsed.check = true,
|
||||
"--semble-only" => parsed.semble_only = true,
|
||||
"-h" | "--help" => parsed.help = true,
|
||||
_ => return Err(format!("unknown argument {argument}")),
|
||||
}
|
||||
}
|
||||
if parsed.repetitions == 0 {
|
||||
return Err("--repetitions must be greater than zero".into());
|
||||
}
|
||||
Ok(parsed)
|
||||
}
|
||||
}
|
||||
|
||||
fn value(arguments: &mut impl Iterator<Item = String>, option: &str) -> Result<String, String> {
|
||||
arguments
|
||||
.next()
|
||||
.ok_or_else(|| format!("{option} requires a value"))
|
||||
}
|
||||
|
||||
fn print_help() {
|
||||
println!(
|
||||
"Semble and CodeGraph React/Vue benchmark\n\n\
|
||||
Options:\n\
|
||||
--cases-dir PATH benchmark suite JSON directory\n\
|
||||
--sources-dir PATH managed pinned Git checkouts\n\
|
||||
--codegraph-sources-dir PATH isolated CodeGraph Git worktrees\n\
|
||||
--cache-dir PATH isolated Semble model and index cache\n\
|
||||
--codegraph PATH CodeGraph executable (default: codegraph)\n\
|
||||
--codegraph-adapter PATH CodeGraph JSON adapter script\n\
|
||||
--quality-gates PATH retrieval quality floor configuration\n\
|
||||
--source NAME=PATH use an existing exact-commit checkout\n\
|
||||
--repetitions N warm search repetitions per query (default 5)\n\
|
||||
--json PATH JSON report path\n\
|
||||
--markdown PATH Markdown report path\n\
|
||||
--keep-indexes keep the existing Semble indexes\n\
|
||||
--check fail when Semble misses a quality floor\n\
|
||||
--semble-only skip CodeGraph for fast Semble iteration"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn relevance_requires_path_and_line_overlap() {
|
||||
let expected = [ExpectedLocation {
|
||||
path: "src/lib.rs".into(),
|
||||
line: 15,
|
||||
}];
|
||||
assert!(summary(1, "src/lib.rs", 10, 20, 1.0, &expected).relevant);
|
||||
assert!(!summary(1, "src/lib.rs", 16, 20, 1.0, &expected).relevant);
|
||||
assert!(!summary(1, "src/other.rs", 10, 20, 1.0, &expected).relevant);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_size_sums_nested_files() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
fs::create_dir(directory.path().join("nested")).unwrap();
|
||||
fs::write(directory.path().join("one"), [1_u8, 2]).unwrap();
|
||||
fs::write(directory.path().join("nested/two"), [3_u8, 4, 5]).unwrap();
|
||||
assert_eq!(directory_size(directory.path()).unwrap(), 5);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
//! Retrieval metrics and latency summaries used by the benchmark report.
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct EffectMetrics {
|
||||
pub query_count: usize,
|
||||
pub recall_at_1: f64,
|
||||
pub recall_at_3: f64,
|
||||
pub recall_at_5: f64,
|
||||
pub recall_at_10: f64,
|
||||
pub mrr_at_10: f64,
|
||||
pub ndcg_at_10: f64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct LatencyMetrics {
|
||||
pub samples: usize,
|
||||
pub min_ms: f64,
|
||||
pub max_ms: f64,
|
||||
pub mean_ms: f64,
|
||||
pub stddev_ms: f64,
|
||||
pub p50_ms: f64,
|
||||
pub p95_ms: f64,
|
||||
pub p99_ms: f64,
|
||||
}
|
||||
|
||||
pub fn effect(ranks: &[Option<usize>]) -> EffectMetrics {
|
||||
let count = ranks.len();
|
||||
let denominator = count.max(1) as f64;
|
||||
let recall = |limit: usize| {
|
||||
ranks
|
||||
.iter()
|
||||
.filter(|rank| rank.is_some_and(|rank| rank <= limit))
|
||||
.count() as f64
|
||||
/ denominator
|
||||
};
|
||||
let mrr = positive_zero(
|
||||
ranks
|
||||
.iter()
|
||||
.filter_map(|rank| rank.filter(|rank| *rank <= 10))
|
||||
.map(|rank| 1.0 / rank as f64)
|
||||
.sum::<f64>()
|
||||
/ denominator,
|
||||
);
|
||||
let ndcg = positive_zero(
|
||||
ranks
|
||||
.iter()
|
||||
.filter_map(|rank| rank.filter(|rank| *rank <= 10))
|
||||
.map(|rank| 1.0 / (rank as f64 + 1.0).log2())
|
||||
.sum::<f64>()
|
||||
/ denominator,
|
||||
);
|
||||
EffectMetrics {
|
||||
query_count: count,
|
||||
recall_at_1: recall(1),
|
||||
recall_at_3: recall(3),
|
||||
recall_at_5: recall(5),
|
||||
recall_at_10: recall(10),
|
||||
mrr_at_10: mrr,
|
||||
ndcg_at_10: ndcg,
|
||||
}
|
||||
}
|
||||
|
||||
fn positive_zero(value: f64) -> f64 {
|
||||
if value == 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
value
|
||||
}
|
||||
}
|
||||
|
||||
pub fn latency(samples_ms: &[f64]) -> LatencyMetrics {
|
||||
let mut sorted = samples_ms.to_vec();
|
||||
sorted.sort_by(f64::total_cmp);
|
||||
let mean = sorted.iter().sum::<f64>() / sorted.len().max(1) as f64;
|
||||
let variance = sorted
|
||||
.iter()
|
||||
.map(|sample| (sample - mean).powi(2))
|
||||
.sum::<f64>()
|
||||
/ sorted.len().max(1) as f64;
|
||||
LatencyMetrics {
|
||||
samples: sorted.len(),
|
||||
min_ms: sorted.first().copied().unwrap_or_default(),
|
||||
max_ms: sorted.last().copied().unwrap_or_default(),
|
||||
mean_ms: mean,
|
||||
stddev_ms: variance.sqrt(),
|
||||
p50_ms: percentile(&sorted, 0.50),
|
||||
p95_ms: percentile(&sorted, 0.95),
|
||||
p99_ms: percentile(&sorted, 0.99),
|
||||
}
|
||||
}
|
||||
|
||||
fn percentile(sorted: &[f64], fraction: f64) -> f64 {
|
||||
if sorted.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let index = ((sorted.len() - 1) as f64 * fraction).ceil() as usize;
|
||||
sorted[index]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn computes_binary_relevance_metrics() {
|
||||
let metrics = effect(&[Some(1), Some(4), None]);
|
||||
assert!((metrics.recall_at_1 - 1.0 / 3.0).abs() < 1e-9);
|
||||
assert!((metrics.recall_at_5 - 2.0 / 3.0).abs() < 1e-9);
|
||||
assert!((metrics.mrr_at_10 - (1.0 + 0.25) / 3.0).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reports_nearest_rank_percentiles() {
|
||||
let metrics = latency(&[5.0, 1.0, 3.0, 2.0, 4.0]);
|
||||
assert_eq!(metrics.mean_ms, 3.0);
|
||||
assert_eq!(metrics.min_ms, 1.0);
|
||||
assert_eq!(metrics.max_ms, 5.0);
|
||||
assert_eq!(metrics.p50_ms, 3.0);
|
||||
assert_eq!(metrics.p95_ms, 5.0);
|
||||
assert_eq!(metrics.p99_ms, 5.0);
|
||||
assert!((metrics.stddev_ms - 2.0_f64.sqrt()).abs() < 1e-9);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
//! Machine-readable comparison report structures and Markdown rendering.
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::metrics::{EffectMetrics, LatencyMetrics};
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct BenchmarkReport {
|
||||
pub schema_version: u32,
|
||||
pub generated_at_unix_seconds: u64,
|
||||
pub environment: Environment,
|
||||
pub configuration: Configuration,
|
||||
pub overall: Vec<OverallReport>,
|
||||
pub suites: Vec<SuiteReport>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct Environment {
|
||||
pub os: String,
|
||||
pub architecture: String,
|
||||
pub rustc: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct Configuration {
|
||||
pub top_k: usize,
|
||||
pub repetitions: usize,
|
||||
pub tracks: Vec<String>,
|
||||
pub relevance: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct OverallReport {
|
||||
pub system: String,
|
||||
pub track: String,
|
||||
pub effect: EffectMetrics,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct SuiteReport {
|
||||
pub name: String,
|
||||
pub repository: String,
|
||||
pub commit: String,
|
||||
pub systems: Vec<SystemReport>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct SystemReport {
|
||||
pub system: String,
|
||||
pub version: String,
|
||||
pub index: IndexMetrics,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub refresh_symbol_latency: Option<LatencyMetrics>,
|
||||
pub tracks: Vec<TrackReport>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct IndexMetrics {
|
||||
pub cold_ready_ms: f64,
|
||||
pub cold_index_ms: f64,
|
||||
pub cached_load_ms: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub in_memory_prepare_ms: Option<f64>,
|
||||
pub indexed_files: usize,
|
||||
pub indexed_units: usize,
|
||||
pub indexed_unit: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub indexed_edges: Option<usize>,
|
||||
pub persisted_index_bytes: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub source_bytes: Option<u64>,
|
||||
pub units_per_second: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub source_mib_per_second: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct TrackReport {
|
||||
pub name: String,
|
||||
pub effect: EffectMetrics,
|
||||
pub query_latency: LatencyMetrics,
|
||||
pub queries: Vec<QueryReport>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct QueryReport {
|
||||
pub id: String,
|
||||
pub query: String,
|
||||
pub first_relevant_rank: Option<usize>,
|
||||
pub latency: LatencyMetrics,
|
||||
pub expected: Vec<String>,
|
||||
pub results: Vec<ResultSummary>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ResultSummary {
|
||||
pub rank: usize,
|
||||
pub path: String,
|
||||
pub start_line: usize,
|
||||
pub end_line: usize,
|
||||
pub score: f32,
|
||||
pub relevant: bool,
|
||||
}
|
||||
|
||||
pub fn markdown(report: &BenchmarkReport) -> String {
|
||||
let mut output = String::new();
|
||||
output.push_str("# Semble 与 CodeGraph React/Vue 对比基准\n\n");
|
||||
output.push_str(&format!(
|
||||
"- 环境：{} {}，{}\n- 查询：Top {}，每条重复 {} 次\n- natural_language：英文行为描述；literal：多词代码/错误字面片段；symbol：精确符号名\n- 对比：natural_language、literal 使用 CodeGraph explore；symbol 使用 CodeGraph searchNodes\n- 判定：返回代码范围必须覆盖人工标注的实现行\n\n",
|
||||
report.environment.os,
|
||||
report.environment.architecture,
|
||||
report.environment.rustc,
|
||||
report.configuration.top_k,
|
||||
report.configuration.repetitions
|
||||
));
|
||||
output.push_str("## 整体效果\n\n");
|
||||
output.push_str("| 系统 | 查询轨道 | Recall@1 | Recall@5 | Recall@10 | MRR@10 | nDCG@10 |\n");
|
||||
output.push_str("| --- | --- | ---: | ---: | ---: | ---: | ---: |\n");
|
||||
for item in &report.overall {
|
||||
output.push_str(&format!(
|
||||
"| {} | {} | {:.1}% | {:.1}% | {:.1}% | {:.3} | {:.3} |\n",
|
||||
item.system,
|
||||
item.track,
|
||||
item.effect.recall_at_1 * 100.0,
|
||||
item.effect.recall_at_5 * 100.0,
|
||||
item.effect.recall_at_10 * 100.0,
|
||||
item.effect.mrr_at_10,
|
||||
item.effect.ndcg_at_10,
|
||||
));
|
||||
}
|
||||
output.push_str("\n## 性能对比\n\n");
|
||||
output.push_str("冷启动就绪包含运行时加载和首次索引；Semble 查询在一秒刷新窗口内直接复用已检查索引，窗口到期或显式 refresh 时重新校验源码指纹。缓存查询与 refresh＋symbol 分开计时；查询耗时均为持久进程中的实际工具处理耗时，不含 CLI 进程启动。每条查询先预热一次。\n\n");
|
||||
output.push_str("| 数据集 | 系统 | 冷启动就绪 | 缓存加载 | 自然语言 P50 / P95 / σ | 字面 P50 / P95 / σ | 缓存符号 P50 / P95 / σ | 刷新＋符号 P50 / P95 | 文件 / 索引单元 | 索引体积 |\n");
|
||||
output.push_str("| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n");
|
||||
for suite in &report.suites {
|
||||
for system in &suite.systems {
|
||||
let natural = track(system, "natural_language");
|
||||
let literal = track(system, "literal");
|
||||
let symbol = track(system, "symbol");
|
||||
output.push_str(&format!(
|
||||
"| {} | {} {} | {:.1} ms | {:.1} ms | {:.2} / {:.2} / {:.2} ms | {:.2} / {:.2} / {:.2} ms | {:.2} / {:.2} / {:.2} ms | {} | {} / {} {}{} | {:.2} MiB |\n",
|
||||
suite.name,
|
||||
system.system,
|
||||
system.version,
|
||||
system.index.cold_ready_ms,
|
||||
system.index.cached_load_ms,
|
||||
natural.query_latency.p50_ms,
|
||||
natural.query_latency.p95_ms,
|
||||
natural.query_latency.stddev_ms,
|
||||
literal.query_latency.p50_ms,
|
||||
literal.query_latency.p95_ms,
|
||||
literal.query_latency.stddev_ms,
|
||||
symbol.query_latency.p50_ms,
|
||||
symbol.query_latency.p95_ms,
|
||||
symbol.query_latency.stddev_ms,
|
||||
system
|
||||
.refresh_symbol_latency
|
||||
.as_ref()
|
||||
.map(|latency| format!("{:.2} / {:.2} ms", latency.p50_ms, latency.p95_ms))
|
||||
.unwrap_or_else(|| "—".into()),
|
||||
system.index.indexed_files,
|
||||
system.index.indexed_units,
|
||||
system.index.indexed_unit,
|
||||
system
|
||||
.index
|
||||
.indexed_edges
|
||||
.map(|edges| format!(" / {edges} edges"))
|
||||
.unwrap_or_default(),
|
||||
as_mib(system.index.persisted_index_bytes),
|
||||
));
|
||||
}
|
||||
}
|
||||
output.push_str("\n## 分数据集效果\n\n");
|
||||
output.push_str("| 数据集 | 系统 | 轨道 | Recall@1 / @5 / @10 | MRR@10 | nDCG@10 |\n");
|
||||
output.push_str("| --- | --- | --- | ---: | ---: | ---: |\n");
|
||||
for suite in &report.suites {
|
||||
for system in &suite.systems {
|
||||
for track in &system.tracks {
|
||||
output.push_str(&format!(
|
||||
"| {} | {} | {} | {:.1}% / {:.1}% / {:.1}% | {:.3} | {:.3} |\n",
|
||||
suite.name,
|
||||
system.system,
|
||||
track.name,
|
||||
track.effect.recall_at_1 * 100.0,
|
||||
track.effect.recall_at_5 * 100.0,
|
||||
track.effect.recall_at_10 * 100.0,
|
||||
track.effect.mrr_at_10,
|
||||
track.effect.ndcg_at_10,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
for suite in &report.suites {
|
||||
for system in &suite.systems {
|
||||
for track in &system.tracks {
|
||||
output.push_str(&format!(
|
||||
"\n## {} · {} · {} 明细\n\n",
|
||||
suite.name, system.system, track.name
|
||||
));
|
||||
output.push_str("| 查询 | 首个命中 | P50 / P95 / σ | Top 1 |\n");
|
||||
output.push_str("| --- | ---: | ---: | --- |\n");
|
||||
for query in &track.queries {
|
||||
let rank = query
|
||||
.first_relevant_rank
|
||||
.map(|rank| rank.to_string())
|
||||
.unwrap_or_else(|| "未命中".into());
|
||||
let top = query
|
||||
.results
|
||||
.first()
|
||||
.map(|result| {
|
||||
format!("{}:{}-{}", result.path, result.start_line, result.end_line)
|
||||
})
|
||||
.unwrap_or_else(|| "—".into());
|
||||
output.push_str(&format!(
|
||||
"| `{}` | {} | {:.2} / {:.2} / {:.2} ms | `{}` |\n",
|
||||
query.id,
|
||||
rank,
|
||||
query.latency.p50_ms,
|
||||
query.latency.p95_ms,
|
||||
query.latency.stddev_ms,
|
||||
top
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
output.push_str("\n## 公平性说明\n\n");
|
||||
output.push_str("自然语言和多词字面轨道调用 CodeGraph 官方推荐的 codegraph_explore，包含图扩展和最终源码读取；符号轨道调用 searchNodes。Semble 三条轨道都调用同一个混合搜索接口。所有查询共享同一组人工标注实现位置，未针对系统选择不同真值。两者输出粒度不同，因此本报告适合比较达到同一代码位置的效果和实际工具延迟，不代表各内部算法的微基准。CodeGraph 的独立 callers、callees 和 impact 能力不属于本次范围。\n");
|
||||
output
|
||||
}
|
||||
|
||||
fn track<'a>(system: &'a SystemReport, name: &str) -> &'a TrackReport {
|
||||
system
|
||||
.tracks
|
||||
.iter()
|
||||
.find(|track| track.name == name)
|
||||
.expect("every system report contains all benchmark tracks")
|
||||
}
|
||||
|
||||
fn as_mib(bytes: u64) -> f64 {
|
||||
bytes as f64 / 1_048_576.0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn effect() -> EffectMetrics {
|
||||
EffectMetrics {
|
||||
query_count: 1,
|
||||
recall_at_1: 1.0,
|
||||
recall_at_3: 1.0,
|
||||
recall_at_5: 1.0,
|
||||
recall_at_10: 1.0,
|
||||
mrr_at_10: 1.0,
|
||||
ndcg_at_10: 1.0,
|
||||
}
|
||||
}
|
||||
|
||||
fn track_report(name: &str) -> TrackReport {
|
||||
TrackReport {
|
||||
name: name.into(),
|
||||
effect: effect(),
|
||||
query_latency: LatencyMetrics {
|
||||
samples: 1,
|
||||
min_ms: 1.0,
|
||||
max_ms: 1.0,
|
||||
mean_ms: 1.0,
|
||||
stddev_ms: 0.0,
|
||||
p50_ms: 1.0,
|
||||
p95_ms: 1.0,
|
||||
p99_ms: 1.0,
|
||||
},
|
||||
queries: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markdown_contains_both_systems_and_tracks() {
|
||||
let report = BenchmarkReport {
|
||||
schema_version: 4,
|
||||
generated_at_unix_seconds: 0,
|
||||
environment: Environment {
|
||||
os: "test".into(),
|
||||
architecture: "test".into(),
|
||||
rustc: "rustc test".into(),
|
||||
},
|
||||
configuration: Configuration {
|
||||
top_k: 10,
|
||||
repetitions: 3,
|
||||
tracks: vec!["natural_language".into(), "literal".into(), "symbol".into()],
|
||||
relevance: "location overlap".into(),
|
||||
},
|
||||
overall: vec![OverallReport {
|
||||
system: "Semble".into(),
|
||||
track: "natural_language".into(),
|
||||
effect: effect(),
|
||||
}],
|
||||
suites: vec![SuiteReport {
|
||||
name: "fixture".into(),
|
||||
repository: "repo".into(),
|
||||
commit: "commit".into(),
|
||||
systems: vec![SystemReport {
|
||||
system: "Semble".into(),
|
||||
version: "0.1".into(),
|
||||
index: IndexMetrics {
|
||||
cold_ready_ms: 1.0,
|
||||
cold_index_ms: 1.0,
|
||||
cached_load_ms: 1.0,
|
||||
in_memory_prepare_ms: Some(1.0),
|
||||
indexed_files: 1,
|
||||
indexed_units: 2,
|
||||
indexed_unit: "chunks".into(),
|
||||
indexed_edges: None,
|
||||
persisted_index_bytes: 3,
|
||||
source_bytes: Some(4),
|
||||
units_per_second: 5.0,
|
||||
source_mib_per_second: Some(6.0),
|
||||
},
|
||||
refresh_symbol_latency: Some(LatencyMetrics {
|
||||
samples: 1,
|
||||
min_ms: 2.0,
|
||||
max_ms: 2.0,
|
||||
mean_ms: 2.0,
|
||||
stddev_ms: 0.0,
|
||||
p50_ms: 2.0,
|
||||
p95_ms: 2.0,
|
||||
p99_ms: 2.0,
|
||||
}),
|
||||
tracks: vec![
|
||||
track_report("natural_language"),
|
||||
track_report("literal"),
|
||||
track_report("symbol"),
|
||||
],
|
||||
}],
|
||||
}],
|
||||
};
|
||||
let rendered = markdown(&report);
|
||||
assert!(rendered.contains("Semble"));
|
||||
assert!(rendered.contains("natural_language"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//! Deterministic Git checkout management for version-pinned benchmark sources.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
process::Command,
|
||||
};
|
||||
|
||||
use crate::case::Suite;
|
||||
|
||||
pub fn resolve_sources(
|
||||
suites: &[Suite],
|
||||
root: &Path,
|
||||
overrides: &HashMap<String, PathBuf>,
|
||||
) -> Result<HashMap<String, PathBuf>, String> {
|
||||
fs::create_dir_all(root)
|
||||
.map_err(|error| format!("create source directory {}: {error}", root.display()))?;
|
||||
suites
|
||||
.iter()
|
||||
.map(|suite| {
|
||||
let path = overrides
|
||||
.get(&suite.name)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| root.join(&suite.name));
|
||||
if overrides.contains_key(&suite.name) {
|
||||
verify_checkout(&path, &suite.commit)?;
|
||||
} else {
|
||||
ensure_checkout(&path, suite)?;
|
||||
}
|
||||
Ok((suite.name.clone(), path))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn prepare_competitor_sources(
|
||||
suites: &[Suite],
|
||||
sources: &HashMap<String, PathBuf>,
|
||||
root: &Path,
|
||||
) -> Result<HashMap<String, PathBuf>, String> {
|
||||
fs::create_dir_all(root).map_err(|error| {
|
||||
format!(
|
||||
"create competitor source directory {}: {error}",
|
||||
root.display()
|
||||
)
|
||||
})?;
|
||||
suites
|
||||
.iter()
|
||||
.map(|suite| {
|
||||
let source = sources
|
||||
.get(&suite.name)
|
||||
.ok_or_else(|| format!("missing source path for {}", suite.name))?;
|
||||
let target = root.join(&suite.name);
|
||||
if target.exists() {
|
||||
fs::remove_dir_all(&target).map_err(|error| {
|
||||
format!("reset competitor checkout {}: {error}", target.display())
|
||||
})?;
|
||||
}
|
||||
run(
|
||||
Command::new("git")
|
||||
.args(["clone", "--shared", "--no-checkout"])
|
||||
.arg(source)
|
||||
.arg(&target),
|
||||
"clone isolated competitor checkout",
|
||||
)?;
|
||||
run(
|
||||
Command::new("git").arg("-C").arg(&target).args([
|
||||
"checkout",
|
||||
"--detach",
|
||||
&suite.commit,
|
||||
]),
|
||||
"checkout competitor commit",
|
||||
)?;
|
||||
Ok((suite.name.clone(), target))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn ensure_checkout(path: &Path, suite: &Suite) -> Result<(), String> {
|
||||
if !path.join(".git").exists() {
|
||||
if path.exists() {
|
||||
fs::remove_dir_all(path).map_err(|error| {
|
||||
format!("remove incomplete checkout {}: {error}", path.display())
|
||||
})?;
|
||||
}
|
||||
run(
|
||||
Command::new("git")
|
||||
.args(["clone", "--filter=blob:none", "--no-checkout"])
|
||||
.arg(&suite.repository)
|
||||
.arg(path),
|
||||
"clone repository",
|
||||
)?;
|
||||
}
|
||||
if verify_checkout(path, &suite.commit).is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
run(
|
||||
Command::new("git").arg("-C").arg(path).args([
|
||||
"fetch",
|
||||
"--depth=1",
|
||||
"origin",
|
||||
&suite.commit,
|
||||
]),
|
||||
"fetch pinned commit",
|
||||
)?;
|
||||
run(
|
||||
Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(path)
|
||||
.args(["checkout", "--detach", &suite.commit]),
|
||||
"checkout pinned commit",
|
||||
)?;
|
||||
verify_checkout(path, &suite.commit)
|
||||
}
|
||||
|
||||
fn verify_checkout(path: &Path, expected: &str) -> Result<(), String> {
|
||||
let output = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(path)
|
||||
.args(["rev-parse", "HEAD"])
|
||||
.output()
|
||||
.map_err(|error| format!("inspect checkout {}: {error}", path.display()))?;
|
||||
if !output.status.success() {
|
||||
return Err(format!("{} is not a readable Git checkout", path.display()));
|
||||
}
|
||||
let actual = String::from_utf8_lossy(&output.stdout).trim().to_owned();
|
||||
if actual != expected {
|
||||
return Err(format!(
|
||||
"checkout {} is at {actual}, expected {expected}",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run(command: &mut Command, action: &str) -> Result<(), String> {
|
||||
let status = command
|
||||
.status()
|
||||
.map_err(|error| format!("{action}: {error}"))?;
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("{action} failed with {status}"))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user