mirror of
https://wget.la/https://github.com/Wxw-Gu/WechatExplorer
synced 2026-10-03 18:33:14 +08:00
feat: 图片文字索引按时间分段优先处理最近图片
- 首次索引先处理最近 7 天,再依次回溯 30 天 / 近一年 / 更早历史 - 完成后新到的图片单独补齐,不受历史回填影响 - 覆盖度增加时间维度,可区分「最近已完整」与「更早仍在补齐」
This commit is contained in:
@@ -445,3 +445,105 @@ describe('图片文字索引卡片 — 修复图片搜索索引', () => {
|
||||
expect(String(onNotice.mock.calls.at(-1)?.[0])).toContain('正在进行中')
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* recent-first 的阶段可见性。
|
||||
*
|
||||
* 这里要守住的是两件**不能混**的事:
|
||||
* - **总进度**永远是 `processed / totalImageMessages`(全量图片消息);
|
||||
* - **阶段文案**只回答"现在在优先做什么"。
|
||||
*
|
||||
* 用"某个分段做完了"去冒充整体完成,是这次改动最容易撒的谎,所以两条都断言。
|
||||
*/
|
||||
describe('recent-first 阶段显示', () => {
|
||||
/** 已建立、且不在运行/暂停 —— 卡片明细块的渲染条件。 */
|
||||
const settled = (
|
||||
coverageOverride: Partial<ImageTextIndexStatus['coverage']>
|
||||
): ImageTextIndexStatus =>
|
||||
status({
|
||||
...running,
|
||||
progress: { ...running.progress, state: 'idle', cancellable: false, paused: false },
|
||||
coverage: { ...running.coverage, ...coverageOverride }
|
||||
})
|
||||
|
||||
const tiers = (
|
||||
completeTier: 'recent_7d' | 'recent_30d' | 'recent_1y' | 'archive' | null
|
||||
): ImageTextIndexStatus['coverage']['tiers'] => [
|
||||
{
|
||||
tier: 'recent_7d',
|
||||
state: completeTier === 'recent_7d' ? 'complete' : 'running',
|
||||
startMs: 7,
|
||||
endMs: 8
|
||||
},
|
||||
{
|
||||
tier: 'recent_30d',
|
||||
state: completeTier === 'recent_30d' ? 'complete' : 'pending',
|
||||
startMs: 6,
|
||||
endMs: 7
|
||||
},
|
||||
{
|
||||
tier: 'recent_1y',
|
||||
state: completeTier === 'recent_1y' ? 'complete' : 'pending',
|
||||
startMs: 5,
|
||||
endMs: 6
|
||||
},
|
||||
{
|
||||
tier: 'archive',
|
||||
state: completeTier === 'archive' ? 'complete' : 'pending',
|
||||
startMs: 4,
|
||||
endMs: 5
|
||||
}
|
||||
]
|
||||
|
||||
it('运行中显示阶段文案,同时总进度仍以全量为分母', async () => {
|
||||
api.getImageTextIndexStatus.mockResolvedValue(
|
||||
status({
|
||||
...running,
|
||||
progress: { ...running.progress, currentPhase: 'recent_30d' }
|
||||
})
|
||||
)
|
||||
|
||||
await renderCard()
|
||||
|
||||
expect(screen.getByTestId('image-text-index-phase').textContent).toBe('正在补齐最近 30 天')
|
||||
// 阶段 ≠ 进度:分子分母仍然是全量数字,不是"这一段处理了多少张"。
|
||||
expect(screen.getByTestId('image-text-index-progress').textContent).toBe(
|
||||
`${running.progress.processed.toLocaleString()} / ${running.progress.totalImageMessages.toLocaleString()}`
|
||||
)
|
||||
})
|
||||
|
||||
it('阶段文案是用户语言,不出现工程术语', async () => {
|
||||
api.getImageTextIndexStatus.mockResolvedValue(
|
||||
status({ ...running, progress: { ...running.progress, currentPhase: 'archive' } })
|
||||
)
|
||||
|
||||
await renderCard()
|
||||
|
||||
const text = screen.getByTestId('image-text-index-phase').textContent ?? ''
|
||||
expect(text).toBe('正在补齐更早图片')
|
||||
expect(text).not.toMatch(/Tier/i)
|
||||
})
|
||||
|
||||
it('分段真的完成时才宣告"已可搜索"', async () => {
|
||||
api.getImageTextIndexStatus.mockResolvedValue(
|
||||
settled({ tiers: tiers('recent_7d'), coveredToMs: 8 })
|
||||
)
|
||||
|
||||
await renderCard()
|
||||
|
||||
expect(screen.getByTestId('image-text-index-searchable-notice').textContent).toBe(
|
||||
'最近图片已可搜索'
|
||||
)
|
||||
})
|
||||
|
||||
it('分段还没完成时不得宣告"已可搜索"', async () => {
|
||||
api.getImageTextIndexStatus.mockResolvedValue(
|
||||
settled({ tiers: tiers(null), coveredToMs: 8 })
|
||||
)
|
||||
|
||||
await renderCard()
|
||||
|
||||
// 这是一句承诺,不是进度提示:没有真正 complete 就不许说。
|
||||
expect(screen.queryByTestId('image-text-index-searchable-notice')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
} from '../../src/main/services/image-text-index-store'
|
||||
import { buildImageOcrCoverage } from '../../src/main/services/local-query-api-service'
|
||||
import type { ImageTextIndexCoverage } from '../../src/shared/image-text-index'
|
||||
import { sourceMessageId } from '../../src/main/knowledge/message-identity'
|
||||
|
||||
const ACCOUNT = 'wxid_fixture_account'
|
||||
const CONVERSATION = 'conversation-md5-fixture'
|
||||
@@ -93,72 +94,161 @@ function makeHarness(options: { messages?: chat.FormattedMessage[] } = {}): Harn
|
||||
}
|
||||
}
|
||||
|
||||
describe('增量水位:只比条数会漏掉「等量替换」', () => {
|
||||
it('水位(条数 + 最大插入序)都没变时才跳过,不读 WCDB', async () => {
|
||||
const harness = makeHarness({ messages: [imageMessage(10, 1000), imageMessage(20, 2000)] })
|
||||
harness.watermark.count = 2
|
||||
harness.watermark.maxLocalId = 20
|
||||
|
||||
await harness.service.startPass()
|
||||
// 走到完成态需要等内部 promise 收敛。
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false))
|
||||
expect(harness.listMessages).toHaveBeenCalledTimes(1)
|
||||
|
||||
// 第二遍:水位完全一致 → 跳过,不再读会话消息。
|
||||
await harness.service.startPass()
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false))
|
||||
expect(harness.listMessages).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('总数相同但最大插入序前进 → 必须重扫(撤回一张旧图 + 新增一张新图)', async () => {
|
||||
const harness = makeHarness({ messages: [imageMessage(10, 1000), imageMessage(20, 2000)] })
|
||||
harness.watermark.count = 2
|
||||
harness.watermark.maxLocalId = 20
|
||||
|
||||
await harness.service.startPass()
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false))
|
||||
expect(harness.listMessages).toHaveBeenCalledTimes(1)
|
||||
|
||||
// 集合变了、条数没变:localId 10 被撤回,新增 localId 30。
|
||||
harness.listMessages.mockImplementation(async () => [
|
||||
imageMessage(20, 2000),
|
||||
imageMessage(30, 3000)
|
||||
])
|
||||
harness.watermark.maxLocalId = 30
|
||||
|
||||
await harness.service.startPass()
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false))
|
||||
// 只看 count 的实现会在这里静默跳过 —— 那正是会漏掉新图片的洞。
|
||||
expect(harness.listMessages).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('水位不可用(数据库不支持该聚合)时一律重扫,宁可慢也不漏', async () => {
|
||||
const harness = makeHarness({ messages: [imageMessage(10, 1000)] })
|
||||
harness.watermark.count = 1
|
||||
/**
|
||||
* 增量判据。
|
||||
*
|
||||
* recent-first 之后,"什么时候读 WCDB"由**分段完成状态 + 增量水位**共同决定,
|
||||
* 但两条硬约束一个字都没变:
|
||||
* 1. 没有新内容时**不得**重复读会话消息、更不得重复 OCR;
|
||||
* 2. 有新内容(含"总数不变但集合变了")时**必须**处理到 —— 宁可慢,也不漏。
|
||||
*
|
||||
* 这里的夹具刻意做成**窗口感知**的:新架构的"这段没有图片就整段跳过"完全依赖
|
||||
* 计数说实话;忽略窗口的夹具测出来的只是"夹具不过滤",不是调度器的行为。
|
||||
*/
|
||||
describe('增量判据:水位不得漏掉新图片', () => {
|
||||
function makeIncrementalHarness(options: {
|
||||
messages: chat.FormattedMessage[]
|
||||
withWatermark?: boolean
|
||||
}): {
|
||||
service: ImageTextIndexService
|
||||
databasePath: string
|
||||
state: { messages: chat.FormattedMessage[]; count: number; maxLocalId: number; now: number }
|
||||
listMessages: ReturnType<typeof vi.fn>
|
||||
listImageMessages: ReturnType<typeof vi.fn>
|
||||
} {
|
||||
const databaseRoot = makeDatabaseRoot()
|
||||
const databasePath = getImageTextIndexDatabasePath(databaseRoot, ACCOUNT)
|
||||
const state = {
|
||||
messages: [...options.messages],
|
||||
count: options.messages.length,
|
||||
maxLocalId: options.messages.reduce((max, m) => Math.max(max, Number(m.localId) || 0), 0),
|
||||
now: 1_800_000_000_000
|
||||
}
|
||||
const inWindow = (
|
||||
message: chat.FormattedMessage,
|
||||
window?: { sinceMs?: number; beforeMs?: number }
|
||||
): boolean => {
|
||||
const createTimeMs = (message.createTime || 0) * 1000
|
||||
if (window?.sinceMs !== undefined && createTimeMs < window.sinceMs) return false
|
||||
if (window?.beforeMs !== undefined && createTimeMs >= window.beforeMs) return false
|
||||
return true
|
||||
}
|
||||
const listMessages = vi.fn(async () => state.messages)
|
||||
const listImageMessages = vi.fn(
|
||||
async (_conversationId: string, window?: { sinceMs?: number; beforeMs?: number }) =>
|
||||
state.messages.filter((message) => inWindow(message, window))
|
||||
)
|
||||
const service = new ImageTextIndexService()
|
||||
const listMessages = vi.fn(async () => [imageMessage(10, 1000)])
|
||||
service.bind({
|
||||
databaseRoot: harness.databaseRoot,
|
||||
databaseRoot,
|
||||
resolveAccountId: () => ACCOUNT,
|
||||
listContacts: async () => [{ md5: CONVERSATION, m_nsUsrName: 'fixture', type: 'group' }],
|
||||
resolveAccountRoot: () => 'C:/fixture/account',
|
||||
now: () => state.now,
|
||||
listContacts: async () => [
|
||||
{ md5: CONVERSATION, m_nsUsrName: 'fixture', type: 'group' as const }
|
||||
],
|
||||
listMessages,
|
||||
countConversationImages: async () => ({ count: 1, typeColumn: 'local_type' }),
|
||||
// 关键:不提供 imageWatermark
|
||||
listImageMessages,
|
||||
countConversationImages: async (
|
||||
_conversationId: string,
|
||||
window?: { sinceMs?: number; beforeMs?: number }
|
||||
) => ({
|
||||
count: state.messages.filter((message) => inWindow(message, window)).length,
|
||||
typeColumn: 'local_type'
|
||||
}),
|
||||
...(options.withWatermark === false
|
||||
? {}
|
||||
: { imageWatermark: async () => ({ count: state.count, maxLocalId: state.maxLocalId }) }),
|
||||
// 没有解密服务 → 每张图片都会被判成 image_missing。这样测试完全不碰真实图片。
|
||||
decryptService: () => ({ findImageFile: () => null, decryptImage: () => null }) as never,
|
||||
capability: async () => ({
|
||||
available: true,
|
||||
engine: 'windows-system-ocr',
|
||||
platform: 'win32',
|
||||
runtimeVersion: null,
|
||||
language: null
|
||||
})
|
||||
runtimeVersion: '1.2.0',
|
||||
language: 'zh-Hans-CN'
|
||||
}),
|
||||
recognize: async () => ({ success: true, text: '', language: 'zh-Hans-CN' })
|
||||
})
|
||||
return { service, databasePath, state, listMessages, listImageMessages }
|
||||
}
|
||||
|
||||
/** 直接读派生库的绑定,回答"到底处理了哪几条",而不是只看调用次数。 */
|
||||
const bindingIds = (databasePath: string): string[] => {
|
||||
const store = new ImageTextIndexStore(databasePath, ACCOUNT)
|
||||
const ids = [...store.getConversationOcr(CONVERSATION).keys()]
|
||||
store.close()
|
||||
return ids
|
||||
}
|
||||
|
||||
it('全部完成之后:第二遍不再读会话消息,也不重复 OCR', async () => {
|
||||
const harness = makeIncrementalHarness({
|
||||
messages: [imageMessage(10, 1000), imageMessage(20, 2000)]
|
||||
})
|
||||
|
||||
await service.startPass()
|
||||
await vi.waitFor(() => expect(service.isRunning()).toBe(false))
|
||||
await service.startPass()
|
||||
await vi.waitFor(() => expect(service.isRunning()).toBe(false))
|
||||
expect(listMessages).toHaveBeenCalledTimes(2)
|
||||
await harness.service.startPass()
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false))
|
||||
// 两条图片的 create_time 都很老 → 只落在归档段,只被那一段读到。
|
||||
expect(harness.listImageMessages).toHaveBeenCalledTimes(1)
|
||||
const firstIds = bindingIds(harness.databasePath)
|
||||
expect(firstIds).toHaveLength(2)
|
||||
|
||||
// 第二遍:水位完全一致 + 各分段已完成 → 一个字都不再读。
|
||||
await harness.service.startPass()
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false))
|
||||
expect(harness.listImageMessages).toHaveBeenCalledTimes(1)
|
||||
expect(bindingIds(harness.databasePath)).toEqual(firstIds)
|
||||
})
|
||||
|
||||
it('总数相同但最大插入序前进 → 必须处理到新图片(即使是旧时间)', async () => {
|
||||
const harness = makeIncrementalHarness({
|
||||
messages: [imageMessage(10, 1000), imageMessage(20, 2000)]
|
||||
})
|
||||
|
||||
await harness.service.startPass()
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false))
|
||||
expect(bindingIds(harness.databasePath)).toHaveLength(2)
|
||||
|
||||
// 集合变了、条数没变:localId 10 被撤回,新增 localId 30 —— 而且它带着**旧时间**,
|
||||
// 只按时间窗读的实现在这里就会漏掉它。
|
||||
harness.state.messages = [imageMessage(20, 2000), imageMessage(30, 2000)]
|
||||
harness.state.count = 2
|
||||
harness.state.maxLocalId = 30
|
||||
harness.state.now += 5_000
|
||||
|
||||
await harness.service.startPass()
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false))
|
||||
|
||||
// 只看 count 的实现会在这里静默跳过 —— 那正是会漏掉新图片的洞。
|
||||
const ids = bindingIds(harness.databasePath)
|
||||
expect(ids).toHaveLength(3)
|
||||
expect(ids).toContain(sourceMessageId(imageMessage(30, 2000)))
|
||||
})
|
||||
|
||||
it('水位不可用(数据库不支持该聚合)时按时间窗兜底重扫,宁可慢也不漏', async () => {
|
||||
const harness = makeIncrementalHarness({
|
||||
messages: [imageMessage(10, 1000)],
|
||||
withWatermark: false
|
||||
})
|
||||
const anchorSeconds = Math.floor(harness.state.now / 1000)
|
||||
|
||||
await harness.service.startPass()
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false))
|
||||
expect(bindingIds(harness.databasePath)).toHaveLength(1)
|
||||
|
||||
// 锚点之后新到一张图;并且时间确实往前走了一段(否则"窗口必然为空"的短路会生效,
|
||||
// 那不是漏,而是正确地判定"还没有新东西")。
|
||||
harness.state.messages = [imageMessage(10, 1000), imageMessage(40, anchorSeconds + 10)]
|
||||
harness.state.count = 2
|
||||
harness.state.maxLocalId = 40
|
||||
harness.state.now += 20_000
|
||||
|
||||
await harness.service.startPass()
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false))
|
||||
|
||||
const ids = bindingIds(harness.databasePath)
|
||||
expect(ids).toHaveLength(2)
|
||||
expect(ids).toContain(sourceMessageId(imageMessage(40, anchorSeconds + 10)))
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -86,8 +86,24 @@ function createHarness(): Harness {
|
||||
const all = mixedMessages()
|
||||
const imagesOnly = all.filter((message) => message.contentData?.type === 'image')
|
||||
|
||||
/**
|
||||
* recent-first 之后,"读哪些行"由**时间分段**决定,所以夹具必须像真 WCDB 一样
|
||||
* 按窗口说话。忽略窗口的夹具测不出调度行为,只会让"读了几次"变成 4 次。
|
||||
*/
|
||||
const inWindow = (
|
||||
message: chat.FormattedMessage,
|
||||
window?: { sinceMs?: number; beforeMs?: number }
|
||||
): boolean => {
|
||||
const createTimeMs = (message.createTime || 0) * 1000
|
||||
if (window?.sinceMs !== undefined && createTimeMs < window.sinceMs) return false
|
||||
if (window?.beforeMs !== undefined && createTimeMs >= window.beforeMs) return false
|
||||
return true
|
||||
}
|
||||
const listMessages = vi.fn(async () => all)
|
||||
const listImageMessages = vi.fn(async () => imagesOnly)
|
||||
const listImageMessages = vi.fn(
|
||||
async (_conversationId: string, window?: { sinceMs?: number; beforeMs?: number }) =>
|
||||
imagesOnly.filter((message) => inWindow(message, window))
|
||||
)
|
||||
|
||||
const service = new ImageTextIndexService()
|
||||
service.bind({
|
||||
@@ -98,7 +114,13 @@ function createHarness(): Harness {
|
||||
listContacts: async () => [
|
||||
{ md5: CONVERSATION, m_nsUsrName: 'boundary', type: 'user' as const }
|
||||
],
|
||||
countConversationImages: async () => ({ count: IMAGE_COUNT, typeColumn: 'local_type' }),
|
||||
countConversationImages: async (
|
||||
_conversationId: string,
|
||||
window?: { sinceMs?: number; beforeMs?: number }
|
||||
) => ({
|
||||
count: imagesOnly.filter((message) => inWindow(message, window)).length,
|
||||
typeColumn: 'local_type'
|
||||
}),
|
||||
imageWatermark: async () => ({ count: IMAGE_COUNT, maxLocalId: TEXT_COUNT + IMAGE_COUNT }),
|
||||
listMessages,
|
||||
listImageMessages,
|
||||
|
||||
@@ -117,8 +117,15 @@ function createService(options: { count: number; notifyIntervalMs: number; perIm
|
||||
return { service, notifications }
|
||||
}
|
||||
|
||||
/**
|
||||
* 等 pass 收尾。
|
||||
*
|
||||
* 显式给足超时:这些用例故意让 240 张图各睡几毫秒来制造可观测的持续时间,
|
||||
* 而 `vi.waitFor` 的默认超时是 1000ms —— 机器稍慢就会以**断言失败**而不是
|
||||
* "超时"的形态报出来。这里等的是"跑完",不是"跑得快"。
|
||||
*/
|
||||
const finish = async (service: ImageTextIndexService): Promise<void> => {
|
||||
await vi.waitFor(() => expect(service.isRunning()).toBe(false))
|
||||
await vi.waitFor(() => expect(service.isRunning()).toBe(false), { timeout: 30_000 })
|
||||
}
|
||||
|
||||
describe('进度通知节流', () => {
|
||||
|
||||
@@ -0,0 +1,543 @@
|
||||
/**
|
||||
* 「recent-first 图片文字索引」的行为测试。
|
||||
*
|
||||
* 这一轮改的是**调度**:不再从最老的历史往下扫,而是先让"最近聊天的图片"可搜索。
|
||||
* 调度改动最容易骗人的地方是"看起来更快了,其实漏了东西",所以这里的断言都落在
|
||||
* **可观察的结果**上(读了哪些窗口、处理了哪些消息、覆盖度怎么说),而不是内部变量。
|
||||
*
|
||||
* 被测的性质:
|
||||
* - 分段计划:互不重叠、无空隙,边界唯一;
|
||||
* - 处理顺序按时间分段从新到旧;
|
||||
* - 断点续跑不重复 OCR;
|
||||
* - 历史分段跑着的时候,新图片仍然优先;
|
||||
* - 覆盖度能按时间范围回答"这段能不能下确定性结论";
|
||||
* - 老版本既有索引 / 旧 checkpoint 不被降级,源侧有新增时必须补上;
|
||||
* - UI 文案不暴露工程术语。
|
||||
*/
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type * as chat from '../../src/main/services/chat-service'
|
||||
import { ImageTextIndexService } from '../../src/main/services/image-text-index-service'
|
||||
import {
|
||||
ImageTextIndexStore,
|
||||
getImageTextIndexDatabasePath
|
||||
} from '../../src/main/services/image-text-index-store'
|
||||
import { sourceMessageId } from '../../src/main/knowledge/message-identity'
|
||||
import {
|
||||
IMAGE_TEXT_BACKFILL_WINDOW_DAYS,
|
||||
buildImageTextBackfillSegments,
|
||||
describeImageTextRangeCoverage,
|
||||
imageTextPhaseLabel,
|
||||
imageTextRangeCoverage,
|
||||
imageTextWindowToSeconds,
|
||||
type ImageTextBackfillTier,
|
||||
type ImageTextIndexCoverage,
|
||||
type ImageTextTierRunState
|
||||
} from '../../src/shared/image-text-index'
|
||||
|
||||
const ACCOUNT = 'wxid_recent_first_fixture'
|
||||
const CONVERSATION = 'conversation-fixture-md5'
|
||||
const DAY_MS = 24 * 60 * 60 * 1000
|
||||
const ANCHOR_MS = 1_800_000_000_000
|
||||
const roots: string[] = []
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) {
|
||||
try {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
} catch {
|
||||
// 测试收尾尽力而为。
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
function makeDatabaseRoot(): string {
|
||||
const root = mkdtempSync(join(tmpdir(), 'tm-recent-first-'))
|
||||
roots.push(root)
|
||||
return root
|
||||
}
|
||||
|
||||
function imageMessage(localId: number, createTimeSeconds: number): chat.FormattedMessage {
|
||||
return {
|
||||
localId: String(localId),
|
||||
createTime: createTimeSeconds,
|
||||
content: '[图片]',
|
||||
contentData: { type: 'image', md5: `md5-${localId}`, datName: `dat-${localId}` }
|
||||
} as unknown as chat.FormattedMessage
|
||||
}
|
||||
|
||||
type Window = { sinceMs?: number; beforeMs?: number }
|
||||
|
||||
interface HarnessState {
|
||||
messages: chat.FormattedMessage[]
|
||||
count: number
|
||||
maxLocalId: number
|
||||
now: number
|
||||
}
|
||||
|
||||
interface HarnessResult {
|
||||
service: ImageTextIndexService
|
||||
databasePath: string
|
||||
state: HarnessState
|
||||
/** 每次向底层请求图片消息时记录的**时间边界**(不含分页上限)。 */
|
||||
windows: Window[]
|
||||
listImageMessages: ReturnType<typeof vi.fn>
|
||||
/** 真正"过了一遍处理"的图片数 —— 用它证明断点续跑没有重做。 */
|
||||
findImageFile: ReturnType<typeof vi.fn>
|
||||
}
|
||||
|
||||
function inWindow(message: chat.FormattedMessage, window?: Window): boolean {
|
||||
const createTimeMs = (message.createTime || 0) * 1000
|
||||
if (window?.sinceMs !== undefined && createTimeMs < window.sinceMs) return false
|
||||
if (window?.beforeMs !== undefined && createTimeMs >= window.beforeMs) return false
|
||||
return true
|
||||
}
|
||||
|
||||
function makeHarness(options: {
|
||||
messages: chat.FormattedMessage[]
|
||||
onWindow?: (window: Window | undefined, callIndex: number) => void
|
||||
}): HarnessResult {
|
||||
const databaseRoot = makeDatabaseRoot()
|
||||
const databasePath = getImageTextIndexDatabasePath(databaseRoot, ACCOUNT)
|
||||
const state = {
|
||||
messages: [...options.messages],
|
||||
count: options.messages.length,
|
||||
maxLocalId: options.messages.reduce((max, m) => Math.max(max, Number(m.localId) || 0), 0),
|
||||
now: ANCHOR_MS
|
||||
}
|
||||
const windows: Window[] = []
|
||||
const findImageFile = vi.fn(() => null)
|
||||
|
||||
const listImageMessages = vi.fn(async (_conversationId: string, window?: Window) => {
|
||||
// 只记时间边界:调用方还会带一个分页上限,那与"读了哪个时间窗"无关。
|
||||
const bounds: Window = {}
|
||||
if (window?.sinceMs !== undefined) bounds.sinceMs = window.sinceMs
|
||||
if (window?.beforeMs !== undefined) bounds.beforeMs = window.beforeMs
|
||||
windows.push(bounds)
|
||||
options.onWindow?.(window, windows.length - 1)
|
||||
return state.messages.filter((message) => inWindow(message, window))
|
||||
})
|
||||
|
||||
const service = new ImageTextIndexService()
|
||||
service.bind({
|
||||
databaseRoot,
|
||||
resolveAccountId: () => ACCOUNT,
|
||||
resolveAccountRoot: () => 'C:/fixture/account',
|
||||
now: () => state.now,
|
||||
listContacts: async () => [
|
||||
{ md5: CONVERSATION, m_nsUsrName: 'fixture', type: 'group' as const }
|
||||
],
|
||||
listImageMessages,
|
||||
// 窗口感知的计数:新架构"这段没图片就整段跳过"完全依赖它说实话。
|
||||
countConversationImages: async (_conversationId: string, window?: Window) => ({
|
||||
count: state.messages.filter((message) => inWindow(message, window)).length,
|
||||
typeColumn: 'local_type'
|
||||
}),
|
||||
imageWatermark: async () => ({ count: state.count, maxLocalId: state.maxLocalId }),
|
||||
// 没有解密能力 → 每张图片都会被判成 image_missing,测试完全不碰真实图片。
|
||||
decryptService: () => ({ findImageFile, decryptImage: () => null }) as never,
|
||||
capability: async () => ({
|
||||
available: true,
|
||||
engine: 'windows-system-ocr',
|
||||
platform: 'win32',
|
||||
runtimeVersion: '1.2.0',
|
||||
language: 'zh-Hans-CN'
|
||||
}),
|
||||
recognize: async () => ({ success: true, text: '', language: 'zh-Hans-CN' })
|
||||
})
|
||||
|
||||
return { service, databasePath, state, windows, listImageMessages, findImageFile }
|
||||
}
|
||||
|
||||
const bindingIds = (databasePath: string): string[] => {
|
||||
const store = new ImageTextIndexStore(databasePath, ACCOUNT)
|
||||
const ids = [...store.getConversationOcr(CONVERSATION).keys()]
|
||||
store.close()
|
||||
return ids
|
||||
}
|
||||
|
||||
const tierStates = (
|
||||
databasePath: string
|
||||
): Partial<Record<ImageTextBackfillTier, ImageTextTierRunState>> => {
|
||||
const store = new ImageTextIndexStore(databasePath, ACCOUNT)
|
||||
const result = store.readBackfillState()
|
||||
store.close()
|
||||
return result.tierStates
|
||||
}
|
||||
|
||||
const runPass = async (
|
||||
harness: HarnessResult,
|
||||
options?: Parameters<ImageTextIndexService['startPass']>[0]
|
||||
): Promise<void> => {
|
||||
harness.service.startPass(options)
|
||||
await vi.waitFor(() => expect(harness.service.isRunning()).toBe(false), { timeout: 15_000 })
|
||||
}
|
||||
|
||||
/** 合成数据集:今天 / 3 天前 / 15 天前 / 6 个月前 / 3 年前,各 10 张。 */
|
||||
function syntheticDataset(anchorMs: number): {
|
||||
messages: chat.FormattedMessage[]
|
||||
byBucket: Map<string, number[]>
|
||||
} {
|
||||
const buckets: Array<{ key: string; offsetDays: number }> = [
|
||||
{ key: 'today', offsetDays: 0 },
|
||||
{ key: '3d', offsetDays: 3 },
|
||||
{ key: '15d', offsetDays: 15 },
|
||||
{ key: '6mo', offsetDays: 183 },
|
||||
{ key: '3y', offsetDays: 1095 }
|
||||
]
|
||||
const messages: chat.FormattedMessage[] = []
|
||||
const byBucket = new Map<string, number[]>()
|
||||
let localId = 1
|
||||
for (const bucket of buckets) {
|
||||
const ids: number[] = []
|
||||
const baseSeconds = Math.floor((anchorMs - bucket.offsetDays * DAY_MS) / 1000) - 60
|
||||
for (let index = 0; index < 10; index += 1) {
|
||||
messages.push(imageMessage(localId, baseSeconds + index))
|
||||
ids.push(localId)
|
||||
localId += 1
|
||||
}
|
||||
byBucket.set(bucket.key, ids)
|
||||
}
|
||||
return { messages, byBucket }
|
||||
}
|
||||
|
||||
describe('分段计划:唯一边界、不重不漏', () => {
|
||||
it('四段从新到旧、相邻且互不重叠', () => {
|
||||
const segments = buildImageTextBackfillSegments(ANCHOR_MS)
|
||||
expect(segments.map((segment) => segment.tier)).toEqual([
|
||||
'recent_7d',
|
||||
'recent_30d',
|
||||
'recent_1y',
|
||||
'archive'
|
||||
])
|
||||
expect(segments[0].endMs).toBe(ANCHOR_MS)
|
||||
for (let index = 1; index < segments.length; index += 1) {
|
||||
// 上一段的上界必须等于下一段的下界:半开区间才既不重叠也不留缝。
|
||||
expect(segments[index].endMs).toBe(segments[index - 1].startMs)
|
||||
}
|
||||
expect(segments[2].startMs).toBe(
|
||||
ANCHOR_MS - IMAGE_TEXT_BACKFILL_WINDOW_DAYS.recent_1y * DAY_MS
|
||||
)
|
||||
// 归档段没有下界。
|
||||
expect(segments[3].startMs).toBe(Number.NEGATIVE_INFINITY)
|
||||
})
|
||||
|
||||
it('边界那一秒只属于一段(两端同一套取整)', () => {
|
||||
const boundaryMs = ANCHOR_MS - 7 * DAY_MS
|
||||
const { beforeSecInclusive } = imageTextWindowToSeconds({ beforeMs: boundaryMs })
|
||||
const { sinceSec } = imageTextWindowToSeconds({ sinceMs: boundaryMs })
|
||||
// 上界段的"含"到 beforeSecInclusive,下界段从 sinceSec 起 —— 必须正好衔接。
|
||||
expect(sinceSec! - beforeSecInclusive!).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('recent-first:合成数据集的真实处理顺序', () => {
|
||||
it('今天 / 3 天 → 15 天 → 6 个月前 → 3 年前,先新后旧', async () => {
|
||||
const harness = makeHarness({ messages: [] })
|
||||
const { messages, byBucket } = syntheticDataset(ANCHOR_MS)
|
||||
harness.state.messages = messages
|
||||
harness.state.count = messages.length
|
||||
harness.state.maxLocalId = Math.max(...messages.map((m) => Number(m.localId)))
|
||||
|
||||
await runPass(harness)
|
||||
|
||||
// 只有"有图片的分段"才会去读消息:今天+3 天 / 15 天 / 6 个月 / 3 年,共 4 次。
|
||||
expect(harness.windows).toHaveLength(4)
|
||||
const [first, second, third, fourth] = harness.windows
|
||||
expect(first).toEqual({ sinceMs: ANCHOR_MS - 7 * DAY_MS, beforeMs: ANCHOR_MS })
|
||||
expect(second).toEqual({ sinceMs: ANCHOR_MS - 30 * DAY_MS, beforeMs: ANCHOR_MS - 7 * DAY_MS })
|
||||
expect(third).toEqual({ sinceMs: ANCHOR_MS - 365 * DAY_MS, beforeMs: ANCHOR_MS - 30 * DAY_MS })
|
||||
expect(fourth).toEqual({ sinceMs: Number.NEGATIVE_INFINITY, beforeMs: ANCHOR_MS - 365 * DAY_MS })
|
||||
|
||||
// 第一次读就拿到了"最近的图"—— 这正是用户要的价值。
|
||||
const ids = bindingIds(harness.databasePath)
|
||||
expect(ids).toHaveLength(50)
|
||||
const recentIds = [...byBucket.get('today')!, ...byBucket.get('3d')!]
|
||||
for (const id of recentIds) expect(ids).toContain(sourceMessageId(imageMessage(id, 0)))
|
||||
expect(Math.max(...byBucket.get('today')!)).toBeLessThan(Math.min(...byBucket.get('3y')!))
|
||||
for (const id of byBucket.get('3y')!) {
|
||||
expect(ids).toContain(sourceMessageId(imageMessage(id, 0)))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('断点续跑:不重复 OCR', () => {
|
||||
it('第一批被预算截断后,重启只补剩下的,已完成的不再走一遍', async () => {
|
||||
const harness = makeHarness({ messages: [] })
|
||||
const { messages } = syntheticDataset(ANCHOR_MS)
|
||||
harness.state.messages = messages
|
||||
harness.state.count = messages.length
|
||||
harness.state.maxLocalId = Math.max(...messages.map((m) => Number(m.localId)))
|
||||
|
||||
// 第一轮:只允许处理 6 张 → 必然停在"最近图片"这一段中间。
|
||||
await runPass(harness, { messageLimit: 6 })
|
||||
expect(harness.findImageFile).toHaveBeenCalledTimes(6)
|
||||
expect(harness.windows).toHaveLength(1)
|
||||
let states = tierStates(harness.databasePath)
|
||||
// 该分段没跑完 → 不允许被标成 complete(否则剩下的图永远不会被处理)。
|
||||
expect(states.recent_7d).not.toBe('complete')
|
||||
|
||||
// 第二轮:预算放开 → 先把最近这段的剩余 4 张补完,再继续往下。
|
||||
await runPass(harness)
|
||||
expect(harness.findImageFile).toHaveBeenCalledTimes(50)
|
||||
states = tierStates(harness.databasePath)
|
||||
expect(states.recent_7d).toBe('complete')
|
||||
expect(states.archive).toBe('complete')
|
||||
expect(bindingIds(harness.databasePath)).toHaveLength(50)
|
||||
})
|
||||
})
|
||||
|
||||
describe('增量优先:历史分段跑着的时候新图片先处理', () => {
|
||||
it('新图片在下一次调度点被处理,而不是排到历史之后', async () => {
|
||||
let injected = false
|
||||
const harness = makeHarness({
|
||||
messages: [],
|
||||
onWindow: (window) => {
|
||||
// 第二轮窗口请求(recent_30d)之后,模拟"用户刚收到一张新图"。
|
||||
if (injected || window?.sinceMs === undefined) return
|
||||
if (window.beforeMs === ANCHOR_MS - 7 * DAY_MS) {
|
||||
injected = true
|
||||
harness.state.messages = [
|
||||
...harness.state.messages,
|
||||
imageMessage(999, Math.floor((ANCHOR_MS + 5_000) / 1000))
|
||||
]
|
||||
harness.state.count += 1
|
||||
harness.state.maxLocalId = 999
|
||||
harness.state.now = ANCHOR_MS + 5_000
|
||||
}
|
||||
}
|
||||
})
|
||||
const { messages } = syntheticDataset(ANCHOR_MS)
|
||||
harness.state.messages = messages
|
||||
harness.state.count = messages.length
|
||||
harness.state.maxLocalId = Math.max(...messages.map((m) => Number(m.localId)))
|
||||
|
||||
await runPass(harness)
|
||||
|
||||
const newMessageId = sourceMessageId(imageMessage(999, 0))
|
||||
expect(bindingIds(harness.databasePath)).toContain(newMessageId)
|
||||
/**
|
||||
* 新图片必须在"更老的分段"之前被处理。
|
||||
*
|
||||
* 增量补齐对"插入序涨了"的会话是**不设时间窗**读取的(这样才接得住晚到的旧时间消息),
|
||||
* 所以这里找的是"没有任何时间边界的那个窗口",它必须早于 1 年 / 归档分段。
|
||||
*/
|
||||
const sweepWindowIndex = harness.windows.findIndex(
|
||||
(window) => window.sinceMs === undefined && window.beforeMs === undefined
|
||||
)
|
||||
const yearWindowIndex = harness.windows.findIndex(
|
||||
(window) => window?.beforeMs === ANCHOR_MS - 30 * DAY_MS
|
||||
)
|
||||
expect(sweepWindowIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(yearWindowIndex).toBeGreaterThan(sweepWindowIndex)
|
||||
})
|
||||
})
|
||||
|
||||
describe('兼容性:老索引不被降级,源侧有新增必须补上', () => {
|
||||
it('老版本已经全量建完且源侧无变化 → 保持完成,一个字都不读', async () => {
|
||||
const harness = makeHarness({ messages: [] })
|
||||
// 造一份"旧版本建完"的库:总数 3、三条 binding 全部 terminal、没有任何分段元数据。
|
||||
const store = new ImageTextIndexStore(harness.databasePath, ACCOUNT)
|
||||
store.writeCountedTotal({ total: 3, countedAt: ANCHOR_MS - 1000, complete: true })
|
||||
// 旧版本每跑完一个会话都会留下 scan_state —— 升级后正是靠它证明源侧没动过。
|
||||
store.writeScanState({
|
||||
conversationId: CONVERSATION,
|
||||
state: 'done',
|
||||
imageTotal: 3,
|
||||
imageProcessed: 3,
|
||||
maxLocalId: 3
|
||||
})
|
||||
for (let index = 1; index <= 3; index += 1) {
|
||||
store.putBinding({
|
||||
accountId: ACCOUNT,
|
||||
conversationId: CONVERSATION,
|
||||
messageId: `local:${index}`,
|
||||
createTime: ANCHOR_MS - 1000,
|
||||
imageIdentity: `sha256:${index}`,
|
||||
artifactKey: `sha256:${index}|legacy`,
|
||||
state: 'indexed',
|
||||
updatedAt: 1
|
||||
})
|
||||
}
|
||||
store.close()
|
||||
|
||||
harness.state.messages = [imageMessage(1, 1000), imageMessage(2, 2000), imageMessage(3, 3000)]
|
||||
harness.state.count = 3
|
||||
harness.state.maxLocalId = 3
|
||||
|
||||
await runPass(harness)
|
||||
|
||||
// 源侧水位一致 → 一个字都不该读:用户已经拥有的东西不能被降级成"重新建立"。
|
||||
expect(harness.listImageMessages).not.toHaveBeenCalled()
|
||||
expect(harness.findImageFile).not.toHaveBeenCalled()
|
||||
const states = tierStates(harness.databasePath)
|
||||
expect(states.recent_7d).toBe('complete')
|
||||
expect(states.archive).toBe('complete')
|
||||
})
|
||||
|
||||
it('老版本自称"已建完"但源侧已新增 → 必须补上新增,不能信库里的进度', async () => {
|
||||
const harness = makeHarness({ messages: [] })
|
||||
const store = new ImageTextIndexStore(harness.databasePath, ACCOUNT)
|
||||
// 库里自称 2/2 已完成(旧版本跑完时的状态)。
|
||||
store.writeCountedTotal({ total: 2, countedAt: ANCHOR_MS - 1000, complete: true })
|
||||
for (const localId of [1, 2]) {
|
||||
store.writeScanState({
|
||||
conversationId: CONVERSATION,
|
||||
state: 'done',
|
||||
imageTotal: 2,
|
||||
imageProcessed: 2,
|
||||
maxLocalId: 2
|
||||
})
|
||||
store.putBinding({
|
||||
accountId: ACCOUNT,
|
||||
conversationId: CONVERSATION,
|
||||
messageId: `local:${localId}`,
|
||||
createTime: ANCHOR_MS - 1000,
|
||||
imageIdentity: `sha256:old-${localId}`,
|
||||
artifactKey: `sha256:old-${localId}|legacy`,
|
||||
state: 'indexed',
|
||||
updatedAt: 1
|
||||
})
|
||||
}
|
||||
store.close()
|
||||
|
||||
// 源侧现在是 3 张(插入序 1..3)—— 只看库里自称的进度会把它误判成"已完成"。
|
||||
harness.state.messages = [imageMessage(1, 1000), imageMessage(2, 2000), imageMessage(3, 3000)]
|
||||
harness.state.count = 3
|
||||
harness.state.maxLocalId = 3
|
||||
|
||||
await runPass(harness)
|
||||
|
||||
const ids = bindingIds(harness.databasePath)
|
||||
expect(ids).toHaveLength(3)
|
||||
expect(ids).toContain(sourceMessageId(imageMessage(3, 3000)))
|
||||
// 老的 terminal 结果必须被复用,不能重新 OCR。
|
||||
expect(harness.findImageFile).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('时间范围覆盖度', () => {
|
||||
const coverageOf = (input: Partial<ImageTextIndexCoverage>): ImageTextIndexCoverage => ({
|
||||
totalImageMessages: 100,
|
||||
processed: 20,
|
||||
indexed: 18,
|
||||
empty: 2,
|
||||
missing: 0,
|
||||
failed: 0,
|
||||
runtimeUnavailable: 0,
|
||||
pending: 80,
|
||||
established: true,
|
||||
complete: false,
|
||||
systemicFailure: false,
|
||||
countedAt: ANCHOR_MS,
|
||||
tiers: [
|
||||
{ tier: 'recent_7d', state: 'complete', startMs: ANCHOR_MS - 7 * DAY_MS, endMs: ANCHOR_MS },
|
||||
{
|
||||
tier: 'recent_30d',
|
||||
state: 'pending',
|
||||
startMs: ANCHOR_MS - 30 * DAY_MS,
|
||||
endMs: ANCHOR_MS - 7 * DAY_MS
|
||||
},
|
||||
{
|
||||
tier: 'recent_1y',
|
||||
state: 'pending',
|
||||
startMs: ANCHOR_MS - 365 * DAY_MS,
|
||||
endMs: ANCHOR_MS - 30 * DAY_MS
|
||||
},
|
||||
{
|
||||
tier: 'archive',
|
||||
state: 'pending',
|
||||
startMs: Number.NEGATIVE_INFINITY,
|
||||
endMs: ANCHOR_MS - 365 * DAY_MS
|
||||
}
|
||||
],
|
||||
coveredToMs: ANCHOR_MS,
|
||||
...input
|
||||
})
|
||||
|
||||
it('A. 最近 7 天已完成 → 查询最近 24h 可以下确定性结论', () => {
|
||||
const coverage = coverageOf({})
|
||||
const range = { sinceMs: ANCHOR_MS - DAY_MS, beforeMs: ANCHOR_MS }
|
||||
const result = imageTextRangeCoverage(coverage, range)
|
||||
expect(result.state).toBe('complete')
|
||||
expect(result.hasUncovered).toBe(false)
|
||||
expect(describeImageTextRangeCoverage(coverage, range)).toContain('已覆盖该时间范围')
|
||||
})
|
||||
|
||||
it('B. 更早历史未完成 → 全历史查询只能是 partial,并明确"不能排除尚未索引的图片"', () => {
|
||||
const coverage = coverageOf({})
|
||||
expect(imageTextRangeCoverage(coverage, {}).state).toBe('partial')
|
||||
expect(describeImageTextRangeCoverage(coverage, {})).toContain('不能排除尚未索引的图片')
|
||||
})
|
||||
|
||||
it('C. 全部完成 → 任意范围都是 complete', () => {
|
||||
const coverage = coverageOf({
|
||||
processed: 100,
|
||||
indexed: 98,
|
||||
empty: 2,
|
||||
pending: 0,
|
||||
complete: true,
|
||||
tiers: coverageOf({}).tiers.map((entry) => ({ ...entry, state: 'complete' as const }))
|
||||
})
|
||||
expect(imageTextRangeCoverage(coverage, {}).state).toBe('complete')
|
||||
expect(
|
||||
imageTextRangeCoverage(coverage, { sinceMs: ANCHOR_MS - 900 * DAY_MS }).state
|
||||
).toBe('complete')
|
||||
})
|
||||
|
||||
it('D. 在更老的历史里被取消 → 最近一段仍可完整,全历史仍是 partial', () => {
|
||||
const coverage = coverageOf({})
|
||||
expect(
|
||||
imageTextRangeCoverage(coverage, {
|
||||
sinceMs: ANCHOR_MS - 3 * DAY_MS,
|
||||
beforeMs: ANCHOR_MS
|
||||
}).state
|
||||
).toBe('complete')
|
||||
expect(imageTextRangeCoverage(coverage, {}).state).toBe('partial')
|
||||
})
|
||||
|
||||
it('未建立时不得声称任何范围完整', () => {
|
||||
const coverage = coverageOf({ established: false, tiers: [], coveredToMs: null })
|
||||
expect(imageTextRangeCoverage(coverage, { sinceMs: ANCHOR_MS - DAY_MS }).state).toBe(
|
||||
'not_built'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('UI 文案与进度', () => {
|
||||
it('阶段文案是用户语言,不出现工程术语', () => {
|
||||
expect(imageTextPhaseLabel('recent_7d')).toBe('正在优先索引最近图片')
|
||||
expect(imageTextPhaseLabel('incremental')).toBe('正在优先索引最近图片')
|
||||
expect(imageTextPhaseLabel('recent_30d')).toBe('正在补齐最近 30 天')
|
||||
expect(imageTextPhaseLabel('recent_1y')).toBe('正在补齐近一年图片')
|
||||
expect(imageTextPhaseLabel('archive')).toBe('正在补齐更早图片')
|
||||
expect(imageTextPhaseLabel('complete')).toBe('图片文字索引已完成')
|
||||
const phases = ['incremental', 'recent_7d', 'recent_30d', 'recent_1y', 'archive', 'complete'] as const
|
||||
for (const phase of phases) {
|
||||
expect(imageTextPhaseLabel(phase)).not.toMatch(/Tier/i)
|
||||
}
|
||||
})
|
||||
|
||||
it('阶段字段在运行中出现,且总进度仍以全量图片数为分母', async () => {
|
||||
const harness = makeHarness({ messages: [] })
|
||||
const { messages } = syntheticDataset(ANCHOR_MS)
|
||||
harness.state.messages = messages
|
||||
harness.state.count = messages.length
|
||||
harness.state.maxLocalId = Math.max(...messages.map((m) => Number(m.localId)))
|
||||
|
||||
await runPass(harness, { messageLimit: 3 })
|
||||
const status = await harness.service.getStatus()
|
||||
// 总分母必须是全部图片消息,而不是"这一段处理了多少"。
|
||||
expect(status.progress.totalImageMessages).toBe(50)
|
||||
expect(status.progress.processed).toBeGreaterThan(0)
|
||||
expect(status.progress.percent).toBeLessThan(100)
|
||||
expect(status.coverage.tiers.map((entry) => entry.tier)).toEqual([
|
||||
'recent_7d',
|
||||
'recent_30d',
|
||||
'recent_1y',
|
||||
'archive'
|
||||
])
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user