/** * Contracts for the local, derived knowledge base. These values deliberately * contain no WCDB handles, Electron objects, database keys, or UI state so the * indexer can run in an isolated process. */ export const KNOWLEDGE_SCHEMA_VERSION = 1 export const DEFAULT_CHUNKER_VERSION = 'conversation-v1' export type KnowledgeMessageKind = 'text' | 'voice' | 'file' | 'link' | 'image' | 'video' | 'sticker' | 'system' | 'other' export type KnowledgeIndexPhase = | 'idle' | 'preflight' | 'indexing' | 'ready' | 'cancelled' | 'error' export type KnowledgeTemporalIntent = 'none' | 'current' | 'historical' | 'timeline' export type KnowledgeFtsTokenizer = 'unicode61' | 'trigram' export type KnowledgeFtsContentMode = 'external' | 'internal' export type KnowledgeFtsDetail = 'full' | 'column' | 'none' export interface KnowledgeAttachmentMetadata { name: string kind?: 'file' | 'link' | 'image' | 'video' | 'other' url?: string sizeBytes?: number } /** A read-only source record prepared by the future WCDB adapter. */ export interface KnowledgeSourceMessage { accountId: string conversationId: string messageId: string /** Unix epoch milliseconds. Adapters must convert source-specific units. */ createTime: number senderId?: string senderName?: string kind: KnowledgeMessageKind text?: string attachment?: KnowledgeAttachmentMetadata voiceTranscript?: string /** Local coverage state only. Error text is never copied into the index. */ voiceTranscriptState?: 'pending' | 'transcribed' | 'failed' /** * 图片里的文字(本地 System OCR 的派生结果)。 * * 与 voiceTranscript 同构:这是 **derived content**,原图片消息仍然是 * authoritative source。它绝不写回 message.body,也绝不产生"OCR 消息"。 */ imageOcrText?: string /** 图片 OCR 的本地状态;与 voiceTranscriptState 一样不含错误正文。 */ imageOcrState?: KnowledgeImageOcrState } /** 图片 OCR 的本地覆盖状态(错误详情绝不进索引)。 */ export type KnowledgeImageOcrState = | 'pending' | 'indexed' | 'empty' | 'metadata_missing' | 'image_missing' | 'decrypt_unavailable' | 'decrypt_failed' | 'decode_failed' | 'ocr_failed' | 'cancelled' export interface KnowledgeNormalizedMessage extends KnowledgeSourceMessage { searchableText: string contentHash: string } export interface KnowledgeChunkerConfig { version: string maxGapMs: number maxMessages: number maxCharacters: number overlapMessages: number } export interface KnowledgeChunk { chunkId: string accountId: string conversationId: string startTime: number endTime: number text: string messageIds: string[] participantIds: string[] messageKinds: KnowledgeMessageKind[] contentHash: string chunkerVersion: string } /** * Every FTS choice is explicit. The first production profile must be selected * from the Task 0 report rather than being silently hard-coded in the UI. */ export interface KnowledgeFtsConfig { profileId: string tokenizer: KnowledgeFtsTokenizer contentMode: KnowledgeFtsContentMode detail: KnowledgeFtsDetail columnsize: 0 | 1 } /** * Chosen after the realistic desensitized WeChat benchmark: trigram preserves * Chinese-substring recall while external content avoids a second text copy. */ export const DEFAULT_KNOWLEDGE_FTS_CONFIG: KnowledgeFtsConfig = { profileId: 'trigram-external-full-columnsize-v1', tokenizer: 'trigram', contentMode: 'external', detail: 'full', columnsize: 1 } export interface KnowledgeConversationInput { conversationId: string /** true means this is a complete read-only snapshot of the conversation. */ completeSnapshot: boolean messages: KnowledgeSourceMessage[] /** * 这个会话在**源侧**(WCDB Session.last_timestamp)已经覆盖到的最后活跃时间(epoch ms)。 * * 与 `KnowledgeIndexRequest.sourceLatestAt` 的区别:后者是整遍 pass 级别的边界,这里是 * per-conversation checkpoint —— 增量 pass 用它判断「这个会话有没有新消息」, * 从而跳过整个会话(不读 WCDB、不传 IPC、不写索引)。 * * 记录**源侧**时间而不是「索引里最后一条可建模消息的时间」:否则最后一条恰好落在 * 图片/空正文上的会话会永远被判定为「有新消息」,增量永远跳不过它。 */ sourceHighWaterTime?: number } export interface KnowledgeIndexRequest { accountId: string databaseRoot: string conversations: KnowledgeConversationInput[] chunker: KnowledgeChunkerConfig fts: KnowledgeFtsConfig /** Written only after a complete source pass; used for truthful coverage. */ sourceMessageCount?: number /** * 这一遍完整 pass 实际扫到的源数据最新消息时间(epoch ms)。 * * 与 `sourceMessageCount` 一样,只在读完全部会话的那一次写入。它是 freshness 的权威口径: * 拿它与当前源数据最新活跃时间比较,就能确定索引是否已经追上,而不必猜测派生索引的过滤落差。 */ sourceLatestAt?: number } export interface KnowledgeIndexProgress { accountId: string phase: KnowledgeIndexPhase conversationId?: string processedMessages: number totalMessages: number indexedChunks: number error?: string } export interface KnowledgeIndexResult { accountId: string processedMessages: number indexedChunks: number updatedChunks: number unchangedConversations: number databaseBytes: number walBytes: number elapsedMs: number cancelled: boolean } export interface KnowledgeCapacityPreflightRequest { accountId: string databaseRoot: string conversations: KnowledgeConversationInput[] chunker: KnowledgeChunkerConfig /** Optional free space supplied by the platform layer; this module never probes WCDB paths. */ availableDiskBytes?: number } export interface KnowledgeCapacityPreflight { accountId: string sourceMessageCount: number indexableMessageCount: number indexableTextBytes: number voiceTranscriptCount: number attachmentMetadataCount: number sampledChunkCount: number estimatedChunkCount: number estimatedDatabaseBytesLow: number estimatedDatabaseBytesHigh: number estimatedBuildPeakBytesLow: number estimatedBuildPeakBytesHigh: number availableDiskBytes?: number hasSufficientDiskSpace?: boolean warnings: string[] } export interface KnowledgeEvidence { chunkId: string conversationId: string startTime: number endTime: number /** Stable source-message identity used by the archive jump action. */ messageId: string senderId?: string sender: string /** Unix epoch milliseconds. */ timestamp: number messageIds: string[] /** The source type belongs to the original message, not the retrieval method. */ sourceKind: KnowledgeMessageKind text: string /** * 这条证据里「从图片里读出来的文字」(本地 System OCR 的派生结果)。 * * 只用于**来源解释**:让用户/模型知道这段内容来自图片,而不是群友真的发了一条文字消息。 * authoritative source 始终是原始图片消息 —— 这里不产生任何"OCR 消息"。 */ imageOcrText?: string /** * 命中所依赖的**派生来源**。 * * 有值 = 这条结果依赖本地派生内容才能命中(而不是原始消息本身的文字)。 * 与 `sourceKind` 正交:`sourceKind` 说的是原始消息是什么,这里说的是"靠什么搜到的"。 */ derivedSource?: 'image_ocr' score?: number } /** * 证据文本面向用户 / 模型时的可读化处理。 * * `searchableText` 里的 `图片文字:` 只是索引期用来区分派生内容的内部标签, * 它**绝不能出现在 Evidence 里**:用户不该看到引擎内部前缀, * 而且"这段文字来自图片"应该由结构化的来源标记表达,而不是靠一个冒号前缀。 */ export function toEvidenceDisplayText(searchableText: string): string { return searchableText .split('\n') .map((line) => line.replace(/^\s*(?:图片文字|OCR|system-ocr)\s*[::]\s*/i, '')) .join('\n') .trim() } export interface KnowledgeVoiceCoverage { voiceMessageCount: number transcribedVoiceCount: number failedVoiceCount: number voiceCoverageComplete: boolean } /** A bounded, local summary of a single conversation retrieval. */ export interface KnowledgeConversationRetrieval { conversationId: string totalMessages: number chunkCount: number candidateMessages: number systemMessagesDeprioritized: number complete: boolean } export interface KnowledgeQuery { accountId: string text: string /** Query-router terms. The raw question remains available for diagnostics. */ terms?: string[] limit: number conversationId?: string conversationIds?: string[] senderIds?: string[] /** Unix epoch milliseconds. */ startTime?: number /** Unix epoch milliseconds. */ endTime?: number temporalIntent?: KnowledgeTemporalIntent conversationBoundary?: 'first' | 'last' } export interface KnowledgeSearchRequest extends KnowledgeQuery { databaseRoot: string fts: KnowledgeFtsConfig } export type KnowledgeSearchState = 'unavailable' | 'indexing' | 'ready' /** Measured in the Worker; never inferred from message counts or UI timers. */ export interface KnowledgeSearchTimings { /** Parent/child-process transport and host scheduling outside SQLite work. */ workerIpcMs: number /** First request only: child process spawn and Node initialization until it received the request. */ workerBootMs: number /** Parent send → Worker handler start. */ dispatchMs: number /** Worker local SQLite/chunk work; equals the Worker-side search total. */ workerSqlMs: number /** Worker response preparation → parent receipt; includes IPC serialization/transfer. */ responseTransferMs: number /** Worker-side serialization preflight for the result payload. */ responseSerializeMs: number /** FTS (or short-term database lookup) query time. */ ftsMs: number /** * `ftsMs` 中**短词回退路径**(<3 字的中文词,走 `LIKE` 而非 FTS `MATCH`)占用的时间。 * * 跨会话 lexical probe 里最常见的 2 字中文词只能走这条路径,没有这个分解就无法区分 * 「FTS MATCH 慢」和「短词全表 LIKE 慢」。 */ shortTermSearchMs?: number /** * `getSearchStatus()`(含统计快照判定)占用的时间。 * 用于验证「搜索热路径不再做全表聚合」这一性能约束没有被回退。 */ statusMs?: number /** Reading source message rows from matching chunks. */ messageLoadMs: number /** Expanding chunk members, scoring terms and per-chunk de-duplication. */ chunkExpandMs: number /** Final result ordering and limit application. */ rankingMs: number /** Worker-side local search total. */ totalMs: number /** Time spent refreshing a stale on-disk statistics snapshot. */ globalCountMs?: number /** Voice coverage aggregation time for this query. */ voiceCoverageMs?: number /** Full Worker handler execution, including status and coverage bookkeeping. */ workerExecutionMs?: number /** Worker queue wait in the main-process host, when observable. */ workerQueueMs?: number /** Main-process WCDB FIFO wait, when observable. */ wcdbQueueMs?: number /** Main-process WCDB operation execution, when observable. */ wcdbExecutionMs?: number /** Main-process sender/contact enrichment duration. */ senderEnrichmentMs?: number /** Main-process IPC/transport duration, when separated from Worker execution. */ ipcMs?: number /** Serialization/encoding duration outside the Worker SQL timer. */ serializationMs?: number /** Other unclassified waiting in the retrieval path. */ otherMs?: number } export const emptyKnowledgeSearchTimings = (): KnowledgeSearchTimings => ({ workerIpcMs: 0, workerBootMs: 0, dispatchMs: 0, workerSqlMs: 0, responseTransferMs: 0, responseSerializeMs: 0, ftsMs: 0, messageLoadMs: 0, chunkExpandMs: 0, rankingMs: 0, totalMs: 0 }) export interface KnowledgeSearchResult { state: KnowledgeSearchState evidence: KnowledgeEvidence[] indexedMessageCount: number indexedChunkCount: number /** * 派生索引里最新的消息时间(epoch ms);null 表示无法判定。 * * `state: 'ready'` 只说明「这个派生库可以被查询」,**不等于**它已经追到源数据最新位置。 * 调用方必须把它与请求的时间范围比较,才能判断本次检索是否覆盖了用户问的时间。 */ indexLatestAt: number | null timings: KnowledgeSearchTimings conversationRetrieval?: KnowledgeConversationRetrieval voiceCoverage?: KnowledgeVoiceCoverage } /** Renderer-facing request. Chat timestamps use Unix seconds in the existing UI. */ export interface KnowledgeSearchIpcRequest { text: string terms: string[] /** Bounded request/session cache key for sender enrichment only. */ retrievalSessionId?: string conversationIds?: string[] senderIds?: string[] startTime?: number endTime?: number conversationBoundary?: 'first' | 'last' limit?: number } export interface KnowledgeSearchIpcResult extends KnowledgeSearchResult { source: 'knowledge' | 'fallback' totalMessages: number fallbackReason?: 'unavailable' | 'indexing' | 'error' /** * 源数据(WCDB Session)里最新的活跃时间(epoch ms);null 表示无法判定。 * * 与 `indexLatestAt` 一起构成 freshness 判据:`sourceLatestAt > indexLatestAt` * 说明源数据里已经有了索引还没覆盖的内容。 */ sourceLatestAt: number | null } export type KnowledgeRuntimeState = 'unavailable' | 'building' | 'syncing' | 'ready' | 'error' | 'cancelled' /** * 一次后台索引 pass 的真实进度(ADDITIVE)。 * * 与新鲜度(`indexLatestAt` / `sourceLatestAt`)回答的是不同问题:这里描述这一遍**在做什么、 * 扫了多少、真正写了多少**。 */ export interface KnowledgePassProgress { /** * 这一遍在做什么: * - `full` 首次全量建立(还没有任何可用分片) * - `catchup` 追最新:已建立 checkpoint 的会话出现了新消息 * - `backfill` 补历史:从来没有 checkpoint 的会话(历史缺口) * - `cancelled` / `error` / `idle` */ phase: 'idle' | 'full' | 'catchup' | 'backfill' | 'cancelled' | 'error' /** 现在是否可以取消(真实在跑且未收到取消请求时才是 true)。 */ cancellable: boolean /** 开始时间(epoch ms)。 */ startedAt: number /** 这一遍从 WCDB **读取/扫描**的源消息条数(含被可索引性过滤掉的)。 */ scannedMessages: number /** 这一遍真正**进入索引**的源消息条数。 */ indexedMessages: number processedConversations: number totalConversations: number /** 因为「没有任何新消息」而整段跳过的会话数(增量 pass 的核心指标)。 */ skippedConversations: number /** 需要「追最新」的会话数(已建立 checkpoint,且源侧有更新)。 */ catchupConversations: number /** 需要「补历史」的会话数(从来没有 checkpoint,必须整段读)。 */ backfillConversations: number /** 这一遍已经补齐的历史会话数。 */ backfillCompletedConversations: number /** 主线程 event loop 在这一遍期间的最大滞后。用于验证重活没有压在 Main 上。 */ mainLoopLagMs: number } export interface KnowledgeRuntimeStatus { accountId: string state: KnowledgeRuntimeState indexedMessageCount: number indexedChunkCount: number /** Null means this source pass has not yet counted every source message. */ sourceMessageCount: number | null processedMessages: number totalMessages: number | null currentConversationId?: string /** Null is displayed as unavailable rather than a fabricated ETA. */ estimatedRemainingMs: number | null databaseBytes: number walBytes: number shmBytes: number lastError?: string /** * 派生索引里最新的消息时间(epoch ms);null 表示无法判定。 * `state: 'ready'`(READY)与「已追到源数据最新」(FRESH)是两个概念,它就是两者的判据之一。 */ indexLatestAt: number | null /** 源数据(WCDB Session)最新活跃时间(epoch ms);null 表示无法判定。 */ sourceLatestAt: number | null /** * 当前/最近一次后台索引 pass 的真实进度(ADDITIVE)。 * * 与 `indexLatestAt` 是**两个不同维度**:前者回答"这一遍在补什么、进度多少", * 后者回答"索引已经覆盖到源数据的哪个时刻"。UI 不能再用一个「已同步」把两者混为一谈。 */ pass?: KnowledgePassProgress } export interface KnowledgeStatusRequest { accountId: string databaseRoot: string fts: KnowledgeFtsConfig } /** * 索引「已覆盖到的源数据时间」与源数据「当前最新活跃时间」之间允许的固定落差。 * * 两侧都来自消息的 create_time(索引侧 = 完整 pass 扫到的最大 create_time, * 源侧 = Session 行的 last_timestamp),用于吸收 Session 元数据晚于消息落库的漂移。 * main 与 renderer 共用这一份定义,避免 UI 与 Engine 的 freshness 口径漂移。 */ export const KNOWLEDGE_FRESHNESS_TOLERANCE_MS = 60 * 1000 /** * 索引是否已经追到源数据最新(FRESH)。 * * 注意与 `state === 'ready'`(READY:这个派生库**可以被查询**)区分: * READY 不代表 FRESH。任一侧口径缺失时返回 null 表示无法判定。 */ export function isKnowledgeFresh( status: Pick ): boolean | null { if (status.indexLatestAt === null || status.sourceLatestAt === null) return null return status.indexLatestAt + KNOWLEDGE_FRESHNESS_TOLERANCE_MS >= status.sourceLatestAt } export interface KnowledgeWorkerRequest { version: 1 type: 'index' | 'preflight' | 'search' | 'status' | 'remove' | 'cancel' | 'close' | 'highWater' requestId: string /** Parent monotonic wall-clock used only for transport timing. */ sentAt?: number payload: | KnowledgeIndexRequest | KnowledgeCapacityPreflightRequest | KnowledgeSearchRequest | KnowledgeStatusRequest | { accountId: string; databaseRoot: string } | { targetRequestId: string } | Record } export interface KnowledgeWorkerResponse { version: 1 type: 'progress' | 'result' | 'error' requestId: string payload?: | KnowledgeIndexProgress | KnowledgeIndexResult | KnowledgeCapacityPreflight | KnowledgeSearchResult | KnowledgeRuntimeStatus | { removed: true } error?: string transport?: { /** IPC message arrival timestamp in the Worker event loop. */ messageReceivedAt?: number workerReceivedAt: number workerCompletedAt: number responseSerializeMs: number workerQueueMs?: number } } export const DEFAULT_KNOWLEDGE_CHUNKER: KnowledgeChunkerConfig = { version: DEFAULT_CHUNKER_VERSION, maxGapMs: 10 * 60 * 1000, maxMessages: 12, maxCharacters: 1200, overlapMessages: 3 }