mirror of
https://wget.la/https://github.com/Wxw-Gu/WechatExplorer
synced 2026-08-21 13:36:58 +08:00
feat: 语音
This commit is contained in:
电摇小子
@@ -0,0 +1,108 @@
|
||||
import { app } from 'electron'
|
||||
import { existsSync } from 'fs'
|
||||
import { createRequire } from 'module'
|
||||
import { join } from 'path'
|
||||
import { isPackagedRuntime } from '../runtime-mode'
|
||||
|
||||
const nodeRequire = createRequire(import.meta.url)
|
||||
|
||||
export interface EncodedVoiceSource {
|
||||
data: Buffer
|
||||
codec: string
|
||||
sourceHash: string
|
||||
}
|
||||
|
||||
export interface DecodedVoiceAudio {
|
||||
pcm: Buffer
|
||||
sampleRate: number
|
||||
channels: number
|
||||
sourceHash: string
|
||||
}
|
||||
|
||||
export interface VoiceAudioDecoder {
|
||||
readonly codec: string
|
||||
decode(source: EncodedVoiceSource): Promise<DecodedVoiceAudio>
|
||||
}
|
||||
|
||||
export type SilkWasmRuntimeLocation = {
|
||||
packagePath: string
|
||||
wasmPath: string
|
||||
source: 'unpacked' | 'resources' | 'asar' | 'development'
|
||||
}
|
||||
|
||||
export function getSilkWasmRuntimeLocations(options?: {
|
||||
packaged?: boolean
|
||||
resourcesPath?: string
|
||||
appPath?: string
|
||||
}): SilkWasmRuntimeLocation[] {
|
||||
const packaged = options?.packaged ?? isPackagedRuntime()
|
||||
const resourcesPath = options?.resourcesPath ?? process.resourcesPath
|
||||
const appPath = options?.appPath ?? app.getAppPath()
|
||||
const location = (
|
||||
packagePath: string,
|
||||
source: SilkWasmRuntimeLocation['source']
|
||||
): SilkWasmRuntimeLocation => ({
|
||||
packagePath,
|
||||
wasmPath: join(packagePath, 'lib', 'silk.wasm'),
|
||||
source
|
||||
})
|
||||
|
||||
if (!packaged) {
|
||||
return [location(join(appPath, 'node_modules', 'silk-wasm'), 'development')]
|
||||
}
|
||||
return [
|
||||
location(join(resourcesPath, 'app.asar.unpacked', 'node_modules', 'silk-wasm'), 'unpacked'),
|
||||
location(join(resourcesPath, 'node_modules', 'silk-wasm'), 'resources'),
|
||||
location(join(appPath, 'node_modules', 'silk-wasm'), 'asar')
|
||||
]
|
||||
}
|
||||
|
||||
export function findSilkWasmRuntimeLocation(
|
||||
locations: SilkWasmRuntimeLocation[]
|
||||
): SilkWasmRuntimeLocation | null {
|
||||
return locations.find((location) => existsSync(location.wasmPath)) || null
|
||||
}
|
||||
|
||||
export class SilkAudioDecoder implements VoiceAudioDecoder {
|
||||
readonly codec = 'silk'
|
||||
|
||||
async decode(source: EncodedVoiceSource): Promise<DecodedVoiceAudio> {
|
||||
const locations = getSilkWasmRuntimeLocations()
|
||||
const runtime = findSilkWasmRuntimeLocation(locations)
|
||||
if (!runtime) throw new Error('silk.wasm 未找到')
|
||||
const silkWasm = nodeRequire(runtime.packagePath) as {
|
||||
decode?: (data: Buffer, sampleRate: number) => Promise<{ data: Uint8Array }>
|
||||
}
|
||||
if (!silkWasm.decode) throw new Error('silk-wasm 运行时无效')
|
||||
const result = await silkWasm.decode(source.data, 24000)
|
||||
const pcm = Buffer.from(result.data)
|
||||
if (!pcm.length) throw new Error('Silk 解码结果为空')
|
||||
return {
|
||||
pcm,
|
||||
sampleRate: 24000,
|
||||
channels: 1,
|
||||
sourceHash: source.sourceHash
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class AudioDecoderRegistry {
|
||||
private readonly decoders = new Map<string, VoiceAudioDecoder>()
|
||||
|
||||
register(decoder: VoiceAudioDecoder): this {
|
||||
if (this.decoders.has(decoder.codec))
|
||||
throw new Error(`Decoder already registered: ${decoder.codec}`)
|
||||
this.decoders.set(decoder.codec, decoder)
|
||||
return this
|
||||
}
|
||||
|
||||
decode(source: EncodedVoiceSource): Promise<DecodedVoiceAudio> {
|
||||
const decoder = this.decoders.get(source.codec)
|
||||
if (!decoder) throw new Error(`Unsupported voice codec: ${source.codec}`)
|
||||
return decoder.decode(source)
|
||||
}
|
||||
}
|
||||
|
||||
export function createDefaultAudioDecoderRegistry(): AudioDecoderRegistry {
|
||||
return new AudioDecoderRegistry().register(new SilkAudioDecoder())
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { AudioProcessor, PipelineAudio } from './types'
|
||||
|
||||
export const VOICE_PROCESSOR_VERSION = 'pcm16-mono-16k-v1'
|
||||
|
||||
export interface PcmProcessorOptions {
|
||||
targetSampleRate?: number
|
||||
silenceThreshold?: number
|
||||
silencePaddingMs?: number
|
||||
normalizePeak?: number
|
||||
}
|
||||
|
||||
export class PcmAudioProcessor implements AudioProcessor {
|
||||
private readonly targetSampleRate: number
|
||||
private readonly silenceThreshold: number
|
||||
private readonly silencePaddingMs: number
|
||||
private readonly normalizePeak: number
|
||||
|
||||
constructor(options: PcmProcessorOptions = {}) {
|
||||
this.targetSampleRate = options.targetSampleRate ?? 16000
|
||||
this.silenceThreshold = options.silenceThreshold ?? 0.008
|
||||
this.silencePaddingMs = options.silencePaddingMs ?? 80
|
||||
this.normalizePeak = options.normalizePeak ?? 0.92
|
||||
}
|
||||
|
||||
process(input: {
|
||||
pcm: Buffer
|
||||
sampleRate: number
|
||||
channels: number
|
||||
sourceHash: string
|
||||
}): PipelineAudio {
|
||||
if (input.channels !== 1) throw new Error('Only mono PCM is supported')
|
||||
if (input.pcm.length < 2) throw new Error('PCM audio is empty')
|
||||
|
||||
const decoded = this.decodePcm16(input.pcm)
|
||||
const trimmed = this.trimSilence(decoded, input.sampleRate)
|
||||
const resampled = this.resample(trimmed, input.sampleRate, this.targetSampleRate)
|
||||
const normalized = this.normalize(resampled)
|
||||
|
||||
return {
|
||||
samples: normalized,
|
||||
sampleRate: this.targetSampleRate,
|
||||
channels: 1,
|
||||
sourceHash: input.sourceHash,
|
||||
processorVersion: VOICE_PROCESSOR_VERSION,
|
||||
durationMs: Math.round((normalized.length / this.targetSampleRate) * 1000)
|
||||
}
|
||||
}
|
||||
|
||||
private decodePcm16(buffer: Buffer): Float32Array {
|
||||
const output = new Float32Array(Math.floor(buffer.length / 2))
|
||||
for (let index = 0; index < output.length; index += 1) {
|
||||
output[index] = buffer.readInt16LE(index * 2) / 32768
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
private trimSilence(samples: Float32Array, sampleRate: number): Float32Array {
|
||||
let first = 0
|
||||
while (first < samples.length && Math.abs(samples[first]) < this.silenceThreshold) first += 1
|
||||
if (first === samples.length) return new Float32Array(0)
|
||||
|
||||
let last = samples.length - 1
|
||||
while (last > first && Math.abs(samples[last]) < this.silenceThreshold) last -= 1
|
||||
const padding = Math.round((sampleRate * this.silencePaddingMs) / 1000)
|
||||
return samples.slice(Math.max(0, first - padding), Math.min(samples.length, last + padding + 1))
|
||||
}
|
||||
|
||||
private resample(samples: Float32Array, sourceRate: number, targetRate: number): Float32Array {
|
||||
if (sourceRate === targetRate || samples.length === 0) return samples.slice()
|
||||
const outputLength = Math.max(1, Math.round((samples.length * targetRate) / sourceRate))
|
||||
const output = new Float32Array(outputLength)
|
||||
const ratio = sourceRate / targetRate
|
||||
for (let index = 0; index < outputLength; index += 1) {
|
||||
const position = index * ratio
|
||||
const left = Math.min(samples.length - 1, Math.floor(position))
|
||||
const right = Math.min(samples.length - 1, left + 1)
|
||||
const fraction = position - left
|
||||
output[index] = samples[left] + (samples[right] - samples[left]) * fraction
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
private normalize(samples: Float32Array): Float32Array {
|
||||
let peak = 0
|
||||
for (const sample of samples) peak = Math.max(peak, Math.abs(sample))
|
||||
if (peak < 0.001 || peak <= this.normalizePeak) return samples
|
||||
const scale = this.normalizePeak / peak
|
||||
return samples.map((sample) => sample * scale)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
import { createHash } from 'crypto'
|
||||
import { createReadStream } from 'fs'
|
||||
import { mkdir, open, readFile, rename, rm, stat, writeFile } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import type { VoiceModelDownloadResult, VoiceModelStatus } from '../../shared/voice-recognition'
|
||||
import { DEFAULT_VOICE_MODEL_ID } from '../../shared/voice-recognition'
|
||||
|
||||
const MODEL_VERSION = '2024-07-17'
|
||||
// SHA-256 values come from the repository's Git LFS object IDs. Hugging Face's
|
||||
// xetHash is a storage-level hash and does not match the downloaded file bytes.
|
||||
export const SENSEVOICE_MODEL_FILES = [
|
||||
{
|
||||
name: 'model.int8.onnx',
|
||||
size: 239_233_841,
|
||||
sha256: 'c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51',
|
||||
url: 'https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/model.int8.onnx'
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
size: 315_894,
|
||||
sha256: 'f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc',
|
||||
url: 'https://huggingface.co/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/tokens.txt'
|
||||
}
|
||||
] as const
|
||||
|
||||
const TOTAL_BYTES = SENSEVOICE_MODEL_FILES.reduce((total, file) => total + file.size, 0)
|
||||
const MODEL_FINGERPRINT = createHash('sha256')
|
||||
.update(SENSEVOICE_MODEL_FILES.map((file) => `${file.name}:${file.sha256}`).join('|'))
|
||||
.digest('hex')
|
||||
|
||||
interface VerifiedManifest {
|
||||
modelId: string
|
||||
version: string
|
||||
fingerprint: string
|
||||
files: Record<string, { size: number; sha256: string }>
|
||||
}
|
||||
|
||||
export interface VoiceModelPaths {
|
||||
model: string
|
||||
tokens: string
|
||||
}
|
||||
|
||||
export class VoiceModelManager {
|
||||
readonly modelId = DEFAULT_VOICE_MODEL_ID
|
||||
readonly version = MODEL_VERSION
|
||||
readonly fingerprint = MODEL_FINGERPRINT
|
||||
private readonly modelRoot: string
|
||||
private downloadController: AbortController | null = null
|
||||
private downloadPromise: Promise<VoiceModelDownloadResult> | null = null
|
||||
private progressBytes = 0
|
||||
private lastProgressAt = 0
|
||||
private progressListener: ((status: VoiceModelStatus) => void) | null = null
|
||||
|
||||
constructor(modelRoot: string) {
|
||||
this.modelRoot = modelRoot
|
||||
}
|
||||
|
||||
get directory(): string {
|
||||
return this.modelRoot
|
||||
}
|
||||
|
||||
setProgressListener(listener: ((status: VoiceModelStatus) => void) | null): void {
|
||||
this.progressListener = listener
|
||||
}
|
||||
|
||||
async getStatus(): Promise<VoiceModelStatus> {
|
||||
if (!this.isRuntimeSupported()) {
|
||||
return this.buildStatus(
|
||||
'unsupported',
|
||||
0,
|
||||
`当前系统暂不支持离线语音识别：${process.platform} ${process.arch}`
|
||||
)
|
||||
}
|
||||
if (this.downloadPromise) return this.buildStatus('downloading', this.progressBytes)
|
||||
const verified = await this.isVerified()
|
||||
if (verified) return this.buildStatus('ready', TOTAL_BYTES)
|
||||
const hasFiles = await this.hasAnyModelFile()
|
||||
return this.buildStatus(
|
||||
hasFiles ? 'invalid' : 'missing',
|
||||
0,
|
||||
hasFiles ? '模型文件不完整或校验失败，请重新下载' : undefined
|
||||
)
|
||||
}
|
||||
|
||||
async getPaths(): Promise<VoiceModelPaths | null> {
|
||||
if (!(await this.isVerified())) return null
|
||||
return {
|
||||
model: join(this.modelRoot, SENSEVOICE_MODEL_FILES[0].name),
|
||||
tokens: join(this.modelRoot, SENSEVOICE_MODEL_FILES[1].name)
|
||||
}
|
||||
}
|
||||
|
||||
download(): Promise<VoiceModelDownloadResult> {
|
||||
if (!this.isRuntimeSupported()) {
|
||||
const status = this.buildStatus(
|
||||
'unsupported',
|
||||
0,
|
||||
`当前系统暂不支持离线语音识别：${process.platform} ${process.arch}`
|
||||
)
|
||||
return Promise.resolve({ success: false, status, error: status.error })
|
||||
}
|
||||
if (this.downloadPromise) return this.downloadPromise
|
||||
this.downloadController = new AbortController()
|
||||
this.progressBytes = 0
|
||||
this.downloadPromise = this.runDownload(this.downloadController.signal).finally(() => {
|
||||
this.downloadPromise = null
|
||||
this.downloadController = null
|
||||
})
|
||||
return this.downloadPromise
|
||||
}
|
||||
|
||||
cancelDownload(): boolean {
|
||||
if (!this.downloadController) return false
|
||||
this.downloadController.abort()
|
||||
return true
|
||||
}
|
||||
|
||||
async remove(): Promise<VoiceModelStatus> {
|
||||
if (this.downloadPromise) return this.buildStatus('downloading', this.progressBytes)
|
||||
await Promise.all([
|
||||
...SENSEVOICE_MODEL_FILES.flatMap((file) => [
|
||||
rm(join(this.modelRoot, file.name), { force: true }),
|
||||
rm(join(this.modelRoot, `${file.name}.partial`), { force: true })
|
||||
]),
|
||||
rm(join(this.modelRoot, 'verified.json'), { force: true }),
|
||||
rm(join(this.modelRoot, 'verified.json.partial'), { force: true })
|
||||
])
|
||||
return this.getStatus()
|
||||
}
|
||||
|
||||
private async runDownload(signal: AbortSignal): Promise<VoiceModelDownloadResult> {
|
||||
try {
|
||||
await mkdir(this.modelRoot, { recursive: true })
|
||||
for (const file of SENSEVOICE_MODEL_FILES) {
|
||||
await this.downloadFile(file, signal)
|
||||
}
|
||||
await this.writeVerifiedManifest()
|
||||
const status = this.buildStatus('ready', TOTAL_BYTES)
|
||||
this.reportProgress(status, true)
|
||||
return { success: true, status }
|
||||
} catch (error) {
|
||||
await Promise.all(
|
||||
SENSEVOICE_MODEL_FILES.map((file) =>
|
||||
rm(join(this.modelRoot, `${file.name}.partial`), { force: true })
|
||||
)
|
||||
)
|
||||
const cancelled = signal.aborted
|
||||
const message = cancelled
|
||||
? '模型下载已取消'
|
||||
: error instanceof Error
|
||||
? error.message
|
||||
: String(error)
|
||||
const status = this.buildStatus(cancelled ? 'missing' : 'error', this.progressBytes, message)
|
||||
this.reportProgress(status, true)
|
||||
return { success: false, status, error: message }
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadFile(
|
||||
file: (typeof SENSEVOICE_MODEL_FILES)[number],
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
const target = join(this.modelRoot, file.name)
|
||||
const partial = `${target}.partial`
|
||||
await rm(partial, { force: true })
|
||||
const response = await fetch(file.url, { signal })
|
||||
if (!response.ok || !response.body) throw new Error(`模型下载失败：HTTP ${response.status}`)
|
||||
|
||||
const handle = await open(partial, 'w')
|
||||
const hash = createHash('sha256')
|
||||
let fileBytes = 0
|
||||
try {
|
||||
const reader = response.body.getReader()
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
if (signal.aborted) throw new DOMException('Download cancelled', 'AbortError')
|
||||
const chunk = Buffer.from(value)
|
||||
await handle.write(chunk)
|
||||
hash.update(chunk)
|
||||
fileBytes += chunk.length
|
||||
this.progressBytes += chunk.length
|
||||
this.reportProgress(this.buildStatus('downloading', this.progressBytes))
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
|
||||
const digest = hash.digest('hex')
|
||||
if (fileBytes !== file.size || digest !== file.sha256) {
|
||||
await rm(partial, { force: true })
|
||||
throw new Error(`模型文件校验失败：${file.name}`)
|
||||
}
|
||||
await rm(target, { force: true })
|
||||
await rename(partial, target)
|
||||
}
|
||||
|
||||
private async isVerified(): Promise<boolean> {
|
||||
try {
|
||||
const manifest = JSON.parse(
|
||||
await readFile(join(this.modelRoot, 'verified.json'), 'utf8')
|
||||
) as VerifiedManifest
|
||||
if (
|
||||
manifest.modelId !== this.modelId ||
|
||||
manifest.version !== this.version ||
|
||||
manifest.fingerprint !== this.fingerprint
|
||||
) {
|
||||
return false
|
||||
}
|
||||
for (const file of SENSEVOICE_MODEL_FILES) {
|
||||
const info = await stat(join(this.modelRoot, file.name))
|
||||
if (info.size !== file.size || manifest.files[file.name]?.sha256 !== file.sha256)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
} catch {
|
||||
return this.verifyExistingFiles()
|
||||
}
|
||||
}
|
||||
|
||||
private async verifyExistingFiles(): Promise<boolean> {
|
||||
try {
|
||||
for (const file of SENSEVOICE_MODEL_FILES) {
|
||||
const path = join(this.modelRoot, file.name)
|
||||
const info = await stat(path)
|
||||
if (info.size !== file.size || (await this.hashFile(path)) !== file.sha256) return false
|
||||
}
|
||||
await this.writeVerifiedManifest()
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private async hasAnyModelFile(): Promise<boolean> {
|
||||
for (const file of SENSEVOICE_MODEL_FILES) {
|
||||
try {
|
||||
await stat(join(this.modelRoot, file.name))
|
||||
return true
|
||||
} catch {
|
||||
// Continue checking the remaining model files.
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private hashFile(path: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = createHash('sha256')
|
||||
const stream = createReadStream(path)
|
||||
stream.on('data', (chunk) => hash.update(chunk))
|
||||
stream.on('error', reject)
|
||||
stream.on('end', () => resolve(hash.digest('hex')))
|
||||
})
|
||||
}
|
||||
|
||||
private async writeVerifiedManifest(): Promise<void> {
|
||||
const manifest: VerifiedManifest = {
|
||||
modelId: this.modelId,
|
||||
version: this.version,
|
||||
fingerprint: this.fingerprint,
|
||||
files: Object.fromEntries(
|
||||
SENSEVOICE_MODEL_FILES.map((file) => [file.name, { size: file.size, sha256: file.sha256 }])
|
||||
)
|
||||
}
|
||||
const temporary = join(this.modelRoot, 'verified.json.partial')
|
||||
const target = join(this.modelRoot, 'verified.json')
|
||||
await writeFile(temporary, JSON.stringify(manifest, null, 2), 'utf8')
|
||||
await rm(target, { force: true })
|
||||
await rename(temporary, target)
|
||||
}
|
||||
|
||||
private buildStatus(
|
||||
state: VoiceModelStatus['state'],
|
||||
downloadedBytes: number,
|
||||
error?: string
|
||||
): VoiceModelStatus {
|
||||
return {
|
||||
modelId: this.modelId,
|
||||
version: this.version,
|
||||
state,
|
||||
downloadedBytes,
|
||||
totalBytes: TOTAL_BYTES,
|
||||
progress: TOTAL_BYTES ? Math.min(1, downloadedBytes / TOTAL_BYTES) : 0,
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
supported: this.isRuntimeSupported(),
|
||||
error
|
||||
}
|
||||
}
|
||||
|
||||
private isRuntimeSupported(): boolean {
|
||||
return (
|
||||
(process.platform === 'win32' && process.arch === 'x64') ||
|
||||
(process.platform === 'darwin' && (process.arch === 'x64' || process.arch === 'arm64'))
|
||||
)
|
||||
}
|
||||
|
||||
private reportProgress(status: VoiceModelStatus, force = false): void {
|
||||
const now = Date.now()
|
||||
if (!force && now - this.lastProgressAt < 100) return
|
||||
this.lastProgressAt = now
|
||||
this.progressListener?.(status)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
import { fork, type ChildProcess } from 'child_process'
|
||||
import { randomUUID } from 'crypto'
|
||||
import type {
|
||||
PipelineAudio,
|
||||
RecognitionMetadata,
|
||||
RecognitionOutput,
|
||||
SpeechRecognizer
|
||||
} from './types'
|
||||
import type { VoiceModelManager } from './model-manager'
|
||||
import {
|
||||
VOICE_WORKER_PROTOCOL_VERSION,
|
||||
type WorkerRecognitionRequest,
|
||||
type WorkerRecognitionResponse
|
||||
} from './worker-protocol'
|
||||
|
||||
type PendingRequest = {
|
||||
resolve: (result: RecognitionOutput) => void
|
||||
reject: (error: Error) => void
|
||||
timer: NodeJS.Timeout
|
||||
removeAbortListener: () => void
|
||||
}
|
||||
|
||||
export class RecognitionHost {
|
||||
private child: ChildProcess | null = null
|
||||
private readonly pending = new Map<string, PendingRequest>()
|
||||
private idleTimer: NodeJS.Timeout | null = null
|
||||
|
||||
constructor(
|
||||
private readonly workerPath: string,
|
||||
private readonly timeoutMs = 120_000,
|
||||
private readonly idleTimeoutMs = 60_000
|
||||
) {}
|
||||
|
||||
async recognize(
|
||||
audio: PipelineAudio,
|
||||
model: { modelPath: string; tokensPath: string; fingerprint: string },
|
||||
signal?: AbortSignal
|
||||
): Promise<RecognitionOutput> {
|
||||
if (signal?.aborted) throw new DOMException('Recognition cancelled', 'AbortError')
|
||||
const child = this.ensureChild()
|
||||
const requestId = randomUUID()
|
||||
const request: WorkerRecognitionRequest = {
|
||||
version: VOICE_WORKER_PROTOCOL_VERSION,
|
||||
type: 'recognize',
|
||||
requestId,
|
||||
payload: {
|
||||
recognizerId: 'sensevoice',
|
||||
samples: audio.samples,
|
||||
sampleRate: audio.sampleRate,
|
||||
modelPath: model.modelPath,
|
||||
tokensPath: model.tokensPath,
|
||||
modelFingerprint: model.fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
return new Promise<RecognitionOutput>((resolve, reject) => {
|
||||
const abort = (): void => {
|
||||
this.terminate(new DOMException('Recognition cancelled', 'AbortError'))
|
||||
}
|
||||
signal?.addEventListener('abort', abort, { once: true })
|
||||
const timer = setTimeout(() => {
|
||||
this.terminate(new Error('Voice recognition timed out'))
|
||||
}, this.timeoutMs)
|
||||
this.pending.set(requestId, {
|
||||
resolve,
|
||||
reject,
|
||||
timer,
|
||||
removeAbortListener: () => signal?.removeEventListener('abort', abort)
|
||||
})
|
||||
child.send(request, (error) => {
|
||||
if (error) this.finish(requestId, null, error)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
this.terminate(new Error('Voice recognition host disposed'))
|
||||
}
|
||||
|
||||
private ensureChild(): ChildProcess {
|
||||
if (this.idleTimer) {
|
||||
clearTimeout(this.idleTimer)
|
||||
this.idleTimer = null
|
||||
}
|
||||
if (this.child?.connected) return this.child
|
||||
const child = fork(this.workerPath, [], {
|
||||
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
||||
serialization: 'advanced',
|
||||
env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }
|
||||
})
|
||||
child.on('message', (message: WorkerRecognitionResponse) => {
|
||||
if (message?.version !== VOICE_WORKER_PROTOCOL_VERSION) return
|
||||
if (message.type === 'result') {
|
||||
this.finish(message.requestId, {
|
||||
text: message.transcript,
|
||||
language: message.language
|
||||
})
|
||||
} else {
|
||||
this.finish(message.requestId, null, new Error(message.error))
|
||||
}
|
||||
})
|
||||
child.once('error', (error) => this.terminate(error))
|
||||
child.once('exit', (code) => {
|
||||
if (this.child === child) {
|
||||
this.child = null
|
||||
this.rejectAll(new Error(`Voice recognition worker exited (${code ?? 'unknown'})`))
|
||||
}
|
||||
})
|
||||
this.child = child
|
||||
return child
|
||||
}
|
||||
|
||||
private finish(requestId: string, result: RecognitionOutput | null, error?: Error): void {
|
||||
const pending = this.pending.get(requestId)
|
||||
if (!pending) return
|
||||
this.pending.delete(requestId)
|
||||
clearTimeout(pending.timer)
|
||||
pending.removeAbortListener()
|
||||
if (error) pending.reject(error)
|
||||
else pending.resolve(result || { text: '' })
|
||||
if (this.pending.size === 0) this.scheduleIdleExit()
|
||||
}
|
||||
|
||||
private terminate(error: Error): void {
|
||||
if (this.idleTimer) {
|
||||
clearTimeout(this.idleTimer)
|
||||
this.idleTimer = null
|
||||
}
|
||||
const child = this.child
|
||||
this.child = null
|
||||
if (child && !child.killed) child.kill()
|
||||
this.rejectAll(error)
|
||||
}
|
||||
|
||||
private rejectAll(error: Error): void {
|
||||
for (const [requestId] of this.pending) this.finish(requestId, null, error)
|
||||
}
|
||||
|
||||
private scheduleIdleExit(): void {
|
||||
if (!this.child || this.idleTimer) return
|
||||
this.idleTimer = setTimeout(() => {
|
||||
this.idleTimer = null
|
||||
this.terminate(new Error('Voice recognition worker idle timeout'))
|
||||
}, this.idleTimeoutMs)
|
||||
}
|
||||
}
|
||||
|
||||
export class WorkerSpeechRecognizer implements SpeechRecognizer {
|
||||
readonly metadata: RecognitionMetadata
|
||||
|
||||
constructor(
|
||||
private readonly host: RecognitionHost,
|
||||
private readonly modelManager: VoiceModelManager
|
||||
) {
|
||||
this.metadata = {
|
||||
recognizerId: 'sensevoice',
|
||||
modelVersion: modelManager.version,
|
||||
modelFingerprint: modelManager.fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
async recognize(audio: PipelineAudio, signal?: AbortSignal): Promise<RecognitionOutput> {
|
||||
const paths = await this.modelManager.getPaths()
|
||||
if (!paths) throw new Error('Voice recognition model is not ready')
|
||||
return this.host.recognize(
|
||||
audio,
|
||||
{
|
||||
modelPath: paths.model,
|
||||
tokensPath: paths.tokens,
|
||||
fingerprint: this.modelManager.fingerprint
|
||||
},
|
||||
signal
|
||||
)
|
||||
}
|
||||
|
||||
dispose(): Promise<void> {
|
||||
return this.host.dispose()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { createRequire } from 'module'
|
||||
import type { WorkerRecognizerEngine, WorkerRecognizerInput } from './worker-recognizer-registry'
|
||||
|
||||
const nodeRequire = createRequire(import.meta.url)
|
||||
|
||||
interface OfflineRecognitionResult {
|
||||
text?: string
|
||||
lang?: string
|
||||
}
|
||||
|
||||
interface OfflineStream {
|
||||
acceptWaveform(input: { samples: Float32Array; sampleRate: number }): void
|
||||
}
|
||||
|
||||
interface OfflineRecognizerInstance {
|
||||
createStream(): OfflineStream
|
||||
decodeAsync(stream: OfflineStream): Promise<OfflineRecognitionResult>
|
||||
}
|
||||
|
||||
interface OfflineRecognizerConstructor {
|
||||
createAsync(config: Record<string, unknown>): Promise<OfflineRecognizerInstance>
|
||||
}
|
||||
|
||||
export class SenseVoiceRecognizer implements WorkerRecognizerEngine {
|
||||
readonly id = 'sensevoice'
|
||||
private recognizer: OfflineRecognizerInstance | null = null
|
||||
private fingerprint = ''
|
||||
|
||||
async recognize(
|
||||
input: WorkerRecognizerInput
|
||||
): Promise<{ transcript: string; language?: string }> {
|
||||
if (!this.recognizer || this.fingerprint !== input.modelFingerprint) {
|
||||
const sherpa = nodeRequire('sherpa-onnx-node') as {
|
||||
OfflineRecognizer: OfflineRecognizerConstructor
|
||||
}
|
||||
this.recognizer = await sherpa.OfflineRecognizer.createAsync({
|
||||
featConfig: { sampleRate: input.sampleRate, featureDim: 80 },
|
||||
modelConfig: {
|
||||
senseVoice: {
|
||||
model: input.modelPath,
|
||||
language: 'auto',
|
||||
useInverseTextNormalization: 1
|
||||
},
|
||||
tokens: input.tokensPath,
|
||||
numThreads: Math.max(1, Math.min(4, Number(process.env.WXE_VOICE_THREADS) || 2)),
|
||||
provider: 'cpu',
|
||||
debug: 0
|
||||
}
|
||||
})
|
||||
this.fingerprint = input.modelFingerprint
|
||||
}
|
||||
|
||||
const stream = this.recognizer.createStream()
|
||||
stream.acceptWaveform({ samples: input.samples, sampleRate: input.sampleRate })
|
||||
const result = await this.recognizer.decodeAsync(stream)
|
||||
return { transcript: String(result.text || '').trim(), language: result.lang || undefined }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
type ScheduledTask<T> = {
|
||||
key: string
|
||||
run: (signal: AbortSignal) => Promise<T>
|
||||
controller: AbortController
|
||||
resolve: (value: T) => void
|
||||
reject: (reason: unknown) => void
|
||||
}
|
||||
|
||||
export class VoiceTaskScheduler {
|
||||
private readonly queue: ScheduledTask<unknown>[] = []
|
||||
private active: ScheduledTask<unknown> | null = null
|
||||
|
||||
schedule<T>(key: string, run: (signal: AbortSignal) => Promise<T>): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
this.queue.push({
|
||||
key,
|
||||
run,
|
||||
controller: new AbortController(),
|
||||
resolve: resolve as (value: unknown) => void,
|
||||
reject
|
||||
})
|
||||
this.pump()
|
||||
})
|
||||
}
|
||||
|
||||
cancel(key: string): boolean {
|
||||
if (this.active?.key === key) {
|
||||
this.active.controller.abort()
|
||||
return true
|
||||
}
|
||||
const index = this.queue.findIndex((task) => task.key === key)
|
||||
if (index < 0) return false
|
||||
const [task] = this.queue.splice(index, 1)
|
||||
task.controller.abort()
|
||||
task.reject(new DOMException('Recognition cancelled', 'AbortError'))
|
||||
return true
|
||||
}
|
||||
|
||||
cancelAll(): void {
|
||||
this.active?.controller.abort()
|
||||
while (this.queue.length) {
|
||||
const task = this.queue.shift()
|
||||
task?.controller.abort()
|
||||
task?.reject(new DOMException('Recognition cancelled', 'AbortError'))
|
||||
}
|
||||
}
|
||||
|
||||
private pump(): void {
|
||||
if (this.active || this.queue.length === 0) return
|
||||
const task = this.queue.shift()
|
||||
if (!task) return
|
||||
this.active = task
|
||||
void task
|
||||
.run(task.controller.signal)
|
||||
.then(task.resolve, task.reject)
|
||||
.finally(() => {
|
||||
this.active = null
|
||||
this.pump()
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
import { dirname } from 'path'
|
||||
import { mkdirSync } from 'fs'
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
import type { TranscriptRecord, TranscriptRepository } from './types'
|
||||
|
||||
type TranscriptKey = Omit<
|
||||
TranscriptRecord,
|
||||
'transcript' | 'language' | 'durationMs' | 'createdAt' | 'updatedAt'
|
||||
>
|
||||
|
||||
export class SqliteTranscriptRepository implements TranscriptRepository {
|
||||
private readonly database: DatabaseSync
|
||||
|
||||
constructor(databasePath: string) {
|
||||
mkdirSync(dirname(databasePath), { recursive: true })
|
||||
this.database = new DatabaseSync(databasePath)
|
||||
this.database.exec(`
|
||||
PRAGMA journal_mode = WAL;
|
||||
CREATE TABLE IF NOT EXISTS voice_transcripts (
|
||||
account_id TEXT NOT NULL,
|
||||
message_identity TEXT NOT NULL,
|
||||
audio_hash TEXT NOT NULL,
|
||||
processor_version TEXT NOT NULL,
|
||||
recognizer_id TEXT NOT NULL,
|
||||
model_version TEXT NOT NULL,
|
||||
model_fingerprint TEXT NOT NULL,
|
||||
transcript TEXT NOT NULL,
|
||||
language TEXT,
|
||||
duration_ms INTEGER NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (
|
||||
account_id, message_identity, audio_hash, processor_version,
|
||||
recognizer_id, model_version, model_fingerprint
|
||||
)
|
||||
) STRICT;
|
||||
`)
|
||||
}
|
||||
|
||||
find(key: TranscriptKey): TranscriptRecord | null {
|
||||
const row = this.database
|
||||
.prepare(
|
||||
`SELECT account_id, message_identity, audio_hash, processor_version,
|
||||
recognizer_id, model_version, model_fingerprint, transcript,
|
||||
language, duration_ms, created_at, updated_at
|
||||
FROM voice_transcripts
|
||||
WHERE account_id = ? AND message_identity = ? AND audio_hash = ?
|
||||
AND processor_version = ? AND recognizer_id = ? AND model_version = ?
|
||||
AND model_fingerprint = ?`
|
||||
)
|
||||
.get(
|
||||
key.accountId,
|
||||
key.messageIdentity,
|
||||
key.audioHash,
|
||||
key.processorVersion,
|
||||
key.recognizerId,
|
||||
key.modelVersion,
|
||||
key.modelFingerprint
|
||||
) as Record<string, unknown> | undefined
|
||||
if (!row) return null
|
||||
return {
|
||||
accountId: String(row.account_id),
|
||||
messageIdentity: String(row.message_identity),
|
||||
audioHash: String(row.audio_hash),
|
||||
processorVersion: String(row.processor_version),
|
||||
recognizerId: String(row.recognizer_id),
|
||||
modelVersion: String(row.model_version),
|
||||
modelFingerprint: String(row.model_fingerprint),
|
||||
transcript: String(row.transcript),
|
||||
language: row.language ? String(row.language) : undefined,
|
||||
durationMs: Number(row.duration_ms),
|
||||
createdAt: Number(row.created_at),
|
||||
updatedAt: Number(row.updated_at)
|
||||
}
|
||||
}
|
||||
|
||||
save(record: TranscriptRecord): void {
|
||||
this.database
|
||||
.prepare(
|
||||
`INSERT INTO voice_transcripts (
|
||||
account_id, message_identity, audio_hash, processor_version,
|
||||
recognizer_id, model_version, model_fingerprint, transcript,
|
||||
language, duration_ms, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (
|
||||
account_id, message_identity, audio_hash, processor_version,
|
||||
recognizer_id, model_version, model_fingerprint
|
||||
) DO UPDATE SET
|
||||
transcript = excluded.transcript,
|
||||
language = excluded.language,
|
||||
duration_ms = excluded.duration_ms,
|
||||
updated_at = excluded.updated_at`
|
||||
)
|
||||
.run(
|
||||
record.accountId,
|
||||
record.messageIdentity,
|
||||
record.audioHash,
|
||||
record.processorVersion,
|
||||
record.recognizerId,
|
||||
record.modelVersion,
|
||||
record.modelFingerprint,
|
||||
record.transcript,
|
||||
record.language ?? null,
|
||||
record.durationMs,
|
||||
record.createdAt,
|
||||
record.updatedAt
|
||||
)
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.database.close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import type { VoiceMessageReference } from '../../shared/voice-recognition'
|
||||
import type { EncodedVoiceSource } from './audio-decoder'
|
||||
|
||||
export interface PipelineAudio {
|
||||
samples: Float32Array
|
||||
sampleRate: number
|
||||
channels: 1
|
||||
sourceHash: string
|
||||
processorVersion: string
|
||||
durationMs: number
|
||||
}
|
||||
|
||||
export interface RecognitionMetadata {
|
||||
recognizerId: string
|
||||
modelVersion: string
|
||||
modelFingerprint: string
|
||||
}
|
||||
|
||||
export interface RecognitionOutput {
|
||||
text: string
|
||||
language?: string
|
||||
}
|
||||
|
||||
export interface SourceResolver {
|
||||
resolve(reference: VoiceMessageReference): Promise<EncodedVoiceSource>
|
||||
}
|
||||
|
||||
export class SpeechRecognizerRegistry {
|
||||
private readonly recognizers = new Map<string, SpeechRecognizer>()
|
||||
|
||||
register(recognizer: SpeechRecognizer): this {
|
||||
const id = recognizer.metadata.recognizerId
|
||||
if (this.recognizers.has(id)) throw new Error(`Recognizer already registered: ${id}`)
|
||||
this.recognizers.set(id, recognizer)
|
||||
return this
|
||||
}
|
||||
|
||||
get(recognizerId: string): SpeechRecognizer {
|
||||
const recognizer = this.recognizers.get(recognizerId)
|
||||
if (!recognizer) throw new Error(`Recognizer is not registered: ${recognizerId}`)
|
||||
return recognizer
|
||||
}
|
||||
}
|
||||
|
||||
export interface AudioProcessor {
|
||||
process(input: {
|
||||
pcm: Buffer
|
||||
sampleRate: number
|
||||
channels: number
|
||||
sourceHash: string
|
||||
}): PipelineAudio
|
||||
}
|
||||
|
||||
export interface SpeechRecognizer {
|
||||
readonly metadata: RecognitionMetadata
|
||||
recognize(audio: PipelineAudio, signal?: AbortSignal): Promise<RecognitionOutput>
|
||||
dispose(): Promise<void>
|
||||
}
|
||||
|
||||
export interface TranscriptRecord extends RecognitionMetadata {
|
||||
accountId: string
|
||||
messageIdentity: string
|
||||
audioHash: string
|
||||
processorVersion: string
|
||||
transcript: string
|
||||
language?: string
|
||||
durationMs: number
|
||||
createdAt: number
|
||||
updatedAt: number
|
||||
}
|
||||
|
||||
export interface TranscriptRepository {
|
||||
find(
|
||||
key: Omit<
|
||||
TranscriptRecord,
|
||||
'transcript' | 'language' | 'durationMs' | 'createdAt' | 'updatedAt'
|
||||
>
|
||||
): TranscriptRecord | null
|
||||
save(record: TranscriptRecord): void
|
||||
close(): void
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { createHash } from 'crypto'
|
||||
import type { VoiceMessageReference } from '../../shared/voice-recognition'
|
||||
import type { VoiceService } from '../voice-service'
|
||||
import type { AudioDecoderRegistry, EncodedVoiceSource } from './audio-decoder'
|
||||
import type {
|
||||
AudioProcessor,
|
||||
SourceResolver,
|
||||
SpeechRecognizer,
|
||||
TranscriptRecord,
|
||||
TranscriptRepository
|
||||
} from './types'
|
||||
|
||||
export class VoiceSourceResolver implements SourceResolver {
|
||||
constructor(private readonly voiceService: VoiceService) {}
|
||||
|
||||
async resolve(reference: VoiceMessageReference): Promise<EncodedVoiceSource> {
|
||||
const result = await this.voiceService.resolveSource(
|
||||
reference.sessionId,
|
||||
reference.localId,
|
||||
reference.createTime,
|
||||
reference.svrId
|
||||
)
|
||||
if (!result.success) throw new Error(result.error)
|
||||
return result.source
|
||||
}
|
||||
}
|
||||
|
||||
export class VoicePipeline {
|
||||
constructor(
|
||||
private readonly sourceResolver: SourceResolver,
|
||||
private readonly decoderRegistry: AudioDecoderRegistry,
|
||||
private readonly audioProcessor: AudioProcessor,
|
||||
private readonly recognizer: SpeechRecognizer,
|
||||
private readonly transcripts: TranscriptRepository
|
||||
) {}
|
||||
|
||||
async run(
|
||||
accountId: string,
|
||||
reference: VoiceMessageReference,
|
||||
signal?: AbortSignal
|
||||
): Promise<{ transcript: string; language?: string; durationMs: number; cached: boolean }> {
|
||||
const source = await this.sourceResolver.resolve(reference)
|
||||
if (signal?.aborted) throw new DOMException('Recognition cancelled', 'AbortError')
|
||||
const decoded = await this.decoderRegistry.decode(source)
|
||||
if (signal?.aborted) throw new DOMException('Recognition cancelled', 'AbortError')
|
||||
const audio = this.audioProcessor.process(decoded)
|
||||
if (audio.samples.length === 0) throw new Error('Voice audio is empty after processing')
|
||||
const messageIdentity = createHash('sha256')
|
||||
.update(
|
||||
`${reference.sessionId}|${reference.localId}|${reference.createTime}|${reference.svrId ?? ''}`
|
||||
)
|
||||
.digest('hex')
|
||||
const key = {
|
||||
accountId,
|
||||
messageIdentity,
|
||||
audioHash: audio.sourceHash,
|
||||
processorVersion: audio.processorVersion,
|
||||
...this.recognizer.metadata
|
||||
}
|
||||
const cached = this.transcripts.find(key)
|
||||
if (cached) {
|
||||
return {
|
||||
transcript: cached.transcript,
|
||||
language: cached.language,
|
||||
durationMs: cached.durationMs,
|
||||
cached: true
|
||||
}
|
||||
}
|
||||
|
||||
const output = await this.recognizer.recognize(audio, signal)
|
||||
const now = Date.now()
|
||||
const record: TranscriptRecord = {
|
||||
...key,
|
||||
transcript: output.text,
|
||||
language: output.language,
|
||||
durationMs: audio.durationMs,
|
||||
createdAt: now,
|
||||
updatedAt: now
|
||||
}
|
||||
this.transcripts.save(record)
|
||||
return {
|
||||
transcript: output.text,
|
||||
language: output.language,
|
||||
durationMs: audio.durationMs,
|
||||
cached: false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
import { createHash } from 'crypto'
|
||||
import type {
|
||||
VoiceMessageReference,
|
||||
VoiceModelDownloadResult,
|
||||
VoiceModelStatus,
|
||||
VoiceRecognitionResult
|
||||
} from '../../shared/voice-recognition'
|
||||
import type { VoiceService } from '../voice-service'
|
||||
import { PcmAudioProcessor } from './audio-processor'
|
||||
import { createDefaultAudioDecoderRegistry } from './audio-decoder'
|
||||
import { VoiceModelManager } from './model-manager'
|
||||
import { RecognitionHost, WorkerSpeechRecognizer } from './recognition-host'
|
||||
import { VoiceTaskScheduler } from './task-scheduler'
|
||||
import { SqliteTranscriptRepository } from './transcript-repository'
|
||||
import { VoicePipeline, VoiceSourceResolver } from './voice-pipeline'
|
||||
import { SpeechRecognizerRegistry } from './types'
|
||||
|
||||
export class VoiceRecognitionUseCase {
|
||||
readonly modelManager: VoiceModelManager
|
||||
private readonly scheduler = new VoiceTaskScheduler()
|
||||
private readonly transcripts: SqliteTranscriptRepository
|
||||
private readonly recognizer: WorkerSpeechRecognizer
|
||||
private readonly recognizers = new SpeechRecognizerRegistry()
|
||||
private pipeline: VoicePipeline | null = null
|
||||
private accountId = ''
|
||||
|
||||
constructor(options: { modelRoot: string; databasePath: string; workerPath: string }) {
|
||||
this.modelManager = new VoiceModelManager(options.modelRoot)
|
||||
this.transcripts = new SqliteTranscriptRepository(options.databasePath)
|
||||
this.recognizer = new WorkerSpeechRecognizer(
|
||||
new RecognitionHost(options.workerPath),
|
||||
this.modelManager
|
||||
)
|
||||
this.recognizers.register(this.recognizer)
|
||||
}
|
||||
|
||||
connect(voiceService: VoiceService, accountRoot: string): void {
|
||||
this.scheduler.cancelAll()
|
||||
this.accountId = createHash('sha256')
|
||||
.update(
|
||||
accountRoot
|
||||
.trim()
|
||||
.replace(/[\\/]+$/, '')
|
||||
.toLowerCase()
|
||||
)
|
||||
.digest('hex')
|
||||
this.pipeline = new VoicePipeline(
|
||||
new VoiceSourceResolver(voiceService),
|
||||
createDefaultAudioDecoderRegistry(),
|
||||
new PcmAudioProcessor(),
|
||||
this.recognizers.get('sensevoice'),
|
||||
this.transcripts
|
||||
)
|
||||
}
|
||||
|
||||
disconnect(): void {
|
||||
this.scheduler.cancelAll()
|
||||
this.pipeline = null
|
||||
this.accountId = ''
|
||||
}
|
||||
|
||||
getModelStatus(): Promise<VoiceModelStatus> {
|
||||
return this.modelManager.getStatus()
|
||||
}
|
||||
|
||||
downloadModel(): Promise<VoiceModelDownloadResult> {
|
||||
return this.modelManager.download()
|
||||
}
|
||||
|
||||
cancelModelDownload(): { success: boolean } {
|
||||
return { success: this.modelManager.cancelDownload() }
|
||||
}
|
||||
|
||||
async removeModel(): Promise<VoiceModelStatus> {
|
||||
this.scheduler.cancelAll()
|
||||
await this.recognizer.dispose()
|
||||
return this.modelManager.remove()
|
||||
}
|
||||
|
||||
recognize(reference: VoiceMessageReference): Promise<VoiceRecognitionResult> {
|
||||
const pipeline = this.pipeline
|
||||
const accountId = this.accountId
|
||||
if (!pipeline || !accountId) {
|
||||
return Promise.resolve({ success: false, code: 'NOT_CONNECTED', error: '请先连接微信数据库' })
|
||||
}
|
||||
const key = this.taskKey(reference)
|
||||
return this.scheduler
|
||||
.schedule(key, async (signal) => {
|
||||
const status = await this.modelManager.getStatus()
|
||||
if (status.state !== 'ready') {
|
||||
return { success: false, code: 'MODEL_NOT_READY', error: '请先下载语音识别模型' } as const
|
||||
}
|
||||
const result = await pipeline.run(accountId, reference, signal)
|
||||
return { success: true, ...result } as const
|
||||
})
|
||||
.catch((error): VoiceRecognitionResult => {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return { success: false, code: 'CANCELLED', error: '语音识别已取消' }
|
||||
}
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
const code = message.toLowerCase().includes('timed out') ? 'TIMEOUT' : 'RECOGNITION_FAILED'
|
||||
return { success: false, code, error: message }
|
||||
})
|
||||
}
|
||||
|
||||
cancelRecognition(reference: VoiceMessageReference): { success: boolean } {
|
||||
return { success: this.scheduler.cancel(this.taskKey(reference)) }
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
this.scheduler.cancelAll()
|
||||
await this.recognizer.dispose()
|
||||
this.transcripts.close()
|
||||
}
|
||||
|
||||
private taskKey(reference: VoiceMessageReference): string {
|
||||
return `${this.accountId}:${reference.sessionId}:${reference.localId}:${reference.createTime}`
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { SenseVoiceRecognizer } from './sensevoice-recognizer'
|
||||
import { WorkerRecognizerRegistry } from './worker-recognizer-registry'
|
||||
import {
|
||||
VOICE_WORKER_PROTOCOL_VERSION,
|
||||
type WorkerRecognitionRequest,
|
||||
type WorkerRecognitionResponse
|
||||
} from './worker-protocol'
|
||||
|
||||
const recognizers = new WorkerRecognizerRegistry().register(new SenseVoiceRecognizer())
|
||||
|
||||
function send(response: WorkerRecognitionResponse): void {
|
||||
if (process.send) process.send(response)
|
||||
}
|
||||
|
||||
process.on('message', async (message: WorkerRecognitionRequest) => {
|
||||
if (
|
||||
message?.version !== VOICE_WORKER_PROTOCOL_VERSION ||
|
||||
message.type !== 'recognize' ||
|
||||
!message.requestId
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const fakeTranscript = process.env.WXE_VOICE_RECOGNITION_FAKE_TEXT
|
||||
const result = fakeTranscript
|
||||
? { transcript: fakeTranscript, language: 'zh' }
|
||||
: await recognizers.get(message.payload.recognizerId).recognize(message.payload)
|
||||
send({
|
||||
version: VOICE_WORKER_PROTOCOL_VERSION,
|
||||
type: 'result',
|
||||
requestId: message.requestId,
|
||||
...result
|
||||
})
|
||||
} catch (error) {
|
||||
send({
|
||||
version: VOICE_WORKER_PROTOCOL_VERSION,
|
||||
type: 'error',
|
||||
requestId: message.requestId,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,30 @@
|
||||
export const VOICE_WORKER_PROTOCOL_VERSION = 1
|
||||
|
||||
export interface WorkerRecognitionRequest {
|
||||
version: typeof VOICE_WORKER_PROTOCOL_VERSION
|
||||
type: 'recognize'
|
||||
requestId: string
|
||||
payload: {
|
||||
recognizerId: string
|
||||
samples: Float32Array
|
||||
sampleRate: number
|
||||
modelPath: string
|
||||
tokensPath: string
|
||||
modelFingerprint: string
|
||||
}
|
||||
}
|
||||
|
||||
export type WorkerRecognitionResponse =
|
||||
| {
|
||||
version: typeof VOICE_WORKER_PROTOCOL_VERSION
|
||||
type: 'result'
|
||||
requestId: string
|
||||
transcript: string
|
||||
language?: string
|
||||
}
|
||||
| {
|
||||
version: typeof VOICE_WORKER_PROTOCOL_VERSION
|
||||
type: 'error'
|
||||
requestId: string
|
||||
error: string
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
export interface WorkerRecognizerInput {
|
||||
samples: Float32Array
|
||||
sampleRate: number
|
||||
modelPath: string
|
||||
tokensPath: string
|
||||
modelFingerprint: string
|
||||
}
|
||||
|
||||
export interface WorkerRecognizerEngine {
|
||||
readonly id: string
|
||||
recognize(input: WorkerRecognizerInput): Promise<{ transcript: string; language?: string }>
|
||||
}
|
||||
|
||||
export class WorkerRecognizerRegistry {
|
||||
private readonly engines = new Map<string, WorkerRecognizerEngine>()
|
||||
|
||||
register(engine: WorkerRecognizerEngine): this {
|
||||
if (this.engines.has(engine.id))
|
||||
throw new Error(`Worker recognizer already registered: ${engine.id}`)
|
||||
this.engines.set(engine.id, engine)
|
||||
return this
|
||||
}
|
||||
|
||||
get(id: string): WorkerRecognizerEngine {
|
||||
const engine = this.engines.get(id)
|
||||
if (!engine) throw new Error(`Worker recognizer is not registered: ${id}`)
|
||||
return engine
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user