From ce4b00bcd9d4bc824555664a6acd999ffdb9699a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B5=E6=91=87=E5=B0=8F=E5=AD=90?= Date: Mon, 13 Jul 2026 17:02:30 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=20AI=20=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E7=90=86=E8=A7=A3=E8=83=BD=E5=8A=9B=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- resources/mobile_daily_report.html | 15 +- src/main/group-report-service.ts | 3 +- src/main/index.ts | 10 +- src/main/services/ai-provider-service.ts | 156 +++++++++++++- src/preload/index.d.ts | 3 + src/preload/index.ts | 8 +- src/renderer/src/assets/main.css | 193 ++++++++++++++++++ .../ai-model/AIImageUnderstandingTest.tsx | 121 +++++++++++ .../ai-model/aiModelSettingsReducer.ts | 47 ++++- .../src/features/settings/ai-model/types.ts | 34 ++- .../ai-model/useAIModelSettingsController.ts | 84 +++++++- .../features/settings/pages/AIModelPage.tsx | 13 ++ src/renderer/src/utils/group-report.ts | 2 +- src/shared/ai-provider.ts | 24 +++ 14 files changed, 696 insertions(+), 17 deletions(-) create mode 100644 src/renderer/src/features/settings/ai-model/AIImageUnderstandingTest.tsx diff --git a/resources/mobile_daily_report.html b/resources/mobile_daily_report.html index 2197bff..784ff69 100644 --- a/resources/mobile_daily_report.html +++ b/resources/mobile_daily_report.html @@ -61,6 +61,13 @@ font-size: 13px; line-height: 1.5; } + .record-note { + color: #485465; + font-weight: 650; + } + .overview { + margin-top: 2px; + } .avatar-grid { width: 58px; height: 58px; @@ -422,7 +429,11 @@

{{GROUP_NAME}}日报

-
{{DATE_RANGE}}
{{RECORD_NOTE}}
+
+
{{DATE_RANGE}}
+
{{RECORD_NOTE}}
+
{{OVERVIEW}}
+
{{HERO_AVATARS}}
@@ -479,7 +490,7 @@ diff --git a/src/main/group-report-service.ts b/src/main/group-report-service.ts index db52b4b..f431b9b 100644 --- a/src/main/group-report-service.ts +++ b/src/main/group-report-service.ts @@ -249,7 +249,8 @@ const renderReportHtml = async (request: GroupReportExportRequest): Promise { aiProviderService.setDefault(providerId) ) ipcMain.handle('ai:testProvider', (_, providerId: string) => aiProviderService.test(providerId)) + ipcMain.handle('ai:testVision', (_, request: AIVisionTestRequest) => + aiProviderService.testVision(request) + ) ipcMain.handle('ai:migrateLegacy', (_, config: LegacyAIConfig) => aiProviderService.migrateLegacy(config) ) diff --git a/src/main/services/ai-provider-service.ts b/src/main/services/ai-provider-service.ts index 70d86f3..5c93aaf 100644 --- a/src/main/services/ai-provider-service.ts +++ b/src/main/services/ai-provider-service.ts @@ -8,6 +8,8 @@ import type { AIProviderListResult, AIProviderSummary, AIRuntimeModelConfig, + AIVisionTestRequest, + AIVisionTestResult, LegacyAIConfig } from '../../shared/ai-provider' import { AIProviderKeyStore } from '../ai-provider-key-store' @@ -18,7 +20,8 @@ interface AIProviderMetadataFile { providers: Array> } -type AIMessage = { role: string; content: string } +type AIMessagePart = { type: 'text'; text: string } | { type: 'image'; dataUrl: string } +type AIMessage = { role: string; content: string | AIMessagePart[] } type AIRequestResult = { data: string usage?: { input?: number; output?: number; total?: number; estimated?: boolean } @@ -151,7 +154,7 @@ export class AIProviderService { } async chat( - messages: AIMessage[], + messages: Array<{ role: string; content: string }>, options?: AIChatRequestOptions ): Promise<{ success: boolean @@ -166,6 +169,42 @@ export class AIProviderService { } } + async testVision(request: AIVisionTestRequest): Promise { + const startedAt = Date.now() + const imageError = validateVisionImage(request.imageDataUrl) + if (imageError) return { success: false, code: 'INVALID_IMAGE', error: imageError } + if (!request.prompt.trim()) { + return { success: false, code: 'INVALID_IMAGE', error: '请填写图片识别提示词' } + } + try { + const resolved = this.resolveProvider(request) + const result = await requestProvider(resolved.provider, resolved.key, resolved.model, [ + { + role: 'user', + content: [ + { type: 'text', text: request.prompt.trim() }, + { type: 'image', dataUrl: request.imageDataUrl } + ] + } + ]) + if (!result.data.trim()) throw new Error('API 未返回识别内容') + this.markVisionCapability(resolved.provider.id, resolved.model) + const model = resolved.provider.models.find((item) => item.id === resolved.model) + return { + success: true, + providerName: resolved.provider.name, + modelId: resolved.model, + modelName: model?.name || resolved.model, + latencyMs: Date.now() - startedAt, + usage: result.usage, + answer: result.data + } + } catch (error) { + const failure = visionFailure(error) + return { success: false, ...failure, latencyMs: Date.now() - startedAt } + } + } + private async request( messages: AIMessage[], options?: AIChatRequestOptions, @@ -175,17 +214,25 @@ export class AIProviderService { usage?: { input?: number; output?: number; total?: number; estimated?: boolean } }> { if (options?.apiKey) return this.requestLegacy(messages, options) + const resolved = this.resolveProvider(options) + return requestProvider(resolved.provider, resolved.key, resolved.model, messages, testing) + } + + private resolveProvider(options?: { providerId?: string; modelId?: string }): { + provider: AIProviderSummary + model: string + key: string + } { const list = this.list() const provider = list.providers.find((item) => item.id === options?.providerId) || list.providers.find((item) => item.id === list.defaultProviderId) if (!provider) throw new Error('尚未配置 AI Provider') const model = options?.modelId || provider.defaultModel + if (!provider.models.some((item) => item.id === model)) throw new Error('当前模型不存在') const key = this.keyStore.get(provider.id).key || '' if (needsApiKey(provider) && !key) throw new Error('当前供应商尚未配置 API Key') - return provider.type === 'anthropic-messages' - ? requestAnthropic(provider, key, model, messages, testing) - : requestOpenAICompatible(provider, key, model, messages, testing) + return { provider, model, key } } private async requestLegacy( @@ -215,6 +262,15 @@ export class AIProviderService { this.writeMetadata(data) } + private markVisionCapability(providerId: string, modelId: string): void { + const data = this.readMetadata() + const provider = data.providers.find((item) => item.id === providerId) + const model = provider?.models.find((item) => item.id === modelId) + if (!provider || !model || model.capabilities.vision) return + model.capabilities.vision = true + this.writeMetadata(data) + } + private ensureEnvironmentMigration(): void { const data = this.readMetadata() if (data.providers.length) return @@ -329,6 +385,51 @@ function buildHeaders(provider: AIProviderSummary, apiKey: string): Record { + return provider.type === 'anthropic-messages' + ? requestAnthropic(provider, apiKey, model, messages, testing) + : requestOpenAICompatible(provider, apiKey, model, messages, testing) +} + +function toOpenAIMessages(messages: AIMessage[]): Array<{ role: string; content: unknown }> { + return messages.map((message) => ({ + role: message.role, + content: + typeof message.content === 'string' + ? message.content + : message.content.map((part) => + part.type === 'text' + ? { type: 'text', text: part.text } + : { type: 'image_url', image_url: { url: part.dataUrl } } + ) + })) +} + +function toAnthropicMessages(messages: AIMessage[]): Array<{ role: string; content: unknown }> { + return messages + .filter((message) => message.role !== 'system') + .map((message) => ({ + role: message.role, + content: + typeof message.content === 'string' + ? message.content + : message.content.map((part) => { + if (part.type === 'text') return { type: 'text', text: part.text } + const image = parseVisionImage(part.dataUrl) + return { + type: 'image', + source: { type: 'base64', media_type: image.mimeType, data: image.base64 } + } + }) + })) +} + async function requestOpenAICompatible( provider: AIProviderSummary, apiKey: string, @@ -346,7 +447,7 @@ async function requestOpenAICompatible( headers: buildHeaders(provider, apiKey), body: JSON.stringify({ model, - messages, + messages: toOpenAIMessages(messages), temperature: provider.advanced.temperature, max_tokens: testing ? 8 : provider.advanced.maxTokens }) @@ -377,9 +478,16 @@ async function requestAnthropic( ): Promise { const system = messages .filter((message) => message.role === 'system') - .map((message) => message.content) + .map((message) => + typeof message.content === 'string' + ? message.content + : message.content + .filter((part) => part.type === 'text') + .map((part) => (part.type === 'text' ? part.text : '')) + .join('\n') + ) .join('\n\n') - const anthropicMessages = messages.filter((message) => message.role !== 'system') + const anthropicMessages = toAnthropicMessages(messages) const headers = buildHeaders(provider, apiKey) if (!headers['anthropic-version']) headers['anthropic-version'] = '2023-06-01' const endpoint = provider.baseUrl.endsWith('/messages') @@ -440,3 +548,35 @@ function safeAIError(error: unknown): string { const message = error instanceof Error ? error.message : String(error) return message.replace(/sk-[a-z0-9_-]+/gi, '***').slice(0, 300) } + +function parseVisionImage(dataUrl: string): { mimeType: string; base64: string; bytes: number } { + const match = /^data:(image\/(?:png|jpeg|webp));base64,([a-z0-9+/=]+)$/i.exec(dataUrl) + if (!match) throw new Error('图片格式不受支持，请选择 PNG、JPG、JPEG 或 WebP') + const bytes = Buffer.byteLength(match[2], 'base64') + return { mimeType: match[1].toLowerCase(), base64: match[2], bytes } +} + +function validateVisionImage(dataUrl: string): string | undefined { + try { + const image = parseVisionImage(dataUrl) + if (!image.bytes) return '图片内容为空' + if (image.bytes > 10 * 1024 * 1024) return '图片不能超过 10 MB' + return undefined + } catch (error) { + return error instanceof Error ? error.message : '图片无法读取' + } +} + +function visionFailure(error: unknown): { + code: 'VISION_UNSUPPORTED' | 'API_ERROR' + error: string +} { + const message = safeAIError(error) + const unsupported = + /vision|multimodal|image[_ ]url|image input|image.*support|support.*image|图片.*不支持|不支持.*图片/i.test( + message + ) + return unsupported + ? { code: 'VISION_UNSUPPORTED', error: '当前模型不支持图片理解' } + : { code: 'API_ERROR', error: message || 'API 返回错误' } +} diff --git a/src/preload/index.d.ts b/src/preload/index.d.ts index 4614298..8301c11 100644 --- a/src/preload/index.d.ts +++ b/src/preload/index.d.ts @@ -26,6 +26,8 @@ import type { AIProviderConfig, AIProviderListResult, AIRuntimeModelConfig, + AIVisionTestRequest, + AIVisionTestResult, LegacyAIConfig } from '../shared/ai-provider' @@ -108,6 +110,7 @@ declare global { deleteAIProvider: (providerId: string) => Promise setDefaultAIProvider: (providerId: string) => Promise testAIProvider: (providerId: string) => Promise + testAIVision: (request: AIVisionTestRequest) => Promise migrateLegacyAIConfig: (config: LegacyAIConfig) => Promise copyImage: (base64String: string) => Promise<{ success: boolean; error?: string }> getVoiceData: ( diff --git a/src/preload/index.ts b/src/preload/index.ts index 8aa1e8a..a4b3a07 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -2,7 +2,12 @@ import { contextBridge, ipcRenderer } from 'electron' import { electronAPI } from '@electron-toolkit/preload' import type { GroupReportExportRequest } from '../shared/group-report' import type { SaveGeneratedReportRequest } from '../shared/report-history' -import type { AIChatRequestOptions, AIProviderConfig, LegacyAIConfig } from '../shared/ai-provider' +import type { + AIChatRequestOptions, + AIProviderConfig, + AIVisionTestRequest, + LegacyAIConfig +} from '../shared/ai-provider' // 渲染器的自定义 API const api = { @@ -29,6 +34,7 @@ const api = { setDefaultAIProvider: (providerId: string) => ipcRenderer.invoke('ai:setDefaultProvider', providerId), testAIProvider: (providerId: string) => ipcRenderer.invoke('ai:testProvider', providerId), + testAIVision: (request: AIVisionTestRequest) => ipcRenderer.invoke('ai:testVision', request), migrateLegacyAIConfig: (config: LegacyAIConfig) => ipcRenderer.invoke('ai:migrateLegacy', config), copyImage: (base64String) => ipcRenderer.invoke('copy-image', base64String), getVoiceData: (sessionId: string, localId: number, createTime: number, svrId?: string | number) => diff --git a/src/renderer/src/assets/main.css b/src/renderer/src/assets/main.css index 1fcc525..aa00fec 100644 --- a/src/renderer/src/assets/main.css +++ b/src/renderer/src/assets/main.css @@ -3429,6 +3429,199 @@ body { } } +.ai-vision-test { + display: grid; + gap: 16px; +} + +.ai-vision-test > header, +.ai-vision-test > footer, +.ai-vision-model { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; +} + +.ai-vision-test h2, +.ai-vision-test h3, +.ai-vision-test p { + margin: 0; +} + +.ai-vision-test h2 { + color: #202724; + font-size: 15px; +} + +.ai-vision-test header h3 { + margin-top: 12px; + color: #35403b; + font-size: 13px; +} + +.ai-vision-test header p, +.ai-vision-model, +.ai-vision-upload small { + margin-top: 4px; + color: #66706b; + font-size: 12px; +} + +.ai-vision-capability { + border-radius: 999px; + padding: 5px 9px; + background: #f1f3f2; + color: #66706b; + white-space: nowrap; + font-size: 12px; +} + +.ai-vision-capability.supported { + background: #eaf5f1; + color: #2e8b68; +} + +.ai-vision-model { + justify-content: flex-start; + margin: 0; +} + +.ai-vision-upload { + display: flex; + min-height: 112px; + align-items: center; + justify-content: center; + gap: 14px; + border: 1px dashed #bfc9c4; + border-radius: 10px; + padding: 14px; + background: #f7f9f8; + color: #35403b; + text-align: center; + cursor: pointer; +} + +.ai-vision-upload:hover { + border-color: #247a63; + background: #f2f8f5; +} + +.ai-vision-upload input { + display: none; +} + +.ai-vision-upload.has-image { + justify-content: flex-start; + text-align: left; +} + +.ai-vision-upload img { + width: 112px; + height: 82px; + flex: 0 0 auto; + border-radius: 8px; + object-fit: cover; +} + +.ai-vision-upload strong, +.ai-vision-upload small { + display: block; +} + +.ai-vision-prompt { + display: grid; + gap: 7px; + color: #46514c; + font-size: 12px; + font-weight: 600; +} + +.ai-vision-prompt textarea { + width: 100%; + border: 1px solid #d5ddda; + border-radius: 8px; + outline: none; + padding: 10px; + resize: vertical; + background: #f4f7f5; + color: #202724; + font: 13px/1.6 var(--wxex-font); +} + +.ai-vision-prompt textarea:focus { + border-color: #247a63; + box-shadow: 0 0 0 2px rgba(36, 122, 99, 0.08); +} + +.ai-vision-privacy { + color: #66706b; + font-size: 12px; + line-height: 1.6; +} + +.ai-vision-error { + border-left: 3px solid #c85a5a; + padding: 9px 11px; + background: #fff2f2; + color: #a84444; + font-size: 12px; +} + +.ai-vision-result { + display: grid; + gap: 12px; + border: 1px solid #dce7e2; + border-radius: 9px; + padding: 14px; + background: #f7faf9; +} + +.ai-vision-result dl { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 12px; + margin: 0; +} + +.ai-vision-result dt { + color: #929a96; + font-size: 11px; +} + +.ai-vision-result dd { + margin: 4px 0 0; + color: #35403b; + font-size: 12px; + font-weight: 600; +} + +.ai-vision-result > p { + white-space: pre-wrap; + color: #3d4742; + font-size: 13px; + line-height: 1.7; +} + +.ai-vision-test > footer { + justify-content: flex-end; +} + +.ai-vision-test > footer button { + min-height: 34px; + border: 1px solid #d5ddda; + border-radius: 7px; + padding: 0 13px; + background: #fff; + color: #46514c; + cursor: pointer; +} + +.ai-vision-test > footer button:disabled { + cursor: not-allowed; + opacity: 0.5; +} + .report-density-options button:disabled:hover { border-color: var(--wxex-border); background: var(--wxex-bg-elevated); diff --git a/src/renderer/src/features/settings/ai-model/AIImageUnderstandingTest.tsx b/src/renderer/src/features/settings/ai-model/AIImageUnderstandingTest.tsx new file mode 100644 index 0000000..79c571d --- /dev/null +++ b/src/renderer/src/features/settings/ai-model/AIImageUnderstandingTest.tsx @@ -0,0 +1,121 @@ +import type { AIProviderSummary, AIRuntimeModelConfig } from '../../../../../shared/ai-provider' +import type { AIVisionTestState } from './types' + +export function AIImageUnderstandingTest({ + runtime, + provider, + state, + onSelectImage, + onPromptChange, + onTest, + onClear +}: { + runtime: AIRuntimeModelConfig | null + provider?: AIProviderSummary + state: AIVisionTestState + onSelectImage: (file: File) => void + onPromptChange: (prompt: string) => void + onTest: () => void + onClear: () => void +}): React.ReactElement { + const model = provider?.models.find((item) => item.id === runtime?.model) + const testing = state.status === 'testing' + const result = state.result + return ( +
+
+
+

AI 能力测试

+

图片理解测试

+

上传图片验证当前模型是否支持视觉理解。

+
+ + 图片理解 {model?.capabilities.vision ? '✓' : '待验证'} + +
+ +
+ 当前供应商：{runtime?.providerName || '尚未配置'} + 当前模型：{runtime?.modelName || '尚未选择'} +
+ + + +