From b475166ba848d8107e51f5f3a2dcb5c2f30bb9a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=83=91=E9=9D=9E?= Date: Thu, 6 Aug 2026 21:05:08 +0800 Subject: [PATCH] Support read image --- build/windows/Taskfile.yml | 315 ++++++++++--------- frontend/src/i18n/generated/catalog.json | 56 ++-- internal/backend/agent/bridge/exec/bridge.go | 17 +- internal/backend/agent/model/anthropic.go | 21 +- internal/backend/agent/model/openai.go | 10 +- internal/backend/forwarder/projector.go | 65 +++- 6 files changed, 291 insertions(+), 193 deletions(-) diff --git a/build/windows/Taskfile.yml b/build/windows/Taskfile.yml index 8d98569..39fc923 100644 --- a/build/windows/Taskfile.yml +++ b/build/windows/Taskfile.yml @@ -1,156 +1,159 @@ -version: "3" - -includes: - common: ../Taskfile.yml - -tasks: - build: - summary: 构建 Windows 程序 - internal: true - cmds: - - task: build:native - vars: - ARCH: - ref: .ARCH - OUTPUT: - ref: .OUTPUT - BUILD_FLAGS: - ref: .BUILD_FLAGS - EXTRA_TAGS: - ref: .EXTRA_TAGS - DEV: - ref: .DEV - SCAN: - ref: .SCAN - - build:native: - summary: 使用 Go 原生交叉编译构建 Windows 程序 - internal: true - deps: - - task: common:go:mod:tidy - - task: common:build:frontend - vars: - BUILD_FLAGS: - ref: .BUILD_FLAGS - DEV: - ref: .DEV - SCAN: - ref: .SCAN - - task: common:generate:icons - cmds: - - task: generate:syso - vars: - ARCH: - ref: .ARCH - - go build {{.BUILD_FLAGS}} -o "{{.OUTPUT}}" - - cmd: powershell Remove-item *.syso - platforms: [windows] - - cmd: rm -f *.syso - platforms: [darwin, linux] - vars: - OUTPUT: '{{.OUTPUT | default (printf "%s/%s.exe" .BIN_DIR .APP_NAME)}}' - BUILD_FLAGS: '{{if eq .DEV "true"}}{{if .EXTRA_TAGS}}-tags {{.EXTRA_TAGS}} {{end}}-buildvcs=false -gcflags=all="-l" -ldflags="-X cursor/internal/buildinfo.Version={{.APP_VERSION}}"{{else}}-tags production{{if .EXTRA_TAGS}},{{.EXTRA_TAGS}}{{end}} -trimpath -buildvcs=false -ldflags="-w -s -H windowsgui -X cursor/internal/buildinfo.Version={{.APP_VERSION}}"{{end}}' - env: - GOOS: windows - CGO_ENABLED: 0 - GOARCH: '{{.ARCH | default ARCH}}' - - package: - summary: 打包 Windows 安装程序 - internal: true - cmds: - - task: '{{if eq (.FORMAT | default "nsis") "msix"}}create:msix:package{{else}}create:nsis:installer{{end}}' - vars: - FORMAT: '{{.FORMAT | default "nsis"}}' - - generate:syso: - summary: 生成 Windows 图标与版本信息资源 - internal: true - dir: build - cmds: - - wails3 generate syso -arch {{.ARCH}} -icon windows/icon.ico -manifest windows/wails.exe.manifest -info windows/info.json -out ../wails_windows_{{.ARCH}}.syso - vars: - ARCH: '{{.ARCH | default ARCH}}' - - create:nsis:installer: - summary: 生成 NSIS 安装包 - internal: true - dir: build/windows/nsis - deps: - - task: build - vars: - SCAN: - ref: .SCAN - cmds: - - wails3 generate webview2bootstrapper -dir "{{.ROOT_DIR}}/build/windows/nsis" - - | - {{if eq OS "windows"}} - makensis -DARG_WAILS_{{.ARG_FLAG}}_BINARY="{{.ROOT_DIR}}\{{.BIN_DIR}}\{{.APP_NAME}}.exe" project.nsi - {{else}} - makensis -DARG_WAILS_{{.ARG_FLAG}}_BINARY="{{.ROOT_DIR}}/{{.BIN_DIR}}/{{.APP_NAME}}.exe" project.nsi - {{end}} - vars: - ARCH: '{{.ARCH | default ARCH}}' - ARG_FLAG: '{{if eq .ARCH "amd64"}}AMD64{{else}}ARM64{{end}}' - - create:msix:package: - summary: 生成 MSIX 安装包 - internal: true - deps: - - task: build - vars: - SCAN: - ref: .SCAN - cmds: - - |- - wails3 tool msix \ - --config "{{.ROOT_DIR}}/wails.json" \ - --name "{{.APP_NAME}}" \ - --executable "{{.ROOT_DIR}}/{{.BIN_DIR}}/{{.APP_NAME}}.exe" \ - --arch "{{.ARCH}}" \ - --out "{{.ROOT_DIR}}/{{.BIN_DIR}}/{{.APP_NAME}}-{{.ARCH}}.msix" \ - {{if .CERT_PATH}}--cert "{{.CERT_PATH}}"{{end}} \ - {{if .PUBLISHER}}--publisher "{{.PUBLISHER}}"{{end}} \ - {{if .USE_MSIX_TOOL}}--use-msix-tool{{else}}--use-makeappx{{end}} - vars: - ARCH: '{{.ARCH | default ARCH}}' - CERT_PATH: '{{.CERT_PATH | default ""}}' - PUBLISHER: '{{.PUBLISHER | default ""}}' - USE_MSIX_TOOL: '{{.USE_MSIX_TOOL | default "false"}}' - - create:zip: - summary: 生成 Windows ZIP 包（包含 exe 与 certs） - internal: true - vars: - ARCH: '{{.ARCH | default ARCH}}' - OUTPUT: '{{.OUTPUT | default (printf "%s/%s.exe" .BIN_DIR .APP_NAME)}}' - ZIP_NAME: '{{.ZIP_NAME | default (printf "%s-windows-%s.zip" .APP_NAME .ARCH)}}' - STAGING_DIR: '{{.BIN_DIR}}/.zip-{{.ARCH}}' - cmds: - - task: build - vars: - ARCH: - ref: .ARCH - OUTPUT: - ref: .OUTPUT - BUILD_FLAGS: - ref: .BUILD_FLAGS - EXTRA_TAGS: - ref: .EXTRA_TAGS - DEV: - ref: .DEV - SCAN: - ref: .SCAN - - rm -rf "{{.STAGING_DIR}}" - - mkdir -p "{{.STAGING_DIR}}" - - cp "{{.OUTPUT}}" "{{.STAGING_DIR}}/" - - (cd "{{.STAGING_DIR}}" && zip -qry "../{{.ZIP_NAME}}" .) - - rm -rf "{{.STAGING_DIR}}" - - rm -f "{{.OUTPUT}}" - - run: - summary: 运行 Windows 可执行文件（仅在 Windows 可用） - vars: - OUTPUT: '{{.OUTPUT | default (printf "%s/%s.exe" .BIN_DIR .APP_NAME)}}' - cmds: - - '{{.OUTPUT}}' +version: "3" + +includes: + common: ../Taskfile.yml + +tasks: + build: + summary: 构建 Windows 程序 + internal: true + cmds: + - task: build:native + vars: + ARCH: + ref: .ARCH + OUTPUT: + ref: .OUTPUT + BUILD_FLAGS: + ref: .BUILD_FLAGS + EXTRA_TAGS: + ref: .EXTRA_TAGS + DEV: + ref: .DEV + SCAN: + ref: .SCAN + + build:native: + summary: 使用 Go 原生交叉编译构建 Windows 程序 + internal: true + deps: + - task: common:go:mod:tidy + - task: common:build:frontend + vars: + BUILD_FLAGS: + ref: .BUILD_FLAGS + DEV: + ref: .DEV + SCAN: + ref: .SCAN + - task: common:generate:icons + cmds: + - task: generate:syso + vars: + ARCH: + ref: .ARCH + - go build {{.BUILD_FLAGS}} -o "{{.OUTPUT}}" + - cmd: powershell Remove-item *.syso + platforms: [windows] + - cmd: rm -f *.syso + platforms: [darwin, linux] + vars: + OUTPUT: '{{.OUTPUT | default (printf "%s/%s.exe" .BIN_DIR .APP_NAME)}}' + BUILD_FLAGS: '{{if eq .DEV "true"}}{{if .EXTRA_TAGS}}-tags {{.EXTRA_TAGS}} {{end}}-buildvcs=false -gcflags=all="-l" -ldflags="-X cursor/internal/buildinfo.Version={{.APP_VERSION}}"{{else}}-tags production{{if .EXTRA_TAGS}},{{.EXTRA_TAGS}}{{end}} -trimpath -buildvcs=false -ldflags="-w -s -H windowsgui -X cursor/internal/buildinfo.Version={{.APP_VERSION}}"{{end}}' + env: + GOOS: windows + CGO_ENABLED: 0 + GOARCH: '{{.ARCH | default ARCH}}' + + package: + summary: 打包 Windows 安装程序 + internal: true + cmds: + - task: '{{if eq (.FORMAT | default "nsis") "msix"}}create:msix:package{{else}}create:nsis:installer{{end}}' + vars: + FORMAT: '{{.FORMAT | default "nsis"}}' + + generate:syso: + summary: 生成 Windows 图标与版本信息资源 + internal: true + dir: build + cmds: + - wails3 generate syso -arch {{.ARCH}} -icon windows/icon.ico -manifest windows/wails.exe.manifest -info windows/info.json -out ../wails_windows_{{.ARCH}}.syso + vars: + ARCH: '{{.ARCH | default ARCH}}' + + create:nsis:installer: + summary: 生成 NSIS 安装包 + internal: true + dir: build/windows/nsis + deps: + - task: build + vars: + SCAN: + ref: .SCAN + cmds: + - wails3 generate webview2bootstrapper -dir "{{.ROOT_DIR}}/build/windows/nsis" + - | + {{if eq OS "windows"}} + makensis -DARG_WAILS_{{.ARG_FLAG}}_BINARY="{{.ROOT_DIR}}\{{.BIN_DIR}}\{{.APP_NAME}}.exe" project.nsi + {{else}} + makensis -DARG_WAILS_{{.ARG_FLAG}}_BINARY="{{.ROOT_DIR}}/{{.BIN_DIR}}/{{.APP_NAME}}.exe" project.nsi + {{end}} + vars: + ARCH: '{{.ARCH | default ARCH}}' + ARG_FLAG: '{{if eq .ARCH "amd64"}}AMD64{{else}}ARM64{{end}}' + + create:msix:package: + summary: 生成 MSIX 安装包 + internal: true + deps: + - task: build + vars: + SCAN: + ref: .SCAN + cmds: + - |- + wails3 tool msix \ + --config "{{.ROOT_DIR}}/wails.json" \ + --name "{{.APP_NAME}}" \ + --executable "{{.ROOT_DIR}}/{{.BIN_DIR}}/{{.APP_NAME}}.exe" \ + --arch "{{.ARCH}}" \ + --out "{{.ROOT_DIR}}/{{.BIN_DIR}}/{{.APP_NAME}}-{{.ARCH}}.msix" \ + {{if .CERT_PATH}}--cert "{{.CERT_PATH}}"{{end}} \ + {{if .PUBLISHER}}--publisher "{{.PUBLISHER}}"{{end}} \ + {{if .USE_MSIX_TOOL}}--use-msix-tool{{else}}--use-makeappx{{end}} + vars: + ARCH: '{{.ARCH | default ARCH}}' + CERT_PATH: '{{.CERT_PATH | default ""}}' + PUBLISHER: '{{.PUBLISHER | default ""}}' + USE_MSIX_TOOL: '{{.USE_MSIX_TOOL | default "false"}}' + + create:zip: + summary: 生成 Windows ZIP 包（包含 exe） + internal: true + vars: + ARCH: '{{.ARCH | default ARCH}}' + OUTPUT: '{{.OUTPUT | default (printf "%s/%s.exe" .BIN_DIR .APP_NAME)}}' + ZIP_NAME: '{{.ZIP_NAME | default (printf "%s-windows-%s.zip" .APP_NAME .ARCH)}}' + STAGING_DIR: '{{.BIN_DIR}}/.zip-{{.ARCH}}' + cmds: + - task: build + vars: + ARCH: + ref: .ARCH + OUTPUT: + ref: .OUTPUT + BUILD_FLAGS: + ref: .BUILD_FLAGS + EXTRA_TAGS: + ref: .EXTRA_TAGS + DEV: + ref: .DEV + SCAN: + ref: .SCAN + - rm -rf "{{.STAGING_DIR}}" + - mkdir -p "{{.STAGING_DIR}}" + - cp "{{.OUTPUT}}" "{{.STAGING_DIR}}/" + - cmd: powershell -NoProfile -Command "Compress-Archive -Path '{{.STAGING_DIR}}/*' -DestinationPath '{{.BIN_DIR}}/{{.ZIP_NAME}}' -Force" + platforms: [windows] + - cmd: (cd "{{.STAGING_DIR}}" && zip -qry "../{{.ZIP_NAME}}" .) + platforms: [darwin, linux] + - rm -rf "{{.STAGING_DIR}}" + - rm -f "{{.OUTPUT}}" + + run: + summary: 运行 Windows 可执行文件（仅在 Windows 可用） + vars: + OUTPUT: '{{.OUTPUT | default (printf "%s/%s.exe" .BIN_DIR .APP_NAME)}}' + cmds: + - '{{.OUTPUT}}' diff --git a/frontend/src/i18n/generated/catalog.json b/frontend/src/i18n/generated/catalog.json index ce0e195..2dbf0cd 100644 --- a/frontend/src/i18n/generated/catalog.json +++ b/frontend/src/i18n/generated/catalog.json @@ -19,8 +19,8 @@ "refs": [ { "file": "src/views/ModelConfig.vue", - "line": 256, - "column": 1 + "line": 255, + "column": 165 } ] }, @@ -137,8 +137,8 @@ "refs": [ { "file": "src/views/Config.vue", - "line": 54, - "column": 1 + "line": 53, + "column": 48 } ] }, @@ -173,8 +173,8 @@ "refs": [ { "file": "src/components/HomeMetricsCard.vue", - "line": 338, - "column": 1 + "line": 337, + "column": 65 } ] }, @@ -400,8 +400,8 @@ "refs": [ { "file": "src/views/Config.vue", - "line": 68, - "column": 1 + "line": 67, + "column": 48 } ] }, @@ -541,8 +541,8 @@ "refs": [ { "file": "src/components/ModelAdapterTestCard.vue", - "line": 151, - "column": 1 + "line": 150, + "column": 7 } ] }, @@ -589,8 +589,8 @@ "refs": [ { "file": "src/components/ModelAdapterTestCard.vue", - "line": 130, - "column": 1 + "line": 129, + "column": 60 } ] }, @@ -698,8 +698,8 @@ }, { "file": "src/components/CursorAccountCard.vue", - "line": 164, - "column": 1 + "line": 163, + "column": 11 } ] }, @@ -948,8 +948,8 @@ "refs": [ { "file": "src/components/ModelAdapterTestCard.vue", - "line": 125, - "column": 1 + "line": 124, + "column": 9 } ] }, @@ -1569,8 +1569,8 @@ "refs": [ { "file": "src/components/HomeMetricsCard.vue", - "line": 390, - "column": 1 + "line": 389, + "column": 65 } ] }, @@ -1822,8 +1822,8 @@ "refs": [ { "file": "src/components/HomeMetricsCard.vue", - "line": 342, - "column": 1 + "line": 341, + "column": 23 } ] }, @@ -1995,8 +1995,8 @@ "refs": [ { "file": "src/views/ModelEditor.vue", - "line": 449, - "column": 1 + "line": 448, + "column": 97 } ] }, @@ -2772,8 +2772,8 @@ "refs": [ { "file": "src/components/CursorAccountCard.vue", - "line": 146, - "column": 1 + "line": 145, + "column": 53 } ] }, @@ -2905,8 +2905,8 @@ "refs": [ { "file": "src/views/Config.vue", - "line": 80, - "column": 1 + "line": 79, + "column": 48 } ] }, @@ -2982,8 +2982,8 @@ "refs": [ { "file": "src/components/CursorAccountCard.vue", - "line": 149, - "column": 1 + "line": 148, + "column": 81 } ] }, diff --git a/internal/backend/agent/bridge/exec/bridge.go b/internal/backend/agent/bridge/exec/bridge.go index eff9a3d..32a8d47 100644 --- a/internal/backend/agent/bridge/exec/bridge.go +++ b/internal/backend/agent/bridge/exec/bridge.go @@ -4,6 +4,7 @@ package execbridge import ( "encoding/json" "fmt" + "net/http" "strings" "sync/atomic" "time" @@ -2285,6 +2286,18 @@ func buildReadMcpResourceCompletedToolCall(argsJSON []byte, result *agentv1.Read } } +func isSupportedReadImage(data []byte) bool { + if len(data) == 0 { + return false + } + switch strings.ToLower(strings.TrimSpace(http.DetectContentType(data))) { + case "image/png", "image/jpeg", "image/gif", "image/webp": + return true + default: + return false + } +} + // convertReadResultToReadToolResult 把 `ReadResult` 映射为 `ReadToolResult`。 func convertReadResultToReadToolResult(result *agentv1.ReadResult) *agentv1.ReadToolResult { if result == nil { @@ -2318,7 +2331,9 @@ func convertReadResultToReadToolResult(result *agentv1.ReadResult) *agentv1.Read if content != "" { toolSuccess.Output = &agentv1.ReadToolSuccess_Content{Content: content} } else if len(data) > 0 { - if len(data) > readReplayBinaryLimit { + if isSupportedReadImage(data) { + toolSuccess.Output = &agentv1.ReadToolSuccess_Data{Data: append([]byte(nil), data...)} + } else if len(data) > readReplayBinaryLimit { toolSuccess.ExceededLimit = true toolSuccess.Output = &agentv1.ReadToolSuccess_Content{ Content: replayTruncationNotice("Read binary data", readReplayBinaryLimit, 0, len(data)), diff --git a/internal/backend/agent/model/anthropic.go b/internal/backend/agent/model/anthropic.go index a409b2b..b2208a9 100644 --- a/internal/backend/agent/model/anthropic.go +++ b/internal/backend/agent/model/anthropic.go @@ -1126,7 +1126,16 @@ func isAnthropicCacheableBlock(block map[string]any) bool { case contentPartTypeText: return strings.TrimSpace(anthropicStringField(block, "text")) != "" case "tool_result": - return strings.TrimSpace(anthropicStringField(block, "content")) != "" + switch content := block["content"].(type) { + case string: + return strings.TrimSpace(content) != "" + case []map[string]any: + return len(content) > 0 + case []any: + return len(content) > 0 + default: + return false + } case "tool_use": return strings.TrimSpace(anthropicStringField(block, "id")) != "" && strings.TrimSpace(anthropicStringField(block, "name")) != "" default: @@ -1168,10 +1177,18 @@ func normalizeAnthropicProviderMessages(input []Message, thinkingEnabled bool, r if toolUseID == "" { return nil, nil, fmt.Errorf("anthropic tool message requires tool_call_id") } + var content any = message.Content + if hasImageContentParts(message.ContentParts) { + contentBlocks, err := anthropicContentBlocks(message) + if err != nil { + return nil, nil, err + } + content = contentBlocks + } pendingToolResults = append(pendingToolResults, map[string]any{ "type": "tool_result", "tool_use_id": toolUseID, - "content": message.Content, + "content": content, }) case "user", "assistant": flushToolResults() diff --git a/internal/backend/agent/model/openai.go b/internal/backend/agent/model/openai.go index 25bf721..95725da 100644 --- a/internal/backend/agent/model/openai.go +++ b/internal/backend/agent/model/openai.go @@ -1967,10 +1967,18 @@ func normalizeOpenAIResponsesInput(messages []Message) (string, []map[string]any } if role == "tool" && strings.TrimSpace(message.ToolCallID) != "" { callID := openAIResponsesToolMessageCallID(message, responsesCallIDs) + var output any = openAIResponsesMessageText(message) + if hasImageContentParts(message.ContentParts) { + content, err := openAIResponsesMessageContent(message, false) + if err != nil { + return "", nil, err + } + output = content + } items = append(items, map[string]any{ "type": "function_call_output", "call_id": callID, - "output": openAIResponsesMessageText(message), + "output": output, }) activeAssistantReasoningKey = "" continue diff --git a/internal/backend/forwarder/projector.go b/internal/backend/forwarder/projector.go index 7095860..7415afe 100644 --- a/internal/backend/forwarder/projector.go +++ b/internal/backend/forwarder/projector.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/json" "fmt" + "net/http" "strings" "google.golang.org/protobuf/encoding/protojson" @@ -215,6 +216,7 @@ func (projector *HistoryProjector) ProjectPromptReplay(conversation *Conversatio if ok { replayMessage.Name = toolName replayMessage.Content = limitProjectedToolResultReplay(toolName, replayMessage.Content, payload.ResultText, true, historicalToolResult) + attachReadImageContentParts(&replayMessage, toolCall) messages = append(messages, toModelMessage(replayMessage)) continue } @@ -249,12 +251,13 @@ func (projector *HistoryProjector) ProjectPromptReplay(conversation *Conversatio replayMessages[index].OpenAIResponsesReasoningSummary = append(json.RawMessage(nil), payload.ReasoningSummary...) applyPromptProviderMetadataToFirstToolCall(&replayMessages[index], payload.ProviderItemID, payload.ProviderCallID, payload.ProviderStatus) } - for _, replay := range replayMessages { - if strings.TrimSpace(replay.Role) == "tool" { - toolName := firstNonEmpty(strings.TrimSpace(replay.Name), strings.TrimSpace(payload.ToolName)) - replay.Content = limitProjectedToolResultReplay(toolName, replay.Content, payload.ResultText, true, historicalToolResult) + for index := range replayMessages { + if strings.TrimSpace(replayMessages[index].Role) == "tool" { + toolName := firstNonEmpty(strings.TrimSpace(replayMessages[index].Name), strings.TrimSpace(payload.ToolName)) + replayMessages[index].Content = limitProjectedToolResultReplay(toolName, replayMessages[index].Content, payload.ResultText, true, historicalToolResult) + attachReadImageContentParts(&replayMessages[index], toolCall) } - messages = append(messages, toModelMessage(replay)) + messages = append(messages, toModelMessage(replayMessages[index])) } continue } @@ -305,6 +308,58 @@ func (projector *HistoryProjector) ProjectPromptReplay(conversation *Conversatio return normalizeReplayMessageSequence(messages), nil } +func attachReadImageContentParts(message *promptengine.Message, toolCall *agentv1.ToolCall) { + if message == nil || toolCall == nil || strings.TrimSpace(message.Role) != "tool" { + return + } + readToolCall := toolCall.GetReadToolCall() + if readToolCall == nil { + return + } + success := readToolCall.GetResult().GetSuccess() + if success == nil { + return + } + data := success.GetData() + mimeType := supportedReadImageMIMEType(data) + if mimeType == "" { + return + } + path := firstNonEmpty(strings.TrimSpace(success.GetPath()), strings.TrimSpace(readToolCall.GetArgs().GetPath())) + summary := fmt.Sprintf("read image path=%q mime=%s bytes=%d", path, mimeType, len(data)) + if fileSize := success.GetFileSize(); fileSize > 0 && uint64(fileSize) != uint64(len(data)) { + summary += fmt.Sprintf(" file_size=%d", fileSize) + } + if success.GetExceededLimit() { + summary += " truncated=true" + } + message.Content = summary + message.ContentParts = []promptengine.ContentPart{ + {Type: "text", Text: summary}, + { + Type: "image", + Image: &promptengine.ImageContent{ + MIMEType: mimeType, + Path: path, + Data: append([]byte(nil), data...), + }, + }, + } +} + +func supportedReadImageMIMEType(data []byte) string { + if len(data) == 0 { + return "" + } + mimeType := strings.ToLower(strings.TrimSpace(http.DetectContentType(data))) + switch mimeType { + case "image/png", "image/jpeg", "image/gif", "image/webp": + return mimeType + default: + return "" + } +} + func compactedPromptProjectionEntries(entries []HistoryEntry) []HistoryEntry { if len(entries) == 0 { return nil