diff --git a/internal/backend/agent/bridge/exec/bridge_read_image_test.go b/internal/backend/agent/bridge/exec/bridge_read_image_test.go new file mode 100644 index 0000000..966eca1 --- /dev/null +++ b/internal/backend/agent/bridge/exec/bridge_read_image_test.go @@ -0,0 +1,54 @@ +package execbridge + +import ( + "bytes" + "strings" + "testing" + + "cursor/gen/agentv1" +) + +func TestConvertReadResultPreservesLargeImagesOnly(t *testing.T) { + largePNG := append([]byte("\x89PNG\r\n\x1a\n"), bytes.Repeat([]byte{0}, readReplayBinaryLimit)...) + imageResult := convertReadResultToReadToolResult(&agentv1.ReadResult{ + Result: &agentv1.ReadResult_Success{ + Success: &agentv1.ReadSuccess{ + Path: "image.png", + Output: &agentv1.ReadSuccess_Data{Data: largePNG}, + }, + }, + }) + imageSuccess := imageResult.GetSuccess() + if imageSuccess == nil { + t.Fatal("large image result is not successful") + } + if !bytes.Equal(imageSuccess.GetData(), largePNG) { + t.Fatalf("large image data bytes = %d, want %d", len(imageSuccess.GetData()), len(largePNG)) + } + if imageSuccess.GetExceededLimit() { + t.Fatal("large image unexpectedly marked as exceeded limit") + } + + largeBinary := bytes.Repeat([]byte{0xff}, readReplayBinaryLimit+1) + binaryResult := convertReadResultToReadToolResult(&agentv1.ReadResult{ + Result: &agentv1.ReadResult_Success{ + Success: &agentv1.ReadSuccess{ + Path: "archive.bin", + Output: &agentv1.ReadSuccess_Data{Data: largeBinary}, + }, + }, + }) + binarySuccess := binaryResult.GetSuccess() + if binarySuccess == nil { + t.Fatal("large binary result is not successful") + } + if !binarySuccess.GetExceededLimit() { + t.Fatal("large non-image binary was not marked as exceeded limit") + } + if binarySuccess.GetData() != nil { + t.Fatal("large non-image binary data was retained") + } + if !strings.Contains(binarySuccess.GetContent(), "Read binary data") { + t.Fatalf("large binary fallback = %q", binarySuccess.GetContent()) + } +} diff --git a/internal/backend/agent/model/tool_image_test.go b/internal/backend/agent/model/tool_image_test.go new file mode 100644 index 0000000..36ab108 --- /dev/null +++ b/internal/backend/agent/model/tool_image_test.go @@ -0,0 +1,86 @@ +package modeladapter + +import ( + "strings" + "testing" +) + +func TestToolImageProviderEncodings(t *testing.T) { + message := toolImageMessageForTest() + + t.Run("openai_chat", func(t *testing.T) { + items, err := normalizeOpenAIProviderMessages([]Message{message}, false) + if err != nil { + t.Fatalf("normalizeOpenAIProviderMessages() error = %v", err) + } + if len(items) != 1 || items[0]["role"] != "tool" || items[0]["tool_call_id"] != "call-1" { + t.Fatalf("openai chat tool message = %#v", items) + } + content, ok := items[0]["content"].([]map[string]any) + if !ok || len(content) != 2 { + t.Fatalf("openai chat content = %#v", items[0]["content"]) + } + imageURL, ok := content[1]["image_url"].(map[string]any) + if content[1]["type"] != "image_url" || !ok || !strings.HasPrefix(imageURL["url"].(string), "data:image/png;base64,") { + t.Fatalf("openai chat image part = %#v", content[1]) + } + }) + + t.Run("openai_responses", func(t *testing.T) { + _, items, err := normalizeOpenAIResponsesInput([]Message{message}) + if err != nil { + t.Fatalf("normalizeOpenAIResponsesInput() error = %v", err) + } + if len(items) != 1 || items[0]["type"] != "function_call_output" { + t.Fatalf("openai responses items = %#v", items) + } + content, ok := items[0]["output"].([]map[string]any) + if !ok || len(content) != 2 { + t.Fatalf("openai responses output = %#v", items[0]["output"]) + } + if content[0]["type"] != "input_text" || content[1]["type"] != "input_image" { + t.Fatalf("openai responses content = %#v", content) + } + }) + + t.Run("anthropic", func(t *testing.T) { + _, messages, err := normalizeAnthropicProviderMessages([]Message{message}, false, false) + if err != nil { + t.Fatalf("normalizeAnthropicProviderMessages() error = %v", err) + } + if len(messages) != 1 || messages[0].Role != "user" || len(messages[0].Content) != 1 { + t.Fatalf("anthropic messages = %#v", messages) + } + toolResult := messages[0].Content[0] + if toolResult["type"] != "tool_result" || toolResult["tool_use_id"] != "call-1" { + t.Fatalf("anthropic tool result = %#v", toolResult) + } + content, ok := toolResult["content"].([]map[string]any) + if !ok || len(content) != 2 { + t.Fatalf("anthropic tool content = %#v", toolResult["content"]) + } + if content[0]["type"] != "text" || content[1]["type"] != "image" { + t.Fatalf("anthropic content blocks = %#v", content) + } + }) +} + +func toolImageMessageForTest() Message { + return Message{ + Role: "tool", + Content: "read binary bytes=16", + ToolCallID: "call-1", + Name: "Read", + ContentParts: []ContentPart{ + {Type: "text", Text: "read binary bytes=16"}, + { + Type: "image", + Image: &ImageContent{ + MIMEType: "image/png", + Path: "diagram.png", + Data: []byte("\x89PNG\r\n\x1a\nimage"), + }, + }, + }, + } +} diff --git a/internal/backend/forwarder/projector_read_image_test.go b/internal/backend/forwarder/projector_read_image_test.go new file mode 100644 index 0000000..5f9d7e4 --- /dev/null +++ b/internal/backend/forwarder/projector_read_image_test.go @@ -0,0 +1,99 @@ +package forwarder + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "google.golang.org/protobuf/encoding/protojson" + + "cursor/gen/agentv1" +) + +func TestProjectPromptReplayAttachesReadImageToToolMessage(t *testing.T) { + testCases := []struct { + name string + imageData []byte + fileSize uint32 + wantSummary string + }{ + { + name: "small image omits result json base64", + imageData: append([]byte("\x89PNG\r\n\x1a\n"), bytes.Repeat([]byte{0}, 64)...), + fileSize: 391998, + wantSummary: `read image path="diagram.png" mime=image/png bytes=72 file_size=391998`, + }, + { + name: "large image omits replay truncation notice", + imageData: append([]byte("\x89PNG\r\n\x1a\n"), bytes.Repeat([]byte{0}, projectedReadReplayLimit)...), + fileSize: uint32(projectedReadReplayLimit + 8), + wantSummary: fmt.Sprintf(`read image path="diagram.png" mime=image/png bytes=%d`, projectedReadReplayLimit+8), + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + toolCall := &agentv1.ToolCall{ + Tool: &agentv1.ToolCall_ReadToolCall{ + ReadToolCall: &agentv1.ReadToolCall{ + Args: &agentv1.ReadToolArgs{Path: "diagram.png"}, + Result: &agentv1.ReadToolResult{ + Result: &agentv1.ReadToolResult_Success{ + Success: &agentv1.ReadToolSuccess{ + FileSize: testCase.fileSize, + Path: "diagram.png", + Output: &agentv1.ReadToolSuccess_Data{Data: testCase.imageData}, + }, + }, + }, + }, + }, + } + encodedToolCall, err := protojson.Marshal(toolCall) + if err != nil { + t.Fatalf("marshal read tool call: %v", err) + } + conversation := &ConversationFile{ + ConversationID: "conversation-1", + NextTurnSeq: 2, + Entries: []HistoryEntry{ + newToolResultEntry(1, "request-1", "call-1", "Read", `{"path":"diagram.png"}`, fmt.Sprintf("read binary bytes=%d", len(testCase.imageData)), "", encodedToolCall), + }, + } + + messages, err := NewHistoryProjector().ProjectPromptReplay(conversation) + if err != nil { + t.Fatalf("ProjectPromptReplay() error = %v", err) + } + if len(messages) != 2 { + t.Fatalf("message count = %d, want assistant tool call and tool result", len(messages)) + } + toolMessage := messages[1] + if toolMessage.Role != "tool" || toolMessage.ToolCallID != "call-1" || toolMessage.Name != "Read" { + t.Fatalf("tool message metadata = %#v", toolMessage) + } + if toolMessage.Content != testCase.wantSummary { + t.Fatalf("tool content = %q, want %q", toolMessage.Content, testCase.wantSummary) + } + for _, forbidden := range []string{`"data"`, "iVBOR", "base64", "tool result replay truncated"} { + if strings.Contains(toolMessage.Content, forbidden) { + t.Fatalf("tool content contains %q: %q", forbidden, toolMessage.Content) + } + } + if len(toolMessage.ContentParts) != 2 { + t.Fatalf("tool content parts = %d, want text and image", len(toolMessage.ContentParts)) + } + if toolMessage.ContentParts[0].Type != "text" || toolMessage.ContentParts[0].Text != testCase.wantSummary { + t.Fatalf("tool text part = %#v", toolMessage.ContentParts[0]) + } + image := toolMessage.ContentParts[1].Image + if toolMessage.ContentParts[1].Type != "image" || image == nil { + t.Fatalf("tool image part = %#v", toolMessage.ContentParts[1]) + } + if image.MIMEType != "image/png" || image.Path != "diagram.png" || !bytes.Equal(image.Data, testCase.imageData) { + t.Fatalf("tool image = %#v", image) + } + }) + } +}