feat(forwarder): persist read images by content hash

2026-08-17 19:47:10 +08:00 · 2026-08-12 01:10:14 +08:00
parent 85a43115c7
commit 8f8d28880d
9 changed files with 685 additions and 168 deletions
@@ -2,8 +2,14 @@
 package execbridge

 import (
+	"bytes"
+	"crypto/sha256"
 	"encoding/json"
 	"fmt"
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	_ "image/png"
 	"net/http"
 	"strings"
 	"sync/atomic"
@@ -32,10 +38,18 @@ type ExecApplyResult struct {
 	ToolResultPayload string
 	// ToolCall 保存可用于发 ToolCallCompletedUpdate 的工具调用对象；当前仅对支持 ToolCall 的执行型工具可用。
 	ToolCall *agentv1.ToolCall
+	// ContentBlobs 保存需要在提交 history 前写入内容寻址存储的二进制内容。
+	ContentBlobs []ContentBlob
 	// ExecuteHookResponse 保存 execute hook 的结构化响应。
 	ExecuteHookResponse *agentv1.ExecuteHookResponse
 }

+// ContentBlob 表示由内容哈希稳定寻址的执行结果二进制数据。
+type ContentBlob struct {
+	ID   []byte
+	Data []byte
+}
+
 // OpenExecContext 表示执行桥打开请求时需要的最小上下文。
 type OpenExecContext struct {
 	ConversationID         string
@@ -146,6 +160,9 @@ func (bridge *Bridge) ApplyExecClientMessage(msg *agentv1.ExecClientMessage, pen
 		readResult := normalizeReadResultForModel(msg.GetReadResult())
 		result.ToolResultPayload = summarizeReadResult(readResult)
 		result.ToolCall = buildReadCompletedToolCall(pending.ToolCallID, pending.ArgsJSON, readResult)
+		if contentBlob, ok := readImageContentBlob(readResult); ok {
+			result.ContentBlobs = []ContentBlob{contentBlob}
+		}
 		result.IsTerminal = true
 		return result, nil
 	case "write":
@@ -2286,16 +2303,62 @@ func buildReadMcpResourceCompletedToolCall(argsJSON []byte, result *agentv1.Read
 	}
 }

-func isSupportedReadImage(data []byte) bool {
+func supportedReadImageMIMEType(data []byte) string {
 	if len(data) == 0 {
-		return false
+		return ""
 	}
-	switch strings.ToLower(strings.TrimSpace(http.DetectContentType(data))) {
-	case "image/png", "image/jpeg", "image/gif", "image/webp":
-		return true
-	default:
-		return false
+	detected := strings.ToLower(strings.TrimSpace(http.DetectContentType(data)))
+	configuration, format, err := image.DecodeConfig(bytes.NewReader(data))
+	if err != nil || configuration.Width <= 0 || configuration.Height <= 0 {
+		return ""
 	}
+	switch strings.ToLower(strings.TrimSpace(format)) {
+	case "png":
+		if detected == "image/png" {
+			return detected
+		}
+	case "jpeg":
+		if detected == "image/jpeg" {
+			return detected
+		}
+	case "gif":
+		if detected == "image/gif" {
+			return detected
+		}
+	}
+	return ""
+}
+
+func readImageContentBlob(result *agentv1.ReadResult) (ContentBlob, bool) {
+	success := result.GetSuccess()
+	if success == nil {
+		return ContentBlob{}, false
+	}
+	data := success.GetData()
+	if supportedReadImageMIMEType(data) == "" {
+		return ContentBlob{}, false
+	}
+	digest := sha256.Sum256(data)
+	return ContentBlob{
+		ID:   append([]byte(nil), digest[:]...),
+		Data: append([]byte(nil), data...),
+	}, true
+}
+
+func readImageBlobID(data []byte) ([]byte, bool) {
+	if supportedReadImageMIMEType(data) == "" {
+		return nil, false
+	}
+	digest := sha256.Sum256(data)
+	return append([]byte(nil), digest[:]...), true
+}
+
+func readImageDataBlobOutput(data []byte) *agentv1.ReadToolSuccess_DataBlobId {
+	blobID, ok := readImageBlobID(data)
+	if !ok {
+		return nil
+	}
+	return &agentv1.ReadToolSuccess_DataBlobId{DataBlobId: blobID}
 }

 // convertReadResultToReadToolResult 把 `ReadResult` 映射为 `ReadToolResult`。
@@ -2331,8 +2394,8 @@ func convertReadResultToReadToolResult(result *agentv1.ReadResult) *agentv1.Read
 		if content != "" {
 			toolSuccess.Output = &agentv1.ReadToolSuccess_Content{Content: content}
 		} else if len(data) > 0 {
-			if isSupportedReadImage(data) {
-				toolSuccess.Output = &agentv1.ReadToolSuccess_Data{Data: append([]byte(nil), data...)}
+			if imageOutput := readImageDataBlobOutput(data); imageOutput != nil {
+				toolSuccess.Output = imageOutput
 			} else if len(data) > readReplayBinaryLimit {
 				toolSuccess.ExceededLimit = true
 				toolSuccess.Output = &agentv1.ReadToolSuccess_Content{
@@ -2,31 +2,94 @@ package execbridge

 import (
 	"bytes"
+	"crypto/sha256"
+	"image"
+	"image/color"
+	"image/png"
 	"strings"
 	"testing"

 	"cursor/gen/agentv1"
+	runtimecore "cursor/internal/backend/agent/core"
 )

-func TestConvertReadResultPreservesLargeImagesOnly(t *testing.T) {
-	largePNG := append([]byte("\x89PNG\r\n\x1a\n"), bytes.Repeat([]byte{0}, readReplayBinaryLimit)...)
-	imageResult := convertReadResultToReadToolResult(&agentv1.ReadResult{
+func TestApplyExecClientMessageReturnsContentAddressedReadImage(t *testing.T) {
+	imageData := validReadTestPNG(t)
+	wantBlobID := sha256.Sum256(imageData)
+	result, err := NewBridge().ApplyExecClientMessage(&agentv1.ExecClientMessage{
+		Message: &agentv1.ExecClientMessage_ReadResult{
+			ReadResult: &agentv1.ReadResult{
+				Result: &agentv1.ReadResult_Success{
+					Success: &agentv1.ReadSuccess{
+						Path:         "diagram.png",
+						FileSize:     int64(len(imageData)),
+						OutputBlobId: append([]byte(nil), wantBlobID[:]...),
+						Output:       &agentv1.ReadSuccess_Data{Data: imageData},
+					},
+				},
+			},
+		},
+	}, runtimecore.PendingExec{
+		ExecKind:   "read",
+		ToolCallID: "call-1",
+		ArgsJSON:   []byte(`{"path":"diagram.png"}`),
+	})
+	if err != nil {
+		t.Fatalf("ApplyExecClientMessage() error = %v", err)
+	}
+	if len(result.ContentBlobs) != 1 {
+		t.Fatalf("content blob count = %d, want 1", len(result.ContentBlobs))
+	}
+	if !bytes.Equal(result.ContentBlobs[0].ID, wantBlobID[:]) || !bytes.Equal(result.ContentBlobs[0].Data, imageData) {
+		t.Fatalf("content blob = %#v", result.ContentBlobs[0])
+	}
+	readSuccess := result.ToolCall.GetReadToolCall().GetResult().GetSuccess()
+	if readSuccess == nil {
+		t.Fatal("read tool result is not successful")
+	}
+	if !bytes.Equal(readSuccess.GetDataBlobId(), wantBlobID[:]) {
+		t.Fatalf("data_blob_id = %x, want %x", readSuccess.GetDataBlobId(), wantBlobID)
+	}
+	if len(readSuccess.GetData()) != 0 {
+		t.Fatalf("read tool result retained %d image bytes", len(readSuccess.GetData()))
+	}
+}
+
+func TestApplyExecClientMessageUsesComputedImageBlobID(t *testing.T) {
+	imageData := validReadTestPNG(t)
+	wantBlobID := sha256.Sum256(imageData)
+	result, err := NewBridge().ApplyExecClientMessage(&agentv1.ExecClientMessage{
+		Message: &agentv1.ExecClientMessage_ReadResult{
+			ReadResult: &agentv1.ReadResult{
+				Result: &agentv1.ReadResult_Success{
+					Success: &agentv1.ReadSuccess{
+						Path:         "diagram.png",
+						OutputBlobId: bytes.Repeat([]byte{0xff}, sha256.Size),
+						Output:       &agentv1.ReadSuccess_Data{Data: imageData},
+					},
+				},
+			},
+		},
+	}, runtimecore.PendingExec{ExecKind: "read", ToolCallID: "call-1"})
+	if err != nil {
+		t.Fatalf("ApplyExecClientMessage() error = %v", err)
+	}
+	if !bytes.Equal(result.ContentBlobs[0].ID, wantBlobID[:]) {
+		t.Fatalf("content blob id = %x, want computed %x", result.ContentBlobs[0].ID, wantBlobID)
+	}
+}
+
+func TestConvertReadResultKeepsTextAndLimitsUnsupportedBinary(t *testing.T) {
+	textResult := convertReadResultToReadToolResult(&agentv1.ReadResult{
 		Result: &agentv1.ReadResult_Success{
 			Success: &agentv1.ReadSuccess{
-				Path:   "image.png",
-				Output: &agentv1.ReadSuccess_Data{Data: largePNG},
+				Path:   "notes.txt",
+				Output: &agentv1.ReadSuccess_Content{Content: "hello"},
 			},
 		},
 	})
-	imageSuccess := imageResult.GetSuccess()
-	if imageSuccess == nil {
-		t.Fatal("large image result is not successful")
-	}
-	if !bytes.Equal(imageSuccess.GetData(), largePNG) {
-		t.Fatalf("large image data bytes = %d, want %d", len(imageSuccess.GetData()), len(largePNG))
-	}
-	if imageSuccess.GetExceededLimit() {
-		t.Fatal("large image unexpectedly marked as exceeded limit")
+	if got := textResult.GetSuccess().GetContent(); got != "hello" {
+		t.Fatalf("text read content = %q, want hello", got)
 	}

 	largeBinary := bytes.Repeat([]byte{0xff}, readReplayBinaryLimit+1)
@@ -39,16 +102,24 @@ func TestConvertReadResultPreservesLargeImagesOnly(t *testing.T) {
 		},
 	})
 	binarySuccess := binaryResult.GetSuccess()
-	if binarySuccess == nil {
-		t.Fatal("large binary result is not successful")
+	if binarySuccess == nil || !binarySuccess.GetExceededLimit() {
+		t.Fatal("large non-image binary was not limited")
 	}
-	if !binarySuccess.GetExceededLimit() {
-		t.Fatal("large non-image binary was not marked as exceeded limit")
-	}
-	if binarySuccess.GetData() != nil {
-		t.Fatal("large non-image binary data was retained")
+	if binarySuccess.GetData() != nil || binarySuccess.GetDataBlobId() != nil {
+		t.Fatal("large non-image binary was retained")
 	}
 	if !strings.Contains(binarySuccess.GetContent(), "Read binary data") {
 		t.Fatalf("large binary fallback = %q", binarySuccess.GetContent())
 	}
 }
+
+func validReadTestPNG(t *testing.T) []byte {
+	t.Helper()
+	value := image.NewRGBA(image.Rect(0, 0, 2, 2))
+	value.Set(0, 0, color.RGBA{R: 0x44, G: 0x88, B: 0xcc, A: 0xff})
+	var encoded bytes.Buffer
+	if err := png.Encode(&encoded, value); err != nil {
+		t.Fatalf("encode test png: %v", err)
+	}
+	return encoded.Bytes()
+}