mirror of
https://wget.la/https://github.com/leookun/cursor-byok
synced 2026-08-17 19:47:10 +08:00
feat(forwarder): make checkpoint recovery idempotent
Preserve interrupted provider output, keep checkpoint projections deterministic, and enforce proto snapshot synchronization.
This commit is contained in:
@@ -690,9 +690,21 @@ func appendEntriesInPlace(conversation *ConversationFile, entries []HistoryEntry
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
assigned := make([]HistoryEntry, 0, len(entries))
|
||||
existingIdempotencyKeys := make(map[string]struct{})
|
||||
for _, existing := range conversation.Entries {
|
||||
if key := strings.TrimSpace(existing.IdempotencyKey); key != "" {
|
||||
existingIdempotencyKeys[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
maxTurnSeq := conversation.NextTurnSeq - 1
|
||||
for _, entry := range entries {
|
||||
next := entry
|
||||
if key := strings.TrimSpace(next.IdempotencyKey); key != "" {
|
||||
if _, exists := existingIdempotencyKeys[key]; exists {
|
||||
continue
|
||||
}
|
||||
existingIdempotencyKeys[key] = struct{}{}
|
||||
}
|
||||
if next.CreatedAt.IsZero() {
|
||||
next.CreatedAt = now
|
||||
}
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
package forwarder
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAppendEntriesDeduplicatesIdempotencyKey(t *testing.T) {
|
||||
store := NewConversationFileStore(t.TempDir())
|
||||
entry := HistoryEntry{
|
||||
TurnSeq: 1,
|
||||
RequestID: "request-1",
|
||||
IdempotencyKey: "provider-interrupted-output:test",
|
||||
Role: "assistant",
|
||||
Kind: "assistant_text",
|
||||
Payload: json.RawMessage(`{"text":"partial"}`),
|
||||
}
|
||||
|
||||
if _, assigned, err := store.AppendEntries("conversation-1", []HistoryEntry{entry}); err != nil {
|
||||
t.Fatalf("first AppendEntries() error = %v", err)
|
||||
} else if len(assigned) != 1 {
|
||||
t.Fatalf("first AppendEntries() assigned = %d, want 1", len(assigned))
|
||||
}
|
||||
if _, assigned, err := store.AppendEntries("conversation-1", []HistoryEntry{entry}); err != nil {
|
||||
t.Fatalf("duplicate AppendEntries() error = %v", err)
|
||||
} else if len(assigned) != 0 {
|
||||
t.Fatalf("duplicate AppendEntries() assigned = %d, want 0", len(assigned))
|
||||
}
|
||||
|
||||
conversation, err := store.LoadConversation("conversation-1")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConversation() error = %v", err)
|
||||
}
|
||||
if len(conversation.Entries) != 1 {
|
||||
t.Fatalf("persisted entries = %d, want 1", len(conversation.Entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelPersistsInterruptedProviderOutputIdempotently(t *testing.T) {
|
||||
service, stream, _ := testCheckpointBlobProjection(t)
|
||||
conversation, _, _, err := service.snapshotCheckpointConversation(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshotCheckpointConversation() error = %v", err)
|
||||
}
|
||||
if _, err := service.store.SaveConversationWithEntries(stream.ConversationID, conversation, conversation.Entries); err != nil {
|
||||
t.Fatalf("SaveConversationWithEntries() error = %v", err)
|
||||
}
|
||||
|
||||
stream.mu.Lock()
|
||||
stream.CurrentModelCallID = "model-call-1"
|
||||
stream.ProviderAccumulatedText = "partial answer"
|
||||
stream.ProviderAccumulatedReasoning = "partial reasoning"
|
||||
stream.mu.Unlock()
|
||||
|
||||
cancel := InboundIntent{
|
||||
Kind: "cancel",
|
||||
RequestID: stream.RequestID,
|
||||
CancelReason: "[canceled] Superseded by newer request",
|
||||
}
|
||||
if err := service.handleCancelIntent(cancel); err != nil {
|
||||
t.Fatalf("first handleCancelIntent() error = %v", err)
|
||||
}
|
||||
stream.mu.Lock()
|
||||
stream.ProviderAccumulatedText = "late duplicate fragment"
|
||||
stream.mu.Unlock()
|
||||
if err := service.handleCancelIntent(cancel); err != nil {
|
||||
t.Fatalf("duplicate handleCancelIntent() error = %v", err)
|
||||
}
|
||||
|
||||
persisted, err := service.store.LoadConversation(stream.ConversationID)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConversation() error = %v", err)
|
||||
}
|
||||
assistantEntries := 0
|
||||
cancelEntries := 0
|
||||
for _, entry := range persisted.Entries {
|
||||
if entry.Kind == "metadata" {
|
||||
var payload metadataPayload
|
||||
if err := json.Unmarshal(entry.Payload, &payload); err != nil {
|
||||
t.Fatalf("decode metadata entry: %v", err)
|
||||
}
|
||||
if payload.Type == "control" && readStringValue(payload.Value["status"]) == "canceled" {
|
||||
cancelEntries++
|
||||
}
|
||||
}
|
||||
if entry.Kind != "assistant_text" {
|
||||
continue
|
||||
}
|
||||
var payload assistantTextPayload
|
||||
if err := json.Unmarshal(entry.Payload, &payload); err != nil {
|
||||
t.Fatalf("decode assistant entry: %v", err)
|
||||
}
|
||||
if payload.Text == "partial answer" {
|
||||
assistantEntries++
|
||||
}
|
||||
}
|
||||
if assistantEntries != 1 {
|
||||
t.Fatalf("persisted interrupted assistant entries = %d, want 1", assistantEntries)
|
||||
}
|
||||
if cancelEntries != 1 {
|
||||
t.Fatalf("persisted cancel metadata entries = %d, want 1", cancelEntries)
|
||||
}
|
||||
|
||||
replay, err := service.projector.ProjectPromptReplay(persisted)
|
||||
if err != nil {
|
||||
t.Fatalf("ProjectPromptReplay() error = %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, message := range replay {
|
||||
if message.Role == "assistant" && strings.TrimSpace(message.Content) == "partial answer" && strings.TrimSpace(message.ReasoningContent) == "partial reasoning" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("replay = %#v, want interrupted assistant output", replay)
|
||||
}
|
||||
checkpoint, err := service.projector.ProjectCheckpointProjection(persisted)
|
||||
if err != nil {
|
||||
t.Fatalf("ProjectCheckpointProjection() error = %v", err)
|
||||
}
|
||||
if checkpoint == nil || checkpoint.State == nil || len(checkpoint.State.GetTurns()) != 1 {
|
||||
t.Fatalf("checkpoint state = %#v, want interrupted turn", checkpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelPreservesPersistedTurnActivityWithoutLiveAccumulator(t *testing.T) {
|
||||
service, stream, _ := testCheckpointBlobProjection(t)
|
||||
conversation, _, _, err := service.snapshotCheckpointConversation(stream)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshotCheckpointConversation() error = %v", err)
|
||||
}
|
||||
if _, err := service.store.SaveConversationWithEntries(stream.ConversationID, conversation, conversation.Entries); err != nil {
|
||||
t.Fatalf("SaveConversationWithEntries() error = %v", err)
|
||||
}
|
||||
|
||||
if err := service.handleCancelIntent(InboundIntent{
|
||||
Kind: "cancel",
|
||||
RequestID: stream.RequestID,
|
||||
CancelReason: "new_message_submitted",
|
||||
}); err != nil {
|
||||
t.Fatalf("handleCancelIntent() error = %v", err)
|
||||
}
|
||||
|
||||
persisted, err := service.store.LoadConversation(stream.ConversationID)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConversation() error = %v", err)
|
||||
}
|
||||
replay, err := service.projector.ProjectPromptReplay(persisted)
|
||||
if err != nil {
|
||||
t.Fatalf("ProjectPromptReplay() error = %v", err)
|
||||
}
|
||||
for _, message := range replay {
|
||||
if message.Role == "assistant" && strings.TrimSpace(message.Content) == "hi" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("replay = %#v, want persisted assistant activity", replay)
|
||||
}
|
||||
|
||||
func TestProjectPromptReplayPreservesLegacyCanceledTurnActivity(t *testing.T) {
|
||||
cancelEntry := newMetadataEntry(1, "request-1", "control", map[string]any{
|
||||
"status": "canceled",
|
||||
"reason": "new_message_submitted",
|
||||
"replay_policy": cancelReplayPolicyKeepStableInput,
|
||||
})
|
||||
conversation := &ConversationFile{
|
||||
ConversationID: "conversation-1",
|
||||
NextTurnSeq: 2,
|
||||
Entries: []HistoryEntry{
|
||||
newAssistantTextEntry(1, "request-1", "persisted activity", "", ""),
|
||||
cancelEntry,
|
||||
},
|
||||
}
|
||||
|
||||
replay, err := NewHistoryProjector().ProjectPromptReplay(conversation)
|
||||
if err != nil {
|
||||
t.Fatalf("ProjectPromptReplay() error = %v", err)
|
||||
}
|
||||
for _, message := range replay {
|
||||
if message.Role == "assistant" && strings.TrimSpace(message.Content) == "persisted activity" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("replay = %#v, want legacy canceled activity", replay)
|
||||
}
|
||||
@@ -355,6 +355,7 @@ const (
|
||||
cancelReplayPolicyDropTurn = "drop_turn"
|
||||
cancelReplayPolicyDropUnstarted = "drop_unstarted_turn"
|
||||
cancelReplayPolicyKeepStableInput = "keep_stable_input"
|
||||
cancelReplayPolicyKeepInterrupted = "keep_interrupted_output"
|
||||
)
|
||||
|
||||
func sanitizeCanceledReplayEntries(entries []HistoryEntry) []HistoryEntry {
|
||||
@@ -370,13 +371,19 @@ func sanitizeCanceledReplayEntries(entries []HistoryEntry) []HistoryEntry {
|
||||
for _, entry := range entries {
|
||||
if entry.TurnSeq > 0 {
|
||||
if policy, canceled := canceledTurns[entry.TurnSeq]; canceled {
|
||||
if policy == cancelReplayPolicyDropUnstarted {
|
||||
if policy == cancelReplayPolicyKeepInterrupted {
|
||||
filtered = append(filtered, entry)
|
||||
continue
|
||||
}
|
||||
if policy != cancelReplayPolicyDropTurn {
|
||||
if _, active := activeCanceledTurns[entry.TurnSeq]; active {
|
||||
policy = cancelReplayPolicyKeepStableInput
|
||||
} else {
|
||||
policy = cancelReplayPolicyDropTurn
|
||||
filtered = append(filtered, entry)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if policy == cancelReplayPolicyDropUnstarted {
|
||||
policy = cancelReplayPolicyDropTurn
|
||||
}
|
||||
if policy == cancelReplayPolicyDropTurn || !isStableCanceledTurnInputEntry(entry) {
|
||||
continue
|
||||
}
|
||||
@@ -450,6 +457,8 @@ func normalizeCancelReplayPolicy(policy string, reason string) string {
|
||||
return cancelReplayPolicyDropUnstarted
|
||||
case cancelReplayPolicyKeepStableInput:
|
||||
return cancelReplayPolicyKeepStableInput
|
||||
case cancelReplayPolicyKeepInterrupted:
|
||||
return cancelReplayPolicyKeepInterrupted
|
||||
default:
|
||||
return cancelReplayPolicyForReason(reason)
|
||||
}
|
||||
@@ -616,9 +625,13 @@ func projectCheckpointTurnBlobs(conversation *ConversationFile, blobs *checkpoin
|
||||
turnIDs := make([][]byte, 0, len(order))
|
||||
for _, turnSeq := range order {
|
||||
entries := grouped[turnSeq]
|
||||
completedToolCalls, err := collectCheckpointCompletedToolCalls(entries)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var userMessageID []byte
|
||||
var turnRequestID string
|
||||
stepIDs := make([][]byte, 0, len(entries))
|
||||
steps := make([]*agentv1.ConversationStep, 0, len(entries))
|
||||
seenToolCalls := make(map[string]struct{})
|
||||
openToolCalls := make(map[string]struct{})
|
||||
for _, entry := range entries {
|
||||
@@ -645,59 +658,48 @@ func projectCheckpointTurnBlobs(conversation *ConversationFile, blobs *checkpoin
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(payload.ReasoningContent) != "" {
|
||||
stepID, err := addCheckpointStepBlob(blobs, &agentv1.ConversationStep{
|
||||
steps = append(steps, &agentv1.ConversationStep{
|
||||
Message: &agentv1.ConversationStep_ThinkingMessage{
|
||||
ThinkingMessage: &agentv1.ThinkingMessage{Text: payload.ReasoningContent},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stepIDs = append(stepIDs, stepID)
|
||||
}
|
||||
if strings.TrimSpace(payload.Text) == "" {
|
||||
continue
|
||||
}
|
||||
stepID, err := addCheckpointStepBlob(blobs, &agentv1.ConversationStep{
|
||||
steps = append(steps, &agentv1.ConversationStep{
|
||||
Message: &agentv1.ConversationStep_AssistantMessage{
|
||||
AssistantMessage: &agentv1.AssistantMessage{Text: strings.TrimSpace(payload.Text)},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stepIDs = append(stepIDs, stepID)
|
||||
case "tool_call":
|
||||
var payload toolCallEntryPayload
|
||||
if err := json.Unmarshal(entry.Payload, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(payload.ReasoningContent) != "" {
|
||||
stepID, err := addCheckpointStepBlob(blobs, &agentv1.ConversationStep{
|
||||
steps = append(steps, &agentv1.ConversationStep{
|
||||
Message: &agentv1.ConversationStep_ThinkingMessage{
|
||||
ThinkingMessage: &agentv1.ThinkingMessage{Text: payload.ReasoningContent},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stepIDs = append(stepIDs, stepID)
|
||||
}
|
||||
toolCall := &agentv1.ToolCall{}
|
||||
toolCallID := strings.TrimSpace(payload.ToolCallID)
|
||||
if err := protojson.Unmarshal(payload.ToolCall, toolCall); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !shouldPersistToolResultName(firstNonEmpty(strings.TrimSpace(payload.ToolName), inferToolName(toolCall))) {
|
||||
continue
|
||||
if completedPayload := completedToolCalls[toolCallID]; len(completedPayload) > 0 {
|
||||
completedToolCall := &agentv1.ToolCall{}
|
||||
if err := protojson.Unmarshal(completedPayload, completedToolCall); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
proto.Merge(toolCall, completedToolCall)
|
||||
}
|
||||
stepID, err := addCheckpointStepBlob(blobs, &agentv1.ConversationStep{
|
||||
steps = append(steps, &agentv1.ConversationStep{
|
||||
Message: &agentv1.ConversationStep_ToolCall{ToolCall: toolCall},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stepIDs = append(stepIDs, stepID)
|
||||
if toolCallID := strings.TrimSpace(payload.ToolCallID); toolCallID != "" {
|
||||
if toolCallID != "" {
|
||||
seenToolCalls[toolCallID] = struct{}{}
|
||||
openToolCalls[toolCallID] = struct{}{}
|
||||
}
|
||||
@@ -706,22 +708,19 @@ func projectCheckpointTurnBlobs(conversation *ConversationFile, blobs *checkpoin
|
||||
if err := json.Unmarshal(entry.Payload, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if toolCallID := strings.TrimSpace(payload.ToolCallID); toolCallID != "" {
|
||||
if _, ok := seenToolCalls[toolCallID]; ok {
|
||||
delete(openToolCalls, toolCallID)
|
||||
continue
|
||||
}
|
||||
toolCallID := strings.TrimSpace(payload.ToolCallID)
|
||||
if toolCallID != "" {
|
||||
delete(openToolCalls, toolCallID)
|
||||
}
|
||||
if _, ok := seenToolCalls[toolCallID]; ok {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(payload.ReasoningContent) != "" {
|
||||
stepID, err := addCheckpointStepBlob(blobs, &agentv1.ConversationStep{
|
||||
steps = append(steps, &agentv1.ConversationStep{
|
||||
Message: &agentv1.ConversationStep_ThinkingMessage{
|
||||
ThinkingMessage: &agentv1.ThinkingMessage{Text: payload.ReasoningContent},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stepIDs = append(stepIDs, stepID)
|
||||
}
|
||||
if len(payload.ToolCall) == 0 {
|
||||
continue
|
||||
@@ -730,21 +729,22 @@ func projectCheckpointTurnBlobs(conversation *ConversationFile, blobs *checkpoin
|
||||
if err := protojson.Unmarshal(payload.ToolCall, toolCall); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !shouldPersistToolResultName(firstNonEmpty(strings.TrimSpace(payload.ToolName), inferToolName(toolCall))) {
|
||||
continue
|
||||
}
|
||||
stepID, err := addCheckpointStepBlob(blobs, &agentv1.ConversationStep{
|
||||
steps = append(steps, &agentv1.ConversationStep{
|
||||
Message: &agentv1.ConversationStep_ToolCall{ToolCall: toolCall},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stepIDs = append(stepIDs, stepID)
|
||||
}
|
||||
}
|
||||
if len(userMessageID) == 0 && len(stepIDs) == 0 {
|
||||
if len(userMessageID) == 0 && len(steps) == 0 {
|
||||
continue
|
||||
}
|
||||
stepIDs := make([][]byte, 0, len(steps))
|
||||
for _, step := range steps {
|
||||
stepID, err := addCheckpointStepBlob(blobs, step)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stepIDs = append(stepIDs, stepID)
|
||||
}
|
||||
agentTurn := &agentv1.AgentConversationTurnStructure{
|
||||
UserMessage: userMessageID,
|
||||
Steps: stepIDs,
|
||||
@@ -765,6 +765,23 @@ func projectCheckpointTurnBlobs(conversation *ConversationFile, blobs *checkpoin
|
||||
return turnIDs, nil
|
||||
}
|
||||
|
||||
func collectCheckpointCompletedToolCalls(entries []HistoryEntry) (map[string]json.RawMessage, error) {
|
||||
completed := make(map[string]json.RawMessage)
|
||||
for _, entry := range entries {
|
||||
if strings.TrimSpace(entry.Kind) != "tool_result" {
|
||||
continue
|
||||
}
|
||||
var payload toolResultEntryPayload
|
||||
if err := json.Unmarshal(entry.Payload, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if toolCallID := strings.TrimSpace(payload.ToolCallID); toolCallID != "" && len(payload.ToolCall) > 0 {
|
||||
completed[toolCallID] = payload.ToolCall
|
||||
}
|
||||
}
|
||||
return completed, nil
|
||||
}
|
||||
|
||||
func addCheckpointStepBlob(blobs *checkpointBlobGraph, step *agentv1.ConversationStep) ([]byte, error) {
|
||||
payload, err := proto.Marshal(step)
|
||||
if err != nil {
|
||||
@@ -1209,7 +1226,7 @@ func trimReplayDanglingAssistantToolCalls(messages []modeladapter.Message) []mod
|
||||
return trimmed
|
||||
}
|
||||
|
||||
func shouldPersistToolResultName(toolName string) bool {
|
||||
func shouldPersistCheckpointReplayToolResultName(toolName string) bool {
|
||||
switch strings.TrimSpace(toolName) {
|
||||
case "PatchEdit", "PatchEditLines", "PatchEditSpan", "Edit", "Write", "GenerateImage":
|
||||
return true
|
||||
@@ -1218,60 +1235,6 @@ func shouldPersistToolResultName(toolName string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func filterCheckpointTurns(rawTurns [][]byte) [][]byte {
|
||||
if len(rawTurns) == 0 {
|
||||
return nil
|
||||
}
|
||||
filtered := make([][]byte, 0, len(rawTurns))
|
||||
for _, rawTurn := range rawTurns {
|
||||
if len(rawTurn) == 0 {
|
||||
continue
|
||||
}
|
||||
turn := &agentv1.ConversationTurnStructure{}
|
||||
if err := proto.Unmarshal(rawTurn, turn); err != nil {
|
||||
filtered = append(filtered, append([]byte(nil), rawTurn...))
|
||||
continue
|
||||
}
|
||||
agentTurn := turn.GetAgentConversationTurn()
|
||||
if agentTurn == nil {
|
||||
filtered = append(filtered, append([]byte(nil), rawTurn...))
|
||||
continue
|
||||
}
|
||||
|
||||
nextSteps := make([][]byte, 0, len(agentTurn.GetSteps()))
|
||||
for _, rawStep := range agentTurn.GetSteps() {
|
||||
if len(rawStep) == 0 {
|
||||
continue
|
||||
}
|
||||
step := &agentv1.ConversationStep{}
|
||||
if err := proto.Unmarshal(rawStep, step); err != nil {
|
||||
continue
|
||||
}
|
||||
if toolCall := step.GetToolCall(); toolCall != nil && !shouldPersistToolResultName(inferToolName(toolCall)) {
|
||||
continue
|
||||
}
|
||||
nextSteps = append(nextSteps, append([]byte(nil), rawStep...))
|
||||
}
|
||||
if len(agentTurn.GetUserMessage()) == 0 && len(nextSteps) == 0 {
|
||||
continue
|
||||
}
|
||||
encoded, err := proto.Marshal(&agentv1.ConversationTurnStructure{
|
||||
Turn: &agentv1.ConversationTurnStructure_AgentConversationTurn{
|
||||
AgentConversationTurn: &agentv1.AgentConversationTurnStructure{
|
||||
UserMessage: append([]byte(nil), agentTurn.GetUserMessage()...),
|
||||
Steps: nextSteps,
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
filtered = append(filtered, append([]byte(nil), rawTurn...))
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, encoded)
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func filterCheckpointPersistentToolReplay(messages []promptengine.Message) []promptengine.Message {
|
||||
if len(messages) == 0 {
|
||||
return nil
|
||||
@@ -1282,7 +1245,7 @@ func filterCheckpointPersistentToolReplay(messages []promptengine.Message) []pro
|
||||
if strings.TrimSpace(message.Role) == "assistant" && len(message.ToolCalls) > 0 {
|
||||
nextToolCalls := make([]promptengine.ToolCallDescriptor, 0, len(message.ToolCalls))
|
||||
for _, toolCall := range message.ToolCalls {
|
||||
if !shouldPersistToolResultName(toolCall.Function.Name) {
|
||||
if !shouldPersistCheckpointReplayToolResultName(toolCall.Function.Name) {
|
||||
skippedToolCallIDs[strings.TrimSpace(toolCall.ID)] = struct{}{}
|
||||
continue
|
||||
}
|
||||
@@ -1299,7 +1262,7 @@ func filterCheckpointPersistentToolReplay(messages []promptengine.Message) []pro
|
||||
if _, ok := skippedToolCallIDs[strings.TrimSpace(message.ToolCallID)]; ok {
|
||||
continue
|
||||
}
|
||||
if !shouldPersistToolResultName(message.Name) {
|
||||
if !shouldPersistCheckpointReplayToolResultName(message.Name) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package forwarder
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -9,6 +11,7 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"cursor/gen/agentv1"
|
||||
promptengine "cursor/internal/backend/agent/prompt"
|
||||
)
|
||||
|
||||
func TestProjectCheckpointProjectionBuildsResolvableForkState(t *testing.T) {
|
||||
@@ -138,3 +141,240 @@ func TestProjectCheckpointProjectionKeepsForkPointIsolatedFromLaterHistory(t *te
|
||||
t.Fatalf("fork snapshots are not isolated: midpoint=%#v latest=%#v", midpointMessages, latestMessages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectCheckpointProjectionMergesToolCallWithCompletedResult(t *testing.T) {
|
||||
userPayload, err := protojson.Marshal(&agentv1.UserMessage{Text: "inspect file", MessageId: "message-1"})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal user message: %v", err)
|
||||
}
|
||||
startedAt := uint64(100)
|
||||
toolCallID := "call-1"
|
||||
startedToolCall := checkpointTestToolCallPayload(t, &agentv1.ToolCall{
|
||||
ToolCallId: &toolCallID,
|
||||
StartedAtMs: &startedAt,
|
||||
Tool: &agentv1.ToolCall_ReadToolCall{
|
||||
ReadToolCall: &agentv1.ReadToolCall{
|
||||
Args: &agentv1.ReadToolArgs{Path: "/tmp/example.txt"},
|
||||
},
|
||||
},
|
||||
})
|
||||
completedAt := uint64(200)
|
||||
completedToolCall := checkpointTestToolCallPayload(t, &agentv1.ToolCall{
|
||||
CompletedAtMs: &completedAt,
|
||||
Tool: &agentv1.ToolCall_ReadToolCall{
|
||||
ReadToolCall: &agentv1.ReadToolCall{
|
||||
Result: &agentv1.ReadToolResult{
|
||||
Result: &agentv1.ReadToolResult_Success{
|
||||
Success: &agentv1.ReadToolSuccess{
|
||||
Path: "/tmp/example.txt",
|
||||
TotalLines: 1,
|
||||
Output: &agentv1.ReadToolSuccess_Content{Content: "file contents"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
conversation := &ConversationFile{
|
||||
ConversationID: "conversation-1",
|
||||
Mode: "agent",
|
||||
NextTurnSeq: 2,
|
||||
Entries: []HistoryEntry{
|
||||
{Seq: 1, TurnSeq: 1, RequestID: "request-1", Role: "user", Kind: "user_message", Payload: userPayload},
|
||||
newAssistantTextEntry(1, "request-1", "before", "", ""),
|
||||
newToolCallEntry(1, "request-1", "call-1", "Read", "", "", startedToolCall),
|
||||
newToolResultEntry(1, "request-1", "call-1", "Read", `{"path":"/tmp/example.txt"}`, "file contents", "", completedToolCall),
|
||||
newAssistantTextEntry(1, "request-1", "after", "", ""),
|
||||
},
|
||||
}
|
||||
|
||||
projection, err := NewHistoryProjector().ProjectCheckpointProjection(conversation)
|
||||
if err != nil {
|
||||
t.Fatalf("ProjectCheckpointProjection() error = %v", err)
|
||||
}
|
||||
if len(projection.Blobs) != 5 {
|
||||
t.Fatalf("checkpoint blobs = %d, want user, three final steps, and turn", len(projection.Blobs))
|
||||
}
|
||||
steps := checkpointProjectionSteps(t, projection)
|
||||
if len(steps) != 3 {
|
||||
t.Fatalf("checkpoint steps = %d, want assistant, completed Read, assistant", len(steps))
|
||||
}
|
||||
if steps[0].GetAssistantMessage().GetText() != "before" || steps[2].GetAssistantMessage().GetText() != "after" {
|
||||
t.Fatalf("checkpoint step ordering changed: %#v", steps)
|
||||
}
|
||||
mergedToolCall := steps[1].GetToolCall()
|
||||
readCall := mergedToolCall.GetReadToolCall()
|
||||
if readCall == nil || readCall.GetResult().GetSuccess().GetContent() != "file contents" {
|
||||
t.Fatalf("checkpoint Read step does not contain completed result: %#v", steps[1].GetToolCall())
|
||||
}
|
||||
if readCall.GetArgs().GetPath() != "/tmp/example.txt" || mergedToolCall.GetToolCallId() != toolCallID || mergedToolCall.GetStartedAtMs() != startedAt || mergedToolCall.GetCompletedAtMs() != completedAt {
|
||||
t.Fatalf("checkpoint Read step lost started-call fields: %#v", mergedToolCall)
|
||||
}
|
||||
|
||||
replay, err := promptengine.DecodeReplayMessages(projection.State.GetRootPromptMessagesJson())
|
||||
if err != nil {
|
||||
t.Fatalf("decode root prompt replay: %v", err)
|
||||
}
|
||||
for _, message := range replay {
|
||||
if message.Name == "Read" || len(message.ToolCalls) > 0 {
|
||||
t.Fatalf("UI-only Read result leaked into root prompt replay: %#v", replay)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectCheckpointProjectionIsIdempotentAndDoesNotMutateHistory(t *testing.T) {
|
||||
userPayload, err := protojson.Marshal(&agentv1.UserMessage{Text: "inspect file", MessageId: "message-1"})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal user message: %v", err)
|
||||
}
|
||||
completedToolCall := checkpointTestReadToolCall(t, &agentv1.ReadToolResult{
|
||||
Result: &agentv1.ReadToolResult_Success{
|
||||
Success: &agentv1.ReadToolSuccess{
|
||||
Path: "/tmp/example.txt",
|
||||
TotalLines: 1,
|
||||
Output: &agentv1.ReadToolSuccess_Content{Content: "file contents"},
|
||||
},
|
||||
},
|
||||
})
|
||||
conversation := &ConversationFile{
|
||||
ConversationID: "conversation-1",
|
||||
Mode: "agent",
|
||||
NextTurnSeq: 2,
|
||||
Entries: []HistoryEntry{
|
||||
{Seq: 1, TurnSeq: 1, RequestID: "request-1", Role: "user", Kind: "user_message", Payload: userPayload},
|
||||
newToolCallEntry(1, "request-1", "call-1", "Read", "", "", checkpointTestReadToolCall(t, nil)),
|
||||
newToolResultEntry(1, "request-1", "call-1", "Read", `{"path":"/tmp/example.txt"}`, "file contents", "", completedToolCall),
|
||||
},
|
||||
}
|
||||
before, err := json.Marshal(conversation)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal conversation before projection: %v", err)
|
||||
}
|
||||
|
||||
projector := NewHistoryProjector()
|
||||
first, err := projector.ProjectCheckpointProjection(conversation)
|
||||
if err != nil {
|
||||
t.Fatalf("first projection: %v", err)
|
||||
}
|
||||
second, err := projector.ProjectCheckpointProjection(conversation)
|
||||
if err != nil {
|
||||
t.Fatalf("second projection: %v", err)
|
||||
}
|
||||
|
||||
if !proto.Equal(first.State, second.State) {
|
||||
t.Fatalf("repeated projection changed checkpoint state: first=%#v second=%#v", first.State, second.State)
|
||||
}
|
||||
if len(first.Blobs) != len(second.Blobs) {
|
||||
t.Fatalf("repeated projection changed blob count: first=%d second=%d", len(first.Blobs), len(second.Blobs))
|
||||
}
|
||||
for index := range first.Blobs {
|
||||
if !bytes.Equal(first.Blobs[index].ID, second.Blobs[index].ID) || !bytes.Equal(first.Blobs[index].Data, second.Blobs[index].Data) {
|
||||
t.Fatalf("repeated projection changed blob %d", index)
|
||||
}
|
||||
}
|
||||
after, err := json.Marshal(conversation)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal conversation after projection: %v", err)
|
||||
}
|
||||
if !bytes.Equal(before, after) {
|
||||
t.Fatalf("checkpoint projection mutated semantic history:\nbefore=%s\nafter=%s", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectCheckpointProjectionKeepsStartedToolCallWhenResultPayloadIsMissing(t *testing.T) {
|
||||
startedToolCall := checkpointTestReadToolCall(t, nil)
|
||||
conversation := &ConversationFile{
|
||||
ConversationID: "conversation-1",
|
||||
Mode: "agent",
|
||||
NextTurnSeq: 2,
|
||||
Entries: []HistoryEntry{
|
||||
newToolCallEntry(1, "request-1", "call-1", "Read", "", "", startedToolCall),
|
||||
newToolResultEntry(1, "request-1", "call-1", "Read", `{"path":"/tmp/example.txt"}`, "read failed", "", nil),
|
||||
},
|
||||
}
|
||||
|
||||
projection, err := NewHistoryProjector().ProjectCheckpointProjection(conversation)
|
||||
if err != nil {
|
||||
t.Fatalf("ProjectCheckpointProjection() error = %v", err)
|
||||
}
|
||||
steps := checkpointProjectionSteps(t, projection)
|
||||
if len(steps) != 1 {
|
||||
t.Fatalf("checkpoint steps = %d, want the original Read step", len(steps))
|
||||
}
|
||||
readCall := steps[0].GetToolCall().GetReadToolCall()
|
||||
if readCall == nil || readCall.GetArgs().GetPath() != "/tmp/example.txt" || readCall.GetResult() != nil {
|
||||
t.Fatalf("checkpoint did not preserve the original Read call: %#v", steps[0].GetToolCall())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectCheckpointProjectionAppendsLegacyResultWithoutToolCallEntry(t *testing.T) {
|
||||
completedToolCall := checkpointTestReadToolCall(t, &agentv1.ReadToolResult{
|
||||
Result: &agentv1.ReadToolResult_Error{Error: &agentv1.ReadToolError{ErrorMessage: "not readable"}},
|
||||
})
|
||||
conversation := &ConversationFile{
|
||||
ConversationID: "conversation-1",
|
||||
Mode: "agent",
|
||||
NextTurnSeq: 2,
|
||||
Entries: []HistoryEntry{
|
||||
newToolResultEntry(1, "request-1", "call-1", "Read", `{"path":"/tmp/example.txt"}`, "not readable", "", completedToolCall),
|
||||
},
|
||||
}
|
||||
|
||||
projection, err := NewHistoryProjector().ProjectCheckpointProjection(conversation)
|
||||
if err != nil {
|
||||
t.Fatalf("ProjectCheckpointProjection() error = %v", err)
|
||||
}
|
||||
steps := checkpointProjectionSteps(t, projection)
|
||||
if len(steps) != 1 || steps[0].GetToolCall().GetReadToolCall().GetResult().GetError().GetErrorMessage() != "not readable" {
|
||||
t.Fatalf("legacy result-only Read step was not preserved: %#v", steps)
|
||||
}
|
||||
}
|
||||
|
||||
func checkpointTestReadToolCall(t *testing.T, result *agentv1.ReadToolResult) []byte {
|
||||
t.Helper()
|
||||
return checkpointTestToolCallPayload(t, &agentv1.ToolCall{
|
||||
Tool: &agentv1.ToolCall_ReadToolCall{
|
||||
ReadToolCall: &agentv1.ReadToolCall{
|
||||
Args: &agentv1.ReadToolArgs{Path: "/tmp/example.txt"},
|
||||
Result: result,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func checkpointTestToolCallPayload(t *testing.T, toolCall *agentv1.ToolCall) []byte {
|
||||
t.Helper()
|
||||
payload, err := protojson.Marshal(toolCall)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal Read tool call: %v", err)
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func checkpointProjectionSteps(t *testing.T, projection *CheckpointProjection) []*agentv1.ConversationStep {
|
||||
t.Helper()
|
||||
if projection == nil || projection.State == nil || len(projection.State.GetTurns()) != 1 {
|
||||
t.Fatalf("checkpoint turns = %#v, want exactly one turn", projection)
|
||||
}
|
||||
blobs := make(map[string][]byte, len(projection.Blobs))
|
||||
for _, blob := range projection.Blobs {
|
||||
blobs[string(blob.ID)] = blob.Data
|
||||
}
|
||||
turn := &agentv1.ConversationTurnStructure{}
|
||||
if err := proto.Unmarshal(blobs[string(projection.State.GetTurns()[0])], turn); err != nil {
|
||||
t.Fatalf("decode checkpoint turn: %v", err)
|
||||
}
|
||||
agentTurn := turn.GetAgentConversationTurn()
|
||||
if agentTurn == nil {
|
||||
t.Fatal("checkpoint turn does not contain an agent turn")
|
||||
}
|
||||
steps := make([]*agentv1.ConversationStep, 0, len(agentTurn.GetSteps()))
|
||||
for _, stepID := range agentTurn.GetSteps() {
|
||||
step := &agentv1.ConversationStep{}
|
||||
if err := proto.Unmarshal(blobs[string(stepID)], step); err != nil {
|
||||
t.Fatalf("decode checkpoint step: %v", err)
|
||||
}
|
||||
steps = append(steps, step)
|
||||
}
|
||||
return steps
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ package forwarder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -834,14 +836,22 @@ func (service *Service) handleCancelIntent(intent InboundIntent) error {
|
||||
}
|
||||
hasCheckpoint := checkpointConversationInitialized(stream)
|
||||
if hasCheckpoint {
|
||||
preservedInterruptedOutput, err := service.persistInterruptedProviderOutput(stream)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cancelReason := firstNonEmpty(intent.CancelReason, "user aborted")
|
||||
_, err := service.appendConversationEntries(stream, stream.ConversationID, []HistoryEntry{
|
||||
newMetadataEntry(stream.TurnSeq, intent.RequestID, "control", map[string]any{
|
||||
"status": "canceled",
|
||||
"reason": cancelReason,
|
||||
"replay_policy": cancelReplayPolicyForReason(cancelReason),
|
||||
}),
|
||||
replayPolicy := cancelReplayPolicyForReason(cancelReason)
|
||||
if preservedInterruptedOutput || checkpointTurnHasReplayActivity(stream) {
|
||||
replayPolicy = cancelReplayPolicyKeepInterrupted
|
||||
}
|
||||
cancelEntry := newMetadataEntry(stream.TurnSeq, intent.RequestID, "control", map[string]any{
|
||||
"status": "canceled",
|
||||
"reason": cancelReason,
|
||||
"replay_policy": replayPolicy,
|
||||
})
|
||||
cancelEntry.IdempotencyKey = cancelMetadataIdempotencyKey(stream.TurnSeq, intent.RequestID)
|
||||
_, err = service.appendConversationEntries(stream, stream.ConversationID, []HistoryEntry{cancelEntry})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -869,6 +879,89 @@ func (service *Service) handleCancelIntent(intent InboundIntent) error {
|
||||
return service.broker.Cancel(intent.RequestID, firstNonEmpty(intent.CancelReason, "[canceled] User aborted request"))
|
||||
}
|
||||
|
||||
func checkpointTurnHasReplayActivity(stream *ActiveStream) bool {
|
||||
if stream == nil {
|
||||
return false
|
||||
}
|
||||
stream.mu.Lock()
|
||||
defer stream.mu.Unlock()
|
||||
if stream.CheckpointConversation == nil {
|
||||
return false
|
||||
}
|
||||
for _, entry := range stream.CheckpointConversation.Entries {
|
||||
if entry.TurnSeq == stream.TurnSeq && isCanceledTurnActivityEntry(entry) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// persistInterruptedProviderOutput commits the current provider pass before cancellation.
|
||||
// The entry key is stable for this provider pass, so repeated cancellation handling is a no-op.
|
||||
func (service *Service) persistInterruptedProviderOutput(stream *ActiveStream) (bool, error) {
|
||||
if stream == nil {
|
||||
return false, nil
|
||||
}
|
||||
stream.mu.Lock()
|
||||
turnSeq := stream.TurnSeq
|
||||
requestID := strings.TrimSpace(stream.RequestID)
|
||||
modelCallID := strings.TrimSpace(stream.CurrentModelCallID)
|
||||
providerPass := stream.ProviderPassCount
|
||||
text := stream.ProviderAccumulatedText
|
||||
reasoning := stream.ProviderAccumulatedReasoning
|
||||
reasoningSignature := stream.ProviderAccumulatedReasoningSignature
|
||||
reasoningSignatureSource := stream.ProviderAccumulatedReasoningSignatureSource
|
||||
reasoningItemID := stream.ProviderAccumulatedReasoningItemID
|
||||
reasoningStatus := stream.ProviderAccumulatedReasoningStatus
|
||||
reasoningSummary := append([]byte(nil), stream.ProviderAccumulatedReasoningSummary...)
|
||||
stream.mu.Unlock()
|
||||
if strings.TrimSpace(text) == "" && !hasReplayableReasoningPayload(reasoning, reasoningSignature, reasoningSignatureSource) {
|
||||
return false, nil
|
||||
}
|
||||
key := interruptedProviderOutputIdempotencyKey(turnSeq, requestID, modelCallID, providerPass)
|
||||
_, err := service.appendConversationEntries(stream, stream.ConversationID, []HistoryEntry{
|
||||
{
|
||||
TurnSeq: turnSeq,
|
||||
RequestID: requestID,
|
||||
IdempotencyKey: key,
|
||||
Role: "assistant",
|
||||
Kind: "assistant_text",
|
||||
Payload: newAssistantTextPayload(
|
||||
text,
|
||||
reasoning,
|
||||
reasoningSignature,
|
||||
reasoningSignatureSource,
|
||||
reasoningItemID,
|
||||
reasoningStatus,
|
||||
reasoningSummary,
|
||||
),
|
||||
},
|
||||
})
|
||||
return true, err
|
||||
}
|
||||
|
||||
func interruptedProviderOutputIdempotencyKey(turnSeq int64, requestID string, modelCallID string, providerPass int) string {
|
||||
payload := strings.Join([]string{
|
||||
"provider_interrupted_output",
|
||||
fmt.Sprintf("%d", turnSeq),
|
||||
strings.TrimSpace(requestID),
|
||||
strings.TrimSpace(modelCallID),
|
||||
fmt.Sprintf("%d", providerPass),
|
||||
}, "\x00")
|
||||
digest := sha256.Sum256([]byte(payload))
|
||||
return "provider-interrupted-output:" + hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func cancelMetadataIdempotencyKey(turnSeq int64, requestID string) string {
|
||||
payload := strings.Join([]string{
|
||||
"cancel",
|
||||
fmt.Sprintf("%d", turnSeq),
|
||||
strings.TrimSpace(requestID),
|
||||
}, "\x00")
|
||||
digest := sha256.Sum256([]byte(payload))
|
||||
return "cancel:" + hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
// handleExecResult 处理客户端返回的执行桥结果,并在终态时把 tool_result 写回 history。
|
||||
func (service *Service) handleExecResult(intent InboundIntent) error {
|
||||
stream, ok := service.broker.Get(intent.RequestID)
|
||||
@@ -2431,6 +2524,16 @@ func newAssistantTextEntry(turnSeq int64, requestID string, text string, reasoni
|
||||
}
|
||||
|
||||
func newAssistantTextEntryWithProviderMetadata(turnSeq int64, requestID string, text string, reasoningContent string, reasoningSignature string, reasoningSignatureSource string, reasoningItemID string, reasoningStatus string, reasoningSummary json.RawMessage) HistoryEntry {
|
||||
return HistoryEntry{
|
||||
TurnSeq: turnSeq,
|
||||
RequestID: strings.TrimSpace(requestID),
|
||||
Role: "assistant",
|
||||
Kind: "assistant_text",
|
||||
Payload: newAssistantTextPayload(text, reasoningContent, reasoningSignature, reasoningSignatureSource, reasoningItemID, reasoningStatus, reasoningSummary),
|
||||
}
|
||||
}
|
||||
|
||||
func newAssistantTextPayload(text string, reasoningContent string, reasoningSignature string, reasoningSignatureSource string, reasoningItemID string, reasoningStatus string, reasoningSummary json.RawMessage) json.RawMessage {
|
||||
payload, _ := json.Marshal(assistantTextPayload{
|
||||
Text: text,
|
||||
ReasoningContent: reasoningContent,
|
||||
@@ -2440,13 +2543,7 @@ func newAssistantTextEntryWithProviderMetadata(turnSeq int64, requestID string,
|
||||
ReasoningStatus: strings.TrimSpace(reasoningStatus),
|
||||
ReasoningSummary: append(json.RawMessage(nil), reasoningSummary...),
|
||||
})
|
||||
return HistoryEntry{
|
||||
TurnSeq: turnSeq,
|
||||
RequestID: strings.TrimSpace(requestID),
|
||||
Role: "assistant",
|
||||
Kind: "assistant_text",
|
||||
Payload: payload,
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
// newToolCallEntry 构造 tool_call entry。
|
||||
|
||||
@@ -79,6 +79,7 @@ type HistoryEntry struct {
|
||||
Seq int64 `json:"seq"`
|
||||
TurnSeq int64 `json:"turn_seq"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
IdempotencyKey string `json:"idempotency_key,omitempty"`
|
||||
Role string `json:"role"`
|
||||
Kind string `json:"kind"`
|
||||
ToolCallID string `json:"tool_call_id,omitempty"`
|
||||
|
||||
Reference in New Issue
Block a user