From efac665ffe7bae6b426f4542df705b0b272b1db3 Mon Sep 17 00:00:00 2001
From: Levi Neely <lkn@darkstar.example.net>
Date: Tue, 28 Apr 2026 12:22:45 +0200
Subject: [PATCH] core: tool schema caching and large tool result caching
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Tool schema caching:
- Mark the last tool with cache_control:ephemeral before each Anthropic
  request; Anthropic caches all preceding tools as a prefix
- Same for OpenRouter+Claude via openAITool.CacheControl extension

Large tool result caching (Anthropic only):
- In buildAnthropicMessages, scan backwards through the encoded output
  and mark the last tool_result block ≥4096 bytes with cache_control
- 4096 bytes ≈ 1024 tokens, Anthropic's minimum cache entry size
- OpenAI wire format has no block structure for tool results; skip

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 pkg/backend/anthropic.go | 40 ++++++++++++++++++++++++++++++----------
 pkg/backend/openai.go    | 11 ++++++++---
 2 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/pkg/backend/anthropic.go b/pkg/backend/anthropic.go
index fa25721..1ce2115 100644
--- a/pkg/backend/anthropic.go
+++ b/pkg/backend/anthropic.go
@@ -78,19 +78,21 @@ type anthropicMessage struct {
 }
 
 type anthropicContentBlock struct {
-	Type      string          `json:"type"`
-	Text      string          `json:"text,omitempty"`
-	ID        string          `json:"id,omitempty"`
-	Name      string          `json:"name,omitempty"`
-	Input     json.RawMessage `json:"input,omitempty"`
-	ToolUseID string          `json:"tool_use_id,omitempty"`
-	Content   string          `json:"content,omitempty"` // tool_result text
+	Type         string              `json:"type"`
+	Text         string              `json:"text,omitempty"`
+	ID           string              `json:"id,omitempty"`
+	Name         string              `json:"name,omitempty"`
+	Input        json.RawMessage     `json:"input,omitempty"`
+	ToolUseID    string              `json:"tool_use_id,omitempty"`
+	Content      string              `json:"content,omitempty"` // tool_result text
+	CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"`
 }
 
 type anthropicTool struct {
-	Name        string          `json:"name"`
-	Description string          `json:"description,omitempty"`
-	InputSchema json.RawMessage `json:"input_schema"`
+	Name         string              `json:"name"`
+	Description  string              `json:"description,omitempty"`
+	InputSchema  json.RawMessage     `json:"input_schema"`
+	CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"`
 }
 
 // -- implementation --
@@ -126,6 +128,9 @@ func (b *AnthropicBackend) ChatStream(ctx context.Context, messages []Message, t
 			InputSchema: schema,
 		})
 	}
+	if len(areq.Tools) > 0 {
+		areq.Tools[len(areq.Tools)-1].CacheControl = &anthropicCacheCtrl{Type: "ephemeral"}
+	}
 
 	data, _ := json.Marshal(areq)
 
@@ -204,6 +209,21 @@ func buildAnthropicMessages(messages []Message) (system []anthropicSystemBlock,
 			CacheControl: &anthropicCacheCtrl{Type: "ephemeral"},
 		}}
 	}
+	// Cache the last large tool result to avoid re-reading it on every turn.
+	// Anthropic requires ≥1024 tokens (~4096 bytes) for a cache entry to be stored.
+outer:
+	for i := len(out) - 1; i >= 0; i-- {
+		if out[i].Role != "user" {
+			continue
+		}
+		for j := len(out[i].Content) - 1; j >= 0; j-- {
+			b := &out[i].Content[j]
+			if b.Type == "tool_result" && len(b.Content) >= 4096 {
+				b.CacheControl = &anthropicCacheCtrl{Type: "ephemeral"}
+				break outer
+			}
+		}
+	}
 	return
 }
 
diff --git a/pkg/backend/openai.go b/pkg/backend/openai.go
index 70422f1..4158050 100644
--- a/pkg/backend/openai.go
+++ b/pkg/backend/openai.go
@@ -154,8 +154,9 @@ type openAIFunctionCall struct {
 }
 
 type openAITool struct {
-	Type     string             `json:"type"`
-	Function openAIToolFunction `json:"function"`
+	Type         string              `json:"type"`
+	Function     openAIToolFunction  `json:"function"`
+	CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"`
 }
 
 type openAIToolFunction struct {
@@ -444,6 +445,7 @@ func streamOpenAISSE(r io.Reader, ch chan<- StreamEvent) {
 
 func (b *OpenAIBackend) ChatStream(ctx context.Context, messages []Message, tools []Tool, params GenerationParams) (<-chan StreamEvent, error) {
 	wireMessages := encodeOpenAIMessages(messages)
+	wireTools := encodeOpenAITools(tools)
 	if b.name == "openrouter" && strings.Contains(strings.ToLower(b.model), "claude-") {
 		for i := range wireMessages {
 			if wireMessages[i].Role == "system" && wireMessages[i].Content != nil {
@@ -455,11 +457,14 @@ func (b *OpenAIBackend) ChatStream(ctx context.Context, messages []Message, tool
 				wireMessages[i].Content = nil
 			}
 		}
+		if len(wireTools) > 0 {
+			wireTools[len(wireTools)-1].CacheControl = &anthropicCacheCtrl{Type: "ephemeral"}
+		}
 	}
 	req := openAIChatRequest{
 		Model:            b.model,
 		Messages:         wireMessages,
-		Tools:            encodeOpenAITools(tools),
+		Tools:            wireTools,
 		Stream:           true,
 		StreamOptions:    &openAIStreamOptions{IncludeUsage: true},
 		MaxTokens:        params.MaxTokens,