backend: add ContentBlock/ImageSource types and multi-modal image encoding for Anthropic and OpenAI

Add canonical ContentBlock and ImageSource types to backend.go.
When Message.ContentBlocks is set, it takes precedence over Content.

Anthropic: new anthropicImageSource type, anthropicContentBlocks() helper
converts canonical blocks to wire format. buildAnthropicMessages handles
ContentBlocks on user and tool messages.

OpenAI: new openAIImageURL type, openAIContentBlocks() helper converts
canonical blocks to wire format (image_url with data: URIs).
encodeOpenAIMessages uses ContentBlocks when present.

pkg/backend/anthropic.go

@@ -80,15 +80,22 @@ type anthropicMessage struct {
 	Content []anthropicContentBlock `json:"content"`
 }
 
+type anthropicImageSource struct {
+	Type      string `json:"type"`       // "base64"
+	MediaType string `json:"media_type"` // "image/png", etc.
+	Data      string `json:"data"`       // base64-encoded bytes
+}
+
 type anthropicContentBlock struct {
-	Type         string              `json:"type"`
+	Type         string                `json:"type"`
-	Text         string              `json:"text,omitempty"`
+	Text         string                `json:"text,omitempty"`
-	ID           string              `json:"id,omitempty"`
+	Source       *anthropicImageSource `json:"source,omitempty"` // for image blocks
-	Name         string              `json:"name,omitempty"`
+	ID           string                `json:"id,omitempty"`
-	Input        json.RawMessage     `json:"input,omitempty"`
+	Name         string                `json:"name,omitempty"`
-	ToolUseID    string              `json:"tool_use_id,omitempty"`
+	Input        json.RawMessage       `json:"input,omitempty"`
-	Content      string              `json:"content,omitempty"` // tool_result text
+	ToolUseID    string                `json:"tool_use_id,omitempty"`
-	CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"`
+	Content      string                `json:"content,omitempty"` // tool_result text
+	CacheControl *anthropicCacheCtrl   `json:"cache_control,omitempty"`
 }
 
 type anthropicTool struct {
@@ -151,6 +158,31 @@ func (b *AnthropicBackend) ChatStream(ctx context.Context, messages []Message, t
 	return streamRequest(b.client, httpReq, "anthropic", streamAnthropicSSE)
 }
 
+// anthropicContentBlocks converts a canonical Message's content (text or ContentBlocks)
+// into Anthropic wire-format content blocks.
+func anthropicContentBlocks(m Message) []anthropicContentBlock {
+	if len(m.ContentBlocks) > 0 {
+		var blocks []anthropicContentBlock
+		for _, cb := range m.ContentBlocks {
+			switch cb.Type {
+			case "image":
+				blocks = append(blocks, anthropicContentBlock{
+					Type: "image",
+					Source: &anthropicImageSource{
+						Type:      cb.ImageSource.Type,
+						MediaType: cb.ImageSource.MediaType,
+						Data:      cb.ImageSource.Data,
+					},
+				})
+			case "text":
+				blocks = append(blocks, anthropicContentBlock{Type: "text", Text: cb.Text})
+			}
+		}
+		return blocks
+	}
+	return []anthropicContentBlock{{Type: "text", Text: m.Content}}
+}
+
 // buildAnthropicMessages converts ollie messages to Anthropic wire format.
 // System messages are collected into a single cacheable block (cache_control:
 // ephemeral marks the prefix as eligible for prompt-cache reuse).
@@ -170,7 +202,7 @@ func buildAnthropicMessages(messages []Message) (system []anthropicSystemBlock,
 		case "user":
 			out = append(out, anthropicMessage{
 				Role:    "user",
-				Content: []anthropicContentBlock{{Type: "text", Text: m.Content}},
+				Content: anthropicContentBlocks(m),
 			})
 			i++
 		case "assistant":
@@ -196,11 +228,37 @@ func buildAnthropicMessages(messages []Message) (system []anthropicSystemBlock,
 			// Batch all consecutive tool messages into one user message.
 			var blocks []anthropicContentBlock
 			for i < len(messages) && messages[i].Role == "tool" {
-				blocks = append(blocks, anthropicContentBlock{
+				tm := messages[i]
-					Type:      "tool_result",
+				if len(tm.ContentBlocks) > 0 {
-					ToolUseID: messages[i].ToolCallID,
+					// Multi-modal tool result: emit image blocks + text as tool_result content.
-					Content:   messages[i].Content,
+					var textParts []string
-				})
+					for _, cb := range tm.ContentBlocks {
+						switch cb.Type {
+						case "image":
+							blocks = append(blocks, anthropicContentBlock{
+								Type: "image",
+								Source: &anthropicImageSource{
+									Type:      cb.ImageSource.Type,
+									MediaType: cb.ImageSource.MediaType,
+									Data:      cb.ImageSource.Data,
+								},
+							})
+						case "text":
+							textParts = append(textParts, cb.Text)
+						}
+					}
+					blocks = append(blocks, anthropicContentBlock{
+						Type:      "tool_result",
+						ToolUseID: tm.ToolCallID,
+						Content:   strings.Join(textParts, ""),
+					})
+				} else {
+					blocks = append(blocks, anthropicContentBlock{
+						Type:      "tool_result",
+						ToolUseID: tm.ToolCallID,
+						Content:   tm.Content,
+					})
+				}
 				i++
 			}
 			out = append(out, anthropicMessage{Role: "user", Content: blocks})

pkg/backend/backend.go

@@ -13,12 +13,28 @@ import (
 	"time"
 )
 
+// ContentBlock is a single block within a multi-modal message.
+// When a Message has ContentBlocks set, it takes precedence over Content.
+type ContentBlock struct {
+	Type        string       `json:"type"`              // "text" | "image"
+	Text        string       `json:"text,omitempty"`    // for text blocks
+	ImageSource *ImageSource `json:"source,omitempty"`  // for image blocks
+}
+
+// ImageSource describes a base64-encoded image.
+type ImageSource struct {
+	Type      string `json:"type"`       // "base64"
+	MediaType string `json:"media_type"` // "image/png", "image/jpeg", "image/gif", "image/webp"
+	Data      string `json:"data"`       // base64-encoded bytes
+}
+
 // Message is a single conversation turn.
 type Message struct {
-	Role       string     `json:"role"`                   // "system" | "user" | "assistant" | "tool"
+	Role          string         `json:"role"`                     // "system" | "user" | "assistant" | "tool"
-	Content    string     `json:"content"`
+	Content       string         `json:"content"`
-	ToolCalls  []ToolCall `json:"tool_calls,omitempty"`   // set by assistant when calling tools
+	ContentBlocks []ContentBlock `json:"content_blocks,omitempty"` // when set, overrides Content
-	ToolCallID string     `json:"tool_call_id,omitempty"` // set on role=tool replies (required by OpenAI)
+	ToolCalls     []ToolCall     `json:"tool_calls,omitempty"`     // set by assistant when calling tools
+	ToolCallID    string         `json:"tool_call_id,omitempty"`   // set on role=tool replies (required by OpenAI)
 }
 
 // Tool describes a callable function exposed to the model.

pkg/backend/openai.go

@@ -119,9 +119,15 @@ func (b *OpenAIBackend) DefaultModel() string {
 
 // -- wire types --
 
+type openAIImageURL struct {
+	URL    string `json:"url"`              // data: URL or http(s) URL
+	Detail string `json:"detail,omitempty"` // "low", "high", "auto"
+}
+
 type openAIContentBlock struct {
 	Type         string              `json:"type"`
-	Text         string              `json:"text"`
+	Text         string              `json:"text,omitempty"`
+	ImageURL     *openAIImageURL     `json:"image_url,omitempty"`
 	CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"`
 }
 
@@ -249,13 +255,36 @@ func encodeOpenAIMessages(messages []Message) []openAIMessage {
 		}
 		// OpenAI spec: content must be null (not "") when tool_calls is present.
 		if len(wm.ToolCalls) == 0 {
-			wm.Content = &m.Content
+			if len(m.ContentBlocks) > 0 {
+				wm.ContentBlocks = openAIContentBlocks(m)
+			} else {
+				wm.Content = &m.Content
+			}
 		}
 		wire[i] = wm
 	}
 	return wire
 }
 
+// openAIContentBlocks converts a canonical Message's ContentBlocks into OpenAI wire format.
+func openAIContentBlocks(m Message) []openAIContentBlock {
+	var blocks []openAIContentBlock
+	for _, cb := range m.ContentBlocks {
+		switch cb.Type {
+		case "image":
+			blocks = append(blocks, openAIContentBlock{
+				Type: "image_url",
+				ImageURL: &openAIImageURL{
+					URL: "data:" + cb.ImageSource.MediaType + ";base64," + cb.ImageSource.Data,
+				},
+			})
+		case "text":
+			blocks = append(blocks, openAIContentBlock{Type: "text", Text: cb.Text})
+		}
+	}
+	return blocks
+}
+
 // encodeOpenAITools converts canonical Tools to the OpenAI wire format.
 func encodeOpenAITools(tools []Tool) []openAITool {
 	var wire []openAITool