From adb686bc1d98e27aa0e86265b3310d4f535f1c23 Mon Sep 17 00:00:00 2001 From: ollie Date: Mon, 25 May 2026 17:08:18 +0200 Subject: [PATCH] backend: add ContentBlock/ImageSource types and multi-modal image encoding for Anthropic and OpenAI Add canonical ContentBlock and ImageSource types to backend.go. When Message.ContentBlocks is set, it takes precedence over Content. Anthropic: new anthropicImageSource type, anthropicContentBlocks() helper converts canonical blocks to wire format. buildAnthropicMessages handles ContentBlocks on user and tool messages. OpenAI: new openAIImageURL type, openAIContentBlocks() helper converts canonical blocks to wire format (image_url with data: URIs). encodeOpenAIMessages uses ContentBlocks when present. --- pkg/backend/anthropic.go | 86 +++++++++++++++++++++++++++++++++------- pkg/backend/backend.go | 24 +++++++++-- pkg/backend/openai.go | 33 ++++++++++++++- 3 files changed, 123 insertions(+), 20 deletions(-) diff --git a/pkg/backend/anthropic.go b/pkg/backend/anthropic.go index 67f4adb..265022d 100644 --- a/pkg/backend/anthropic.go +++ b/pkg/backend/anthropic.go @@ -80,15 +80,22 @@ type anthropicMessage struct { Content []anthropicContentBlock `json:"content"` } +type anthropicImageSource struct { + Type string `json:"type"` // "base64" + MediaType string `json:"media_type"` // "image/png", etc. + Data string `json:"data"` // base64-encoded bytes +} + type anthropicContentBlock struct { - Type string `json:"type"` - Text string `json:"text,omitempty"` - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Input json.RawMessage `json:"input,omitempty"` - ToolUseID string `json:"tool_use_id,omitempty"` - Content string `json:"content,omitempty"` // tool_result text - CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"` + Type string `json:"type"` + Text string `json:"text,omitempty"` + Source *anthropicImageSource `json:"source,omitempty"` // for image blocks + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Input json.RawMessage `json:"input,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` + Content string `json:"content,omitempty"` // tool_result text + CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"` } type anthropicTool struct { @@ -151,6 +158,31 @@ func (b *AnthropicBackend) ChatStream(ctx context.Context, messages []Message, t return streamRequest(b.client, httpReq, "anthropic", streamAnthropicSSE) } +// anthropicContentBlocks converts a canonical Message's content (text or ContentBlocks) +// into Anthropic wire-format content blocks. +func anthropicContentBlocks(m Message) []anthropicContentBlock { + if len(m.ContentBlocks) > 0 { + var blocks []anthropicContentBlock + for _, cb := range m.ContentBlocks { + switch cb.Type { + case "image": + blocks = append(blocks, anthropicContentBlock{ + Type: "image", + Source: &anthropicImageSource{ + Type: cb.ImageSource.Type, + MediaType: cb.ImageSource.MediaType, + Data: cb.ImageSource.Data, + }, + }) + case "text": + blocks = append(blocks, anthropicContentBlock{Type: "text", Text: cb.Text}) + } + } + return blocks + } + return []anthropicContentBlock{{Type: "text", Text: m.Content}} +} + // buildAnthropicMessages converts ollie messages to Anthropic wire format. // System messages are collected into a single cacheable block (cache_control: // ephemeral marks the prefix as eligible for prompt-cache reuse). @@ -170,7 +202,7 @@ func buildAnthropicMessages(messages []Message) (system []anthropicSystemBlock, case "user": out = append(out, anthropicMessage{ Role: "user", - Content: []anthropicContentBlock{{Type: "text", Text: m.Content}}, + Content: anthropicContentBlocks(m), }) i++ case "assistant": @@ -196,11 +228,37 @@ func buildAnthropicMessages(messages []Message) (system []anthropicSystemBlock, // Batch all consecutive tool messages into one user message. var blocks []anthropicContentBlock for i < len(messages) && messages[i].Role == "tool" { - blocks = append(blocks, anthropicContentBlock{ - Type: "tool_result", - ToolUseID: messages[i].ToolCallID, - Content: messages[i].Content, - }) + tm := messages[i] + if len(tm.ContentBlocks) > 0 { + // Multi-modal tool result: emit image blocks + text as tool_result content. + var textParts []string + for _, cb := range tm.ContentBlocks { + switch cb.Type { + case "image": + blocks = append(blocks, anthropicContentBlock{ + Type: "image", + Source: &anthropicImageSource{ + Type: cb.ImageSource.Type, + MediaType: cb.ImageSource.MediaType, + Data: cb.ImageSource.Data, + }, + }) + case "text": + textParts = append(textParts, cb.Text) + } + } + blocks = append(blocks, anthropicContentBlock{ + Type: "tool_result", + ToolUseID: tm.ToolCallID, + Content: strings.Join(textParts, ""), + }) + } else { + blocks = append(blocks, anthropicContentBlock{ + Type: "tool_result", + ToolUseID: tm.ToolCallID, + Content: tm.Content, + }) + } i++ } out = append(out, anthropicMessage{Role: "user", Content: blocks}) diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go index 26c7d30..54e06cf 100644 --- a/pkg/backend/backend.go +++ b/pkg/backend/backend.go @@ -13,12 +13,28 @@ import ( "time" ) +// ContentBlock is a single block within a multi-modal message. +// When a Message has ContentBlocks set, it takes precedence over Content. +type ContentBlock struct { + Type string `json:"type"` // "text" | "image" + Text string `json:"text,omitempty"` // for text blocks + ImageSource *ImageSource `json:"source,omitempty"` // for image blocks +} + +// ImageSource describes a base64-encoded image. +type ImageSource struct { + Type string `json:"type"` // "base64" + MediaType string `json:"media_type"` // "image/png", "image/jpeg", "image/gif", "image/webp" + Data string `json:"data"` // base64-encoded bytes +} + // Message is a single conversation turn. type Message struct { - Role string `json:"role"` // "system" | "user" | "assistant" | "tool" - Content string `json:"content"` - ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set by assistant when calling tools - ToolCallID string `json:"tool_call_id,omitempty"` // set on role=tool replies (required by OpenAI) + Role string `json:"role"` // "system" | "user" | "assistant" | "tool" + Content string `json:"content"` + ContentBlocks []ContentBlock `json:"content_blocks,omitempty"` // when set, overrides Content + ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set by assistant when calling tools + ToolCallID string `json:"tool_call_id,omitempty"` // set on role=tool replies (required by OpenAI) } // Tool describes a callable function exposed to the model. diff --git a/pkg/backend/openai.go b/pkg/backend/openai.go index 7b444dd..79690ff 100644 --- a/pkg/backend/openai.go +++ b/pkg/backend/openai.go @@ -119,9 +119,15 @@ func (b *OpenAIBackend) DefaultModel() string { // -- wire types -- +type openAIImageURL struct { + URL string `json:"url"` // data: URL or http(s) URL + Detail string `json:"detail,omitempty"` // "low", "high", "auto" +} + type openAIContentBlock struct { Type string `json:"type"` - Text string `json:"text"` + Text string `json:"text,omitempty"` + ImageURL *openAIImageURL `json:"image_url,omitempty"` CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"` } @@ -249,13 +255,36 @@ func encodeOpenAIMessages(messages []Message) []openAIMessage { } // OpenAI spec: content must be null (not "") when tool_calls is present. if len(wm.ToolCalls) == 0 { - wm.Content = &m.Content + if len(m.ContentBlocks) > 0 { + wm.ContentBlocks = openAIContentBlocks(m) + } else { + wm.Content = &m.Content + } } wire[i] = wm } return wire } +// openAIContentBlocks converts a canonical Message's ContentBlocks into OpenAI wire format. +func openAIContentBlocks(m Message) []openAIContentBlock { + var blocks []openAIContentBlock + for _, cb := range m.ContentBlocks { + switch cb.Type { + case "image": + blocks = append(blocks, openAIContentBlock{ + Type: "image_url", + ImageURL: &openAIImageURL{ + URL: "data:" + cb.ImageSource.MediaType + ";base64," + cb.ImageSource.Data, + }, + }) + case "text": + blocks = append(blocks, openAIContentBlock{Type: "text", Text: cb.Text}) + } + } + return blocks +} + // encodeOpenAITools converts canonical Tools to the OpenAI wire format. func encodeOpenAITools(tools []Tool) []openAITool { var wire []openAITool