lkn
/
ollie-core
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backend: add ContentBlock/ImageSource types and multi-modal image encoding for Anthropic and OpenAI 

Add canonical ContentBlock and ImageSource types to backend.go.
When Message.ContentBlocks is set, it takes precedence over Content.

Anthropic: new anthropicImageSource type, anthropicContentBlocks() helper
converts canonical blocks to wire format. buildAnthropicMessages handles
ContentBlocks on user and tool messages.

OpenAI: new openAIImageURL type, openAIContentBlocks() helper converts
canonical blocks to wire format (image_url with data: URIs).
encodeOpenAIMessages uses ContentBlocks when present.
This commit is contained in:
ollie 2026-05-25 17:08:18 +02:00
parent 5985ab7d20
commit adb686bc1d
3 changed files with 123 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
pkg/backend/anthropic.go
View File

@ -80,15 +80,22 @@ type anthropicMessage struct {
Content []anthropicContentBlock `json:"content"`
}
type anthropicImageSource struct {
Type string `json:"type"` // "base64"
MediaType string `json:"media_type"` // "image/png", etc.
Data string `json:"data"` // base64-encoded bytes
}
type anthropicContentBlock struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Input json.RawMessage `json:"input,omitempty"`
ToolUseID string `json:"tool_use_id,omitempty"`
Content string `json:"content,omitempty"` // tool_result text
CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"`
Type string `json:"type"`
Text string `json:"text,omitempty"`
Source *anthropicImageSource `json:"source,omitempty"` // for image blocks
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Input json.RawMessage `json:"input,omitempty"`
ToolUseID string `json:"tool_use_id,omitempty"`
Content string `json:"content,omitempty"` // tool_result text
CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"`
}
type anthropicTool struct {
@ -151,6 +158,31 @@ func (b *AnthropicBackend) ChatStream(ctx context.Context, messages []Message, t
return streamRequest(b.client, httpReq, "anthropic", streamAnthropicSSE)
}
// anthropicContentBlocks converts a canonical Message's content (text or ContentBlocks)
// into Anthropic wire-format content blocks.
func anthropicContentBlocks(m Message) []anthropicContentBlock {
if len(m.ContentBlocks) > 0 {
var blocks []anthropicContentBlock
for _, cb := range m.ContentBlocks {
switch cb.Type {
case "image":
blocks = append(blocks, anthropicContentBlock{
Type: "image",
Source: &anthropicImageSource{
Type: cb.ImageSource.Type,
MediaType: cb.ImageSource.MediaType,
Data: cb.ImageSource.Data,
},
})
case "text":
blocks = append(blocks, anthropicContentBlock{Type: "text", Text: cb.Text})
}
}
return blocks
}
return []anthropicContentBlock{{Type: "text", Text: m.Content}}
}
// buildAnthropicMessages converts ollie messages to Anthropic wire format.
// System messages are collected into a single cacheable block (cache_control:
// ephemeral marks the prefix as eligible for prompt-cache reuse).
@ -170,7 +202,7 @@ func buildAnthropicMessages(messages []Message) (system []anthropicSystemBlock,
case "user":
out = append(out, anthropicMessage{
Role: "user",
Content: []anthropicContentBlock{{Type: "text", Text: m.Content}},
Content: anthropicContentBlocks(m),
})
i++
case "assistant":
@ -196,11 +228,37 @@ func buildAnthropicMessages(messages []Message) (system []anthropicSystemBlock,
// Batch all consecutive tool messages into one user message.
var blocks []anthropicContentBlock
for i < len(messages) && messages[i].Role == "tool" {
blocks = append(blocks, anthropicContentBlock{
Type: "tool_result",
ToolUseID: messages[i].ToolCallID,
Content: messages[i].Content,
})
tm := messages[i]
if len(tm.ContentBlocks) > 0 {
// Multi-modal tool result: emit image blocks + text as tool_result content.
var textParts []string
for _, cb := range tm.ContentBlocks {
switch cb.Type {
case "image":
blocks = append(blocks, anthropicContentBlock{
Type: "image",
Source: &anthropicImageSource{
Type: cb.ImageSource.Type,
MediaType: cb.ImageSource.MediaType,
Data: cb.ImageSource.Data,
},
})
case "text":
textParts = append(textParts, cb.Text)
}
}
blocks = append(blocks, anthropicContentBlock{
Type: "tool_result",
ToolUseID: tm.ToolCallID,
Content: strings.Join(textParts, ""),
})
} else {
blocks = append(blocks, anthropicContentBlock{
Type: "tool_result",
ToolUseID: tm.ToolCallID,
Content: tm.Content,
})
}
i++
}
out = append(out, anthropicMessage{Role: "user", Content: blocks})

24
pkg/backend/backend.go
View File

@ -13,12 +13,28 @@ import (
"time"
)
// ContentBlock is a single block within a multi-modal message.
// When a Message has ContentBlocks set, it takes precedence over Content.
type ContentBlock struct {
Type string `json:"type"` // "text" | "image"
Text string `json:"text,omitempty"` // for text blocks
ImageSource *ImageSource `json:"source,omitempty"` // for image blocks
}
// ImageSource describes a base64-encoded image.
type ImageSource struct {
Type string `json:"type"` // "base64"
MediaType string `json:"media_type"` // "image/png", "image/jpeg", "image/gif", "image/webp"
Data string `json:"data"` // base64-encoded bytes
}
// Message is a single conversation turn.
type Message struct {
Role string `json:"role"` // "system" | "user" | "assistant" | "tool"
Content string `json:"content"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set by assistant when calling tools
ToolCallID string `json:"tool_call_id,omitempty"` // set on role=tool replies (required by OpenAI)
Role string `json:"role"` // "system" | "user" | "assistant" | "tool"
Content string `json:"content"`
ContentBlocks []ContentBlock `json:"content_blocks,omitempty"` // when set, overrides Content
ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set by assistant when calling tools
ToolCallID string `json:"tool_call_id,omitempty"` // set on role=tool replies (required by OpenAI)
}
// Tool describes a callable function exposed to the model.

33
pkg/backend/openai.go
View File

@ -119,9 +119,15 @@ func (b *OpenAIBackend) DefaultModel() string {
// -- wire types --
type openAIImageURL struct {
URL string `json:"url"` // data: URL or http(s) URL
Detail string `json:"detail,omitempty"` // "low", "high", "auto"
}
type openAIContentBlock struct {
Type string `json:"type"`
Text string `json:"text"`
Text string `json:"text,omitempty"`
ImageURL *openAIImageURL `json:"image_url,omitempty"`
CacheControl *anthropicCacheCtrl `json:"cache_control,omitempty"`
}
@ -249,13 +255,36 @@ func encodeOpenAIMessages(messages []Message) []openAIMessage {
}
// OpenAI spec: content must be null (not "") when tool_calls is present.
if len(wm.ToolCalls) == 0 {
wm.Content = &m.Content
if len(m.ContentBlocks) > 0 {
wm.ContentBlocks = openAIContentBlocks(m)
} else {
wm.Content = &m.Content
}
}
wire[i] = wm
}
return wire
}
// openAIContentBlocks converts a canonical Message's ContentBlocks into OpenAI wire format.
func openAIContentBlocks(m Message) []openAIContentBlock {
var blocks []openAIContentBlock
for _, cb := range m.ContentBlocks {
switch cb.Type {
case "image":
blocks = append(blocks, openAIContentBlock{
Type: "image_url",
ImageURL: &openAIImageURL{
URL: "data:" + cb.ImageSource.MediaType + ";base64," + cb.ImageSource.Data,
},
})
case "text":
blocks = append(blocks, openAIContentBlock{Type: "text", Text: cb.Text})
}
}
return blocks
}
// encodeOpenAITools converts canonical Tools to the OpenAI wire format.
func encodeOpenAITools(tools []Tool) []openAITool {
var wire []openAITool