244 lines
9.6 KiB
Go
244 lines
9.6 KiB
Go
// Package backend defines the Backend interface and shared types for LLM
|
|
// providers. All backends speak the same canonical types; provider-specific
|
|
// wire formats are handled inside each implementation.
|
|
package backend
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ContentBlock is a single block within a multi-modal message.
|
|
// When a Message has ContentBlocks set, it takes precedence over Content.
|
|
type ContentBlock struct {
|
|
Type string `json:"type"` // "text" | "image"
|
|
Text string `json:"text,omitempty"` // for text blocks
|
|
ImageSource *ImageSource `json:"source,omitempty"` // for image blocks
|
|
}
|
|
|
|
// ImageSource describes a base64-encoded image.
|
|
type ImageSource struct {
|
|
Type string `json:"type"` // "base64"
|
|
MediaType string `json:"media_type"` // "image/png", "image/jpeg", "image/gif", "image/webp"
|
|
Data string `json:"data"` // base64-encoded bytes
|
|
}
|
|
|
|
// Message is a single conversation turn.
|
|
type Message struct {
|
|
ID string `json:"id,omitempty"` // stable local ID; not sent to providers
|
|
Role string `json:"role"` // "system" | "user" | "assistant" | "tool"
|
|
Content string `json:"content"`
|
|
Reasoning string `json:"reasoning,omitempty"` // thinking/reasoning content (DeepSeek, OpenAI o-series)
|
|
ContentBlocks []ContentBlock `json:"content_blocks,omitempty"` // when set, overrides Content
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set by assistant when calling tools
|
|
ToolCallID string `json:"tool_call_id,omitempty"` // set on role=tool replies (required by OpenAI)
|
|
}
|
|
|
|
// Tool describes a callable function exposed to the model.
|
|
type Tool struct {
|
|
Name string `json:"name"`
|
|
Description string `json:"description"`
|
|
Parameters json.RawMessage `json:"parameters"` // JSON Schema object
|
|
}
|
|
|
|
// ToolCall is the model's request to invoke a function.
|
|
type ToolCall struct {
|
|
ID string `json:"id,omitempty"` // provider-assigned; may be empty (Ollama)
|
|
Name string `json:"name"`
|
|
Arguments json.RawMessage `json:"arguments"` // always a JSON object
|
|
}
|
|
|
|
// Usage holds token counts and optional cost for a single Chat call.
|
|
// CostUSD is non-zero only when the backend reports it directly (e.g. OpenRouter).
|
|
// CachedInputTokens are tokens served from the prompt cache; they are NOT
|
|
// included in InputTokens (Anthropic normalises this way; we do the same for
|
|
// OpenAI by subtracting cached_tokens from prompt_tokens before setting
|
|
// InputTokens).
|
|
// CacheCreationTokens are tokens written to the cache (Anthropic only).
|
|
type Usage struct {
|
|
InputTokens int
|
|
CachedInputTokens int
|
|
CacheCreationTokens int
|
|
OutputTokens int
|
|
CostUSD float64
|
|
}
|
|
|
|
// StreamEvent is a single increment from a streaming chat call.
|
|
// Content is an incremental text delta (append, not replace).
|
|
// Reasoning is an incremental thinking/reasoning delta (append, not replace).
|
|
// ToolCalls accumulates complete calls; they may arrive on any event.
|
|
// The final event has Done==true.
|
|
type StreamEvent struct {
|
|
Content string // incremental text delta (may be "")
|
|
Reasoning string // incremental thinking delta (may be ""); Anthropic extended thinking, OpenAI reasoning
|
|
ToolCalls []ToolCall // complete tool calls assembled so far
|
|
Done bool
|
|
StopReason string // meaningful when Done==true
|
|
Usage Usage // meaningful only when Done==true
|
|
}
|
|
|
|
// RateLimitError is returned when the backend responds with HTTP 429.
|
|
// RetryAfter is the suggested wait duration; zero means no hint was given.
|
|
type RateLimitError struct {
|
|
RetryAfter time.Duration
|
|
Message string
|
|
}
|
|
|
|
func (e *RateLimitError) Error() string {
|
|
if e.RetryAfter > 0 {
|
|
return fmt.Sprintf("rate limited (retry after %v): %s", e.RetryAfter, e.Message)
|
|
}
|
|
return fmt.Sprintf("rate limited: %s", e.Message)
|
|
}
|
|
|
|
// TransientError is returned for retryable backend failures: 5xx HTTP responses
|
|
// and network-level errors (connection reset, DNS failure, etc.).
|
|
type TransientError struct {
|
|
Message string
|
|
}
|
|
|
|
func (e *TransientError) Error() string { return e.Message }
|
|
|
|
// ContextOverflowError is returned when the request exceeds the model's
|
|
// context window. The caller should compact history and retry.
|
|
type ContextOverflowError struct {
|
|
Message string
|
|
}
|
|
|
|
func (e *ContextOverflowError) Error() string { return e.Message }
|
|
|
|
// ToolUnsupportedError is returned when the model or backend does not support
|
|
// tool/function calling. The request should not be retried with the same model.
|
|
type ToolUnsupportedError struct {
|
|
Message string
|
|
}
|
|
|
|
func (e *ToolUnsupportedError) Error() string { return e.Message }
|
|
|
|
// GenerationParams controls sampling behaviour for a single ChatStream call.
|
|
// Zero values mean "use the API default".
|
|
type GenerationParams struct {
|
|
MaxTokens int // 0 = no limit
|
|
MaxCompletionTokens int // OpenAI o-series; 0 = use MaxTokens
|
|
Temperature *float64 // nil = API default
|
|
TopP *float64 // nil = API default
|
|
TopK *int // nil = API default (Anthropic, Ollama)
|
|
MinP *float64 // nil = API default (Ollama)
|
|
TopA *float64 // nil = API default (Ollama)
|
|
FrequencyPenalty *float64 // nil = API default
|
|
PresencePenalty *float64 // nil = API default
|
|
RepetitionPenalty *float64 // nil = API default (Ollama)
|
|
ThinkingBudget int // Anthropic extended thinking token budget; 0 = disabled
|
|
ReasoningEffort string // OpenAI o-series: "low", "medium", "high"
|
|
IncludeReasoning *bool // include reasoning in response (OpenRouter)
|
|
ResponseFormat string // "json_object", "text", etc.
|
|
Stop []string // stop sequences
|
|
Verbosity string // internal: controls output detail level
|
|
}
|
|
|
|
// Backend is the interface all LLM providers must implement.
|
|
// Streaming is the only supported mode; backends that wrap blocking APIs
|
|
// should implement ChatStream as a single-event stream.
|
|
type Backend interface {
|
|
ChatStream(ctx context.Context, messages []Message, tools []Tool, params GenerationParams) (<-chan StreamEvent, error)
|
|
// Name returns a short human-readable label for this backend (e.g. "ollama", "openrouter").
|
|
Name() string
|
|
// DefaultModel returns a reasonable default model name for this backend.
|
|
DefaultModel() string
|
|
// Model returns the currently active model name.
|
|
Model() string
|
|
// SetModel changes the active model.
|
|
SetModel(model string)
|
|
// ContextLength returns the context window size in tokens for the active model.
|
|
// Returns 0 if unknown. May make an API call; results should be cached by the implementation.
|
|
ContextLength(ctx context.Context) int
|
|
// Models returns the list of available model IDs from the provider.
|
|
Models(ctx context.Context) []string
|
|
}
|
|
|
|
// streamRequest executes req via client, handles status errors, and spawns a
|
|
// goroutine that feeds resp.Body through parseFn into the returned channel.
|
|
// label is used in error messages (e.g. "openai", "ollama").
|
|
// Returns *RateLimitError for 429, *TransientError for 5xx and network errors,
|
|
// and a plain error for other non-200 responses.
|
|
func streamRequest(client *http.Client, req *http.Request, label string, parseFn func(io.Reader, chan<- StreamEvent)) (<-chan StreamEvent, error) {
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, &TransientError{Message: err.Error()}
|
|
}
|
|
if resp.StatusCode == http.StatusTooManyRequests {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
return nil, &RateLimitError{RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")), Message: string(body)}
|
|
}
|
|
if resp.StatusCode >= 500 {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
return nil, &TransientError{Message: fmt.Sprintf("%s HTTP %d: %s", label, resp.StatusCode, string(body))}
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
msg := string(body)
|
|
if resp.StatusCode == http.StatusBadRequest && isContextOverflow(msg) {
|
|
return nil, &ContextOverflowError{Message: fmt.Sprintf("%s HTTP %d: %s", label, resp.StatusCode, msg)}
|
|
}
|
|
if (resp.StatusCode == http.StatusBadRequest || resp.StatusCode == http.StatusUnprocessableEntity || resp.StatusCode == http.StatusNotFound) && isToolUnsupported(msg) {
|
|
return nil, &ToolUnsupportedError{Message: fmt.Sprintf("%s HTTP %d: %s", label, resp.StatusCode, msg)}
|
|
}
|
|
return nil, fmt.Errorf("%s HTTP %d: %s", label, resp.StatusCode, msg)
|
|
}
|
|
ch := make(chan StreamEvent, 8)
|
|
go func() {
|
|
defer close(ch)
|
|
defer resp.Body.Close()
|
|
parseFn(resp.Body, ch)
|
|
}()
|
|
return ch, nil
|
|
}
|
|
|
|
// isToolUnsupported returns true when a 400/422 response body indicates the
|
|
// model does not support tool/function calling.
|
|
func isToolUnsupported(body string) bool {
|
|
markers := []string{
|
|
"tool use is not supported",
|
|
"tools is not supported",
|
|
"function calling is not supported",
|
|
"does not support tools",
|
|
"does not support function",
|
|
"tool_use is not enabled",
|
|
"not support tool",
|
|
"no endpoints found that support tool",
|
|
}
|
|
lower := strings.ToLower(body)
|
|
for _, m := range markers {
|
|
if strings.Contains(lower, m) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// isContextOverflow returns true when a 400 response body indicates the request
|
|
// exceeded the model's context window.
|
|
func isContextOverflow(body string) bool {
|
|
markers := []string{
|
|
"context_length_exceeded", // OpenAI/OpenRouter error code
|
|
"prompt is too long", // Anthropic
|
|
"Please reduce the length", // OpenAI prose
|
|
"too many tokens",
|
|
}
|
|
lower := strings.ToLower(body)
|
|
for _, m := range markers {
|
|
if strings.Contains(lower, strings.ToLower(m)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|