// Package backend defines the Backend interface and shared types for LLM // providers. All backends speak the same canonical types; provider-specific // wire formats are handled inside each implementation. package backend import ( "bufio" "context" "encoding/json" "fmt" "io" "net/http" "strconv" "strings" "time" ) // ContentBlock is a single block within a multi-modal message. // When a Message has ContentBlocks set, it takes precedence over Content. type ContentBlock struct { Type string `json:"type"` // "text" | "image" Text string `json:"text,omitempty"` // for text blocks ImageSource *ImageSource `json:"source,omitempty"` // for image blocks } // ImageSource describes a base64-encoded image. type ImageSource struct { Type string `json:"type"` // "base64" MediaType string `json:"media_type"` // "image/png", "image/jpeg", "image/gif", "image/webp" Data string `json:"data"` // base64-encoded bytes } // Message is a single conversation turn. type Message struct { ID string `json:"id,omitempty"` // stable local ID; not sent to providers Role string `json:"role"` // "system" | "user" | "assistant" | "tool" Content string `json:"content"` Reasoning string `json:"reasoning,omitempty"` // thinking/reasoning content (DeepSeek, OpenAI o-series) ContentBlocks []ContentBlock `json:"content_blocks,omitempty"` // when set, overrides Content ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set by assistant when calling tools ToolCallID string `json:"tool_call_id,omitempty"` // set on role=tool replies (required by OpenAI) ToolName string `json:"tool_name,omitempty"` // local tool identity for history/cache semantics SummaryApplied bool `json:"summary_applied,omitempty"` // local history state } // Tool describes a callable function exposed to the model. type Tool struct { Name string `json:"name"` Description string `json:"description"` Parameters json.RawMessage `json:"parameters"` // JSON Schema object } // ToolCall is the model's request to invoke a function. type ToolCall struct { ID string `json:"id,omitempty"` // provider-assigned; may be empty (Ollama) Name string `json:"name"` Arguments json.RawMessage `json:"arguments"` // always a JSON object } // Usage holds token counts and optional cost for a single Chat call. // CostUSD is non-zero only when the backend reports it directly (e.g. OpenRouter). // CachedInputTokens are tokens served from the prompt cache; they are NOT // included in InputTokens (Anthropic normalises this way; we do the same for // OpenAI by subtracting cached_tokens from prompt_tokens before setting // InputTokens). // CacheCreationTokens are tokens written to the cache (Anthropic only). type Usage struct { InputTokens int CachedInputTokens int CacheCreationTokens int OutputTokens int CostUSD float64 } // ModelPricing holds per-token costs for a model. All values are in USD per token. // Zero values mean pricing is not available or not applicable. type ModelPricing struct { Input float64 // cost per input token Output float64 // cost per output token CacheRead float64 // cost per cached input token (if different from Input) CacheWrite float64 // cost per cache creation token (if applicable) Estimated bool // true if pricing is estimated from static table, not API } // ModelInfo holds extended information about a model. type ModelInfo struct { ID string ContextLength int Pricing *ModelPricing // nil if pricing not available } // StreamEvent is a single increment from a streaming chat call. // Content is an incremental text delta (append, not replace). // Reasoning is an incremental thinking/reasoning delta (append, not replace). // ToolCalls accumulates complete calls; they may arrive on any event. // The final event has Done==true. type StreamEvent struct { Content string // incremental text delta (may be "") Reasoning string // incremental thinking delta (may be ""); Anthropic extended thinking, OpenAI reasoning ToolCalls []ToolCall // complete tool calls assembled so far Done bool StopReason string // meaningful when Done==true Usage Usage // meaningful only when Done==true } // RateLimitError is returned when the backend responds with HTTP 429. // RetryAfter is the suggested wait duration; zero means no hint was given. type RateLimitError struct { RetryAfter time.Duration Message string } func (e *RateLimitError) Error() string { if e.RetryAfter > 0 { return fmt.Sprintf("rate limited (retry after %v): %s", e.RetryAfter, e.Message) } return fmt.Sprintf("rate limited: %s", e.Message) } // TransientError is returned for retryable backend failures: 5xx HTTP responses // and network-level errors (connection reset, DNS failure, etc.). type TransientError struct { Message string } func (e *TransientError) Error() string { return e.Message } // ContextOverflowError is returned when the request exceeds the model's // context window. The caller should compact history and retry. type ContextOverflowError struct { Message string } func (e *ContextOverflowError) Error() string { return e.Message } // ToolUnsupportedError is returned when the model or backend does not support // tool/function calling. The request should not be retried with the same model. type ToolUnsupportedError struct { Message string } func (e *ToolUnsupportedError) Error() string { return e.Message } // GenerationParams controls sampling behaviour for a single ChatStream call. // Zero values mean "use the API default". type GenerationParams struct { MaxTokens int `json:"maxTokens,omitempty"` MaxCompletionTokens int `json:"maxCompletionTokens,omitempty"` Temperature *float64 `json:"temperature,omitempty"` TopP *float64 `json:"topP,omitempty"` TopK *int `json:"topK,omitempty"` MinP *float64 `json:"minP,omitempty"` TopA *float64 `json:"topA,omitempty"` FrequencyPenalty *float64 `json:"frequencyPenalty,omitempty"` PresencePenalty *float64 `json:"presencePenalty,omitempty"` RepetitionPenalty *float64 `json:"repetitionPenalty,omitempty"` ThinkingBudget int `json:"thinkingBudget,omitempty"` ReasoningEffort string `json:"reasoningEffort,omitempty"` IncludeReasoning *bool `json:"includeReasoning,omitempty"` ResponseFormat string `json:"responseFormat,omitempty"` Stop []string `json:"stop,omitempty"` // Environment state for backends that support it (e.g., Kiro). // These are passed through to the backend API but not used for sampling. CWD string `json:"-"` // Current working directory } // Reasoning effort levels. ReasoningEffort is the primary, human-facing knob // for controlling how much a reasoning-capable model deliberates before // answering. Backends that speak a discrete effort level (OpenAI-family // reasoning_effort) send these strings verbatim. Backends that require a // numeric thinking-token budget (Anthropic extended thinking) translate the // level via EffortThinkingBudget. const ( EffortLow = "low" EffortMedium = "medium" EffortHigh = "high" ) // Thinking-token budgets that each reasoning effort level maps to, for backends // that require a numeric budget rather than a discrete level. These are the // single documented source of truth for the thresholds; keep data/agents/README.md // in sync. const ( thinkingBudgetLow = 4096 thinkingBudgetMedium = 8192 thinkingBudgetHigh = 16384 ) // ValidReasoningEffort reports whether s is a recognised effort level. The // empty string is valid and means "unset" (use the API default). func ValidReasoningEffort(s string) bool { switch s { case "", EffortLow, EffortMedium, EffortHigh: return true } return false } // EffortThinkingBudget maps a reasoning effort level to a thinking-token budget. // Returns 0 for an unset or unrecognised level, meaning "no explicit budget". func EffortThinkingBudget(effort string) int { switch effort { case EffortLow: return thinkingBudgetLow case EffortMedium: return thinkingBudgetMedium case EffortHigh: return thinkingBudgetHigh } return 0 } // ResolvedThinkingBudget returns the thinking-token budget to use for backends // that require a numeric budget. An explicit ThinkingBudget always wins; when // it is unset (0), the budget is derived from ReasoningEffort. Returns 0 when // neither is set, meaning the backend should not enable extended thinking. func (p GenerationParams) ResolvedThinkingBudget() int { if p.ThinkingBudget > 0 { return p.ThinkingBudget } return EffortThinkingBudget(p.ReasoningEffort) } // Backend is the interface all LLM providers must implement. // Streaming is the only supported mode; backends that wrap blocking APIs // should implement ChatStream as a single-event stream. type Backend interface { ChatStream(ctx context.Context, messages []Message, tools []Tool, params GenerationParams) (<-chan StreamEvent, error) // Name returns a short human-readable label for this backend (e.g. "ollama", "openrouter"). Name() string // DefaultModel returns a reasonable default model name for this backend. DefaultModel() string // Model returns the currently active model name. Model() string // SetModel changes the active model. SetModel(model string) // ContextLength returns the context window size in tokens for the active model. // Returns 0 if unknown. May make an API call; results should be cached by the implementation. ContextLength(ctx context.Context) int // Models returns the list of available model IDs from the provider. Models(ctx context.Context) []string } // ModelLister is an optional interface backends can implement to provide // extended model information including pricing. The ModelCache uses this // when available. type ModelLister interface { // ModelsInfo returns extended information about available models. // Backends that support pricing should return it in the Pricing field. ModelsInfo(ctx context.Context) []ModelInfo } // staticPriceEntry holds USD per 1M input/output tokens for a model family. // Source: public API pricing pages. More specific prefixes must appear before general ones. type staticPriceEntry struct { prefix string inputPer1M float64 outputPer1M float64 cacheRead float64 // discount rate (0.10 = 10% of input), 0 means use default cacheWrite float64 // premium rate (1.25 = 125% of input), 0 means use default } // staticPrices maps case-insensitive model name prefixes to USD/1M token prices. // Used for estimated pricing when backends don't provide it via API. var staticPrices = []staticPriceEntry{ // Anthropic (cache: read=10%, write=125%) {"claude-opus-4", 15.00, 75.00, 0.10, 1.25}, {"claude-sonnet-4", 3.00, 15.00, 0.10, 1.25}, {"claude-haiku-4", 0.80, 4.00, 0.10, 1.25}, {"claude-3-opus", 15.00, 75.00, 0.10, 1.25}, {"claude-3-5-sonnet", 3.00, 15.00, 0.10, 1.25}, {"claude-3-5-haiku", 1.00, 5.00, 0.10, 1.25}, {"claude-3-sonnet", 3.00, 15.00, 0.10, 1.25}, {"claude-3-haiku", 0.25, 1.25, 0.10, 1.25}, // OpenAI — specific before general (cache: read=50%, write not applicable) {"gpt-4o-mini", 0.15, 0.60, 0.50, 0}, {"gpt-4o", 2.50, 10.00, 0.50, 0}, {"gpt-4-turbo", 10.00, 30.00, 0.50, 0}, {"gpt-4.1", 2.00, 8.00, 0.50, 0}, {"gpt-4.1-mini", 0.40, 1.60, 0.50, 0}, {"gpt-4.1-nano", 0.10, 0.40, 0.50, 0}, {"o4-mini", 1.10, 4.40, 0.275, 0}, {"o3-mini", 1.10, 4.40, 0.275, 0}, {"o3", 10.00, 40.00, 0.25, 0}, {"o1-mini", 1.10, 4.40, 0.275, 0}, {"o1", 15.00, 60.00, 0.375, 0}, // Google Gemini (cache: read=25%) {"gemini-2.5-pro", 1.25, 10.00, 0.25, 0}, {"gemini-2.5-flash", 0.15, 0.60, 0.25, 0}, {"gemini-2.0-flash", 0.10, 0.40, 0.25, 0}, {"gemini-1.5-pro", 1.25, 5.00, 0.25, 0}, {"gemini-1.5-flash", 0.075, 0.30, 0.25, 0}, // DeepSeek {"deepseek-chat", 0.27, 1.10, 0.035, 0}, {"deepseek-reasoner", 0.55, 2.19, 0.14, 0}, {"deepseek-v3", 0.27, 1.10, 0.035, 0}, } // LookupStaticPricing returns estimated pricing for a model ID based on the static table. // Returns nil if no matching prefix is found. Pricing is marked as Estimated. func LookupStaticPricing(modelID string) *ModelPricing { lower := strings.ToLower(modelID) for _, e := range staticPrices { if strings.Contains(lower, e.prefix) { p := &ModelPricing{ Input: e.inputPer1M / 1_000_000, Output: e.outputPer1M / 1_000_000, Estimated: true, } if e.cacheRead > 0 { p.CacheRead = p.Input * e.cacheRead } if e.cacheWrite > 0 { p.CacheWrite = p.Input * e.cacheWrite } return p } } return nil } // streamRequest executes req via client, handles status errors, and spawns a // goroutine that feeds resp.Body through parseFn into the returned channel. // label is used in error messages (e.g. "openai", "ollama"). // Returns *RateLimitError for 429, *TransientError for 5xx and network errors, // and a plain error for other non-200 responses. func streamRequest(client *http.Client, req *http.Request, label string, parseFn func(context.Context, io.Reader, chan<- StreamEvent)) (<-chan StreamEvent, error) { resp, err := client.Do(req) if err != nil { return nil, &TransientError{Message: err.Error()} } if resp.StatusCode == http.StatusTooManyRequests { body, _ := io.ReadAll(resp.Body) resp.Body.Close() return nil, &RateLimitError{RetryAfter: parseRetryAfter(resp.Header.Get("Retry-After")), Message: string(body)} } if resp.StatusCode >= 500 { body, _ := io.ReadAll(resp.Body) resp.Body.Close() return nil, &TransientError{Message: fmt.Sprintf("%s HTTP %d: %s", label, resp.StatusCode, string(body))} } if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) resp.Body.Close() msg := string(body) if resp.StatusCode == http.StatusBadRequest && isContextOverflow(msg) { return nil, &ContextOverflowError{Message: fmt.Sprintf("%s HTTP %d: %s", label, resp.StatusCode, msg)} } if (resp.StatusCode == http.StatusBadRequest || resp.StatusCode == http.StatusUnprocessableEntity || resp.StatusCode == http.StatusNotFound) && isToolUnsupported(msg) { return nil, &ToolUnsupportedError{Message: fmt.Sprintf("%s HTTP %d: %s", label, resp.StatusCode, msg)} } return nil, fmt.Errorf("%s HTTP %d: %s", label, resp.StatusCode, msg) } ch := make(chan StreamEvent, 8) go func() { defer close(ch) defer resp.Body.Close() parseFn(req.Context(), resp.Body, ch) }() return ch, nil } func sendStreamEvent(ctx context.Context, ch chan<- StreamEvent, ev StreamEvent) bool { select { case ch <- ev: return true case <-ctx.Done(): return false } } // model does not support tool/function calling. func isToolUnsupported(body string) bool { markers := []string{ "tool use is not supported", "tools is not supported", "function calling is not supported", "does not support tools", "does not support function", "tool_use is not enabled", "not support tool", "no endpoints found that support tool", } lower := strings.ToLower(body) for _, m := range markers { if strings.Contains(lower, m) { return true } } return false } // isContextOverflow returns true when a 400 response body indicates the request // exceeded the model's context window. func isContextOverflow(body string) bool { markers := []string{ "context_length_exceeded", // OpenAI/OpenRouter error code "prompt is too long", // Anthropic "Please reduce the length", // OpenAI prose "too many tokens", } lower := strings.ToLower(body) for _, m := range markers { if strings.Contains(lower, strings.ToLower(m)) { return true } } return false } // parseRetryAfter parses the Retry-After header value, which may be an integer // number of seconds or an HTTP-date. Returns zero if the header is absent or // unparseable. func parseRetryAfter(header string) time.Duration { if header == "" { return 0 } header = strings.TrimSpace(header) if secs, err := strconv.Atoi(header); err == nil { return time.Duration(secs) * time.Second } if t, err := http.ParseTime(header); err == nil { if d := time.Until(t); d > 0 { return d } } return 0 } // newStreamScanner returns a bufio.Scanner suitable for reading SSE/NDJSON // streaming responses. The buffer is sized for typical LLM chunks (64K initial, // 4MB max). func newStreamScanner(r io.Reader) *bufio.Scanner { s := bufio.NewScanner(r) s.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) return s }