backend: shared HTTP transport pool for connection multiplexing

All backend constructors now use SharedClient() which returns an
*http.Client backed by a per-host shared *http.Transport. This enables
HTTP/2 multiplexing: multiple sessions targeting the same API host
share a single TLS connection instead of each maintaining their own.

The pool is keyed by "backend:baseURL" so different endpoints get
separate transports while same-host sessions multiplex.
This commit is contained in:
ollie 2026-07-18 22:02:10 +02:00
parent b4c88a33d3
commit 2f93a45031
6 changed files with 75 additions and 6 deletions

View File

@ -24,7 +24,7 @@ type AnthropicBackend struct {
func NewAnthropic(apiKey string) (*AnthropicBackend, error) {
u, _ := url.Parse("https://api.anthropic.com")
b := &AnthropicBackend{apiKey: apiKey, client: &http.Client{}, BaseURL: u}
b := &AnthropicBackend{apiKey: apiKey, client: SharedClient("anthropic:" + u.String()), BaseURL: u}
b.model = b.DefaultModel()
return b, nil
}

View File

@ -57,7 +57,7 @@ func NewCodeWhisperer(apiKey string) (*CodeWhispererBackend, error) {
b := &CodeWhispererBackend{
authSource: authSource,
authInitErr: err,
httpClient: &http.Client{},
httpClient: SharedClient("kiro:https://codewhisperer.us-east-1.amazonaws.com"),
}
b.model = b.DefaultModel()
return b, nil

View File

@ -1,7 +1,6 @@
package backend
import (
"net/http"
"net/url"
)
@ -13,7 +12,7 @@ func NewCopilot(token string) (*OpenAIBackend, error) {
name: "copilot",
baseURL: u,
apiKey: token,
client: &http.Client{},
client: SharedClient("copilot:" + u.String()),
pathPrefix: "/",
extraHeaders: map[string]string{
"Openai-Intent": "conversation-edits",

View File

@ -29,7 +29,7 @@ func NewOllama(baseURL string) (*OllamaBackend, error) {
if err != nil {
return nil, fmt.Errorf("invalid base URL: %w", err)
}
b := &OllamaBackend{baseURL: u, client: &http.Client{}}
b := &OllamaBackend{baseURL: u, client: SharedClient("ollama:" + u.String())}
b.model = b.DefaultModel()
return b, nil
}

View File

@ -42,7 +42,7 @@ func NewOpenAI(name, baseURL, apiKey string) (*OpenAIBackend, error) {
if err != nil {
return nil, fmt.Errorf("invalid base URL: %w", err)
}
b := &OpenAIBackend{name: name, baseURL: u, apiKey: apiKey, client: &http.Client{}}
b := &OpenAIBackend{name: name, baseURL: u, apiKey: apiKey, client: SharedClient(name + ":" + u.String())}
b.model = b.DefaultModel()
return b, nil
}

70
pkg/backend/pool.go Normal file
View File

@ -0,0 +1,70 @@
// Package backend — shared HTTP transport pool.
//
// SharedTransport returns a per-host *http.Transport that all backend instances
// targeting the same base URL share. This enables HTTP/2 multiplexing: multiple
// concurrent ChatStream calls share a single TLS connection to the same host.
package backend
import (
"crypto/tls"
"net"
"net/http"
"sync"
"time"
)
var (
transportsMu sync.Mutex
transports = make(map[string]*http.Transport)
)
// SharedTransport returns a shared *http.Transport for the given host key.
// The key should be "backend:baseURL" (e.g. "anthropic:https://api.anthropic.com").
// All callers sharing the same key share the same underlying TCP+TLS connections.
func SharedTransport(key string) *http.Transport {
transportsMu.Lock()
defer transportsMu.Unlock()
if t, ok := transports[key]; ok {
return t
}
t := &http.Transport{
ForceAttemptHTTP2: true,
MaxIdleConnsPerHost: 4,
IdleConnTimeout: 5 * time.Minute,
TLSHandshakeTimeout: 10 * time.Second,
// No ResponseHeaderTimeout — streaming responses have no header deadline.
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
TLSClientConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
},
}
transports[key] = t
return t
}
// SharedClient returns an *http.Client backed by the shared transport for key.
func SharedClient(key string) *http.Client {
return &http.Client{Transport: SharedTransport(key)}
}
// PoolStats returns the number of active transport entries.
func PoolStats() int {
transportsMu.Lock()
defer transportsMu.Unlock()
return len(transports)
}
// ClosePool closes all idle connections in all transports and clears the pool.
func ClosePool() {
transportsMu.Lock()
defer transportsMu.Unlock()
for _, t := range transports {
t.CloseIdleConnections()
}
transports = make(map[string]*http.Transport)
}