backend: shared HTTP transport pool for connection multiplexing
All backend constructors now use SharedClient() which returns an *http.Client backed by a per-host shared *http.Transport. This enables HTTP/2 multiplexing: multiple sessions targeting the same API host share a single TLS connection instead of each maintaining their own. The pool is keyed by "backend:baseURL" so different endpoints get separate transports while same-host sessions multiplex.
This commit is contained in:
parent
b4c88a33d3
commit
2f93a45031
|
|
@ -24,7 +24,7 @@ type AnthropicBackend struct {
|
|||
|
||||
func NewAnthropic(apiKey string) (*AnthropicBackend, error) {
|
||||
u, _ := url.Parse("https://api.anthropic.com")
|
||||
b := &AnthropicBackend{apiKey: apiKey, client: &http.Client{}, BaseURL: u}
|
||||
b := &AnthropicBackend{apiKey: apiKey, client: SharedClient("anthropic:" + u.String()), BaseURL: u}
|
||||
b.model = b.DefaultModel()
|
||||
return b, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ func NewCodeWhisperer(apiKey string) (*CodeWhispererBackend, error) {
|
|||
b := &CodeWhispererBackend{
|
||||
authSource: authSource,
|
||||
authInitErr: err,
|
||||
httpClient: &http.Client{},
|
||||
httpClient: SharedClient("kiro:https://codewhisperer.us-east-1.amazonaws.com"),
|
||||
}
|
||||
b.model = b.DefaultModel()
|
||||
return b, nil
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package backend
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
|
|
@ -13,7 +12,7 @@ func NewCopilot(token string) (*OpenAIBackend, error) {
|
|||
name: "copilot",
|
||||
baseURL: u,
|
||||
apiKey: token,
|
||||
client: &http.Client{},
|
||||
client: SharedClient("copilot:" + u.String()),
|
||||
pathPrefix: "/",
|
||||
extraHeaders: map[string]string{
|
||||
"Openai-Intent": "conversation-edits",
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ func NewOllama(baseURL string) (*OllamaBackend, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid base URL: %w", err)
|
||||
}
|
||||
b := &OllamaBackend{baseURL: u, client: &http.Client{}}
|
||||
b := &OllamaBackend{baseURL: u, client: SharedClient("ollama:" + u.String())}
|
||||
b.model = b.DefaultModel()
|
||||
return b, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ func NewOpenAI(name, baseURL, apiKey string) (*OpenAIBackend, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid base URL: %w", err)
|
||||
}
|
||||
b := &OpenAIBackend{name: name, baseURL: u, apiKey: apiKey, client: &http.Client{}}
|
||||
b := &OpenAIBackend{name: name, baseURL: u, apiKey: apiKey, client: SharedClient(name + ":" + u.String())}
|
||||
b.model = b.DefaultModel()
|
||||
return b, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
// Package backend — shared HTTP transport pool.
|
||||
//
|
||||
// SharedTransport returns a per-host *http.Transport that all backend instances
|
||||
// targeting the same base URL share. This enables HTTP/2 multiplexing: multiple
|
||||
// concurrent ChatStream calls share a single TLS connection to the same host.
|
||||
package backend
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
transportsMu sync.Mutex
|
||||
transports = make(map[string]*http.Transport)
|
||||
)
|
||||
|
||||
// SharedTransport returns a shared *http.Transport for the given host key.
|
||||
// The key should be "backend:baseURL" (e.g. "anthropic:https://api.anthropic.com").
|
||||
// All callers sharing the same key share the same underlying TCP+TLS connections.
|
||||
func SharedTransport(key string) *http.Transport {
|
||||
transportsMu.Lock()
|
||||
defer transportsMu.Unlock()
|
||||
|
||||
if t, ok := transports[key]; ok {
|
||||
return t
|
||||
}
|
||||
|
||||
t := &http.Transport{
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConnsPerHost: 4,
|
||||
IdleConnTimeout: 5 * time.Minute,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
// No ResponseHeaderTimeout — streaming responses have no header deadline.
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
TLSClientConfig: &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
},
|
||||
}
|
||||
transports[key] = t
|
||||
return t
|
||||
}
|
||||
|
||||
// SharedClient returns an *http.Client backed by the shared transport for key.
|
||||
func SharedClient(key string) *http.Client {
|
||||
return &http.Client{Transport: SharedTransport(key)}
|
||||
}
|
||||
|
||||
// PoolStats returns the number of active transport entries.
|
||||
func PoolStats() int {
|
||||
transportsMu.Lock()
|
||||
defer transportsMu.Unlock()
|
||||
return len(transports)
|
||||
}
|
||||
|
||||
// ClosePool closes all idle connections in all transports and clears the pool.
|
||||
func ClosePool() {
|
||||
transportsMu.Lock()
|
||||
defer transportsMu.Unlock()
|
||||
for _, t := range transports {
|
||||
t.CloseIdleConnections()
|
||||
}
|
||||
transports = make(map[string]*http.Transport)
|
||||
}
|
||||
Reference in New Issue