lkn
/
ollie
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cache cold result summaries

This commit is contained in:
Ollie Agent 2026-08-15 13:10:15 +02:00
parent b6584d35e7
commit 25d502cfc2
2 changed files with 74 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
cmd/olliesrv/internal/agent/history.go
View File

@ -38,7 +38,8 @@ Be concise. Capture what another LLM needs to seamlessly continue.`
// Used for multi-agent session restore where each agent has separate messages.
func RestoreHistoryFromMessages(messages []backend.Message) *History {
s := &History{
messages: messages,
messages: messages,
summaryCache: make(map[string]string),
}
for i := range s.messages {
if s.messages[i].Role == "assistant" && s.messages[i].ID == "" {
@ -65,6 +66,10 @@ func RestoreHistoryFromMessages(messages []backend.Message) *History {
type History struct {
goal string
messages []backend.Message
// summaryCache reuses validated summaries for identical tool-result content
// during the lifetime of this agent. SummaryApplied persists per-message
// state across session restore; this cache avoids new LLM calls for repeats.
summaryCache map[string]string
// Cumulative usage tracking.
TotalInputTokens int
TotalCachedInputTokens int
@ -85,7 +90,7 @@ type History struct {
// newHistory creates a new empty History. The caller is responsible for
// appending the initial user message via appendUserMessage.
func newHistory(goal string) *History {
return &History{goal: goal}
return &History{goal: goal, summaryCache: make(map[string]string)}
}
func toolMessageID(m backend.Message) string {
@ -96,6 +101,14 @@ func toolMessageID(m backend.Message) string {
return "tool:content:" + hex.EncodeToString(h[:])
}
func toolSummaryHash(m backend.Message) string {
h := sha256.New()
h.Write([]byte(m.ToolCallID))
h.Write([]byte{0})
h.Write([]byte(m.Content))
return hex.EncodeToString(h.Sum(nil))
}
func (s *History) history() []backend.Message {
return s.messages
}
@ -404,6 +417,9 @@ func (s *History) estimateTokens() int {
// using a single batched LLM call. Messages in the last hotTailSize slots
// are left verbatim.
func (s *History) stripCold(ctx context.Context, b backend.Backend) (backend.Usage, bool) {
if s.summaryCache == nil {
s.summaryCache = make(map[string]string)
}
hot := len(s.messages) - hotTailSize
if hot < 0 {
return backend.Usage{}, false
@ -411,10 +427,10 @@ func (s *History) stripCold(ctx context.Context, b backend.Backend) (backend.Usa
// Collect cold-zone tool results exceeding 200 chars.
type item struct {
id int
idx int // index into s.messages
orig string
messageID string
hash string
}
var items []item
for i := range s.messages {
@ -433,7 +449,12 @@ func (s *History) stripCold(ctx context.Context, b backend.Backend) (backend.Usa
m.ID = id
}
if !m.SummaryApplied {
items = append(items, item{id: len(items), idx: i, orig: m.Content})
if summary, ok := s.summaryCache[toolSummaryHash(*m)]; ok {
m.Content = summary
m.SummaryApplied = true
continue
}
items = append(items, item{idx: i, orig: m.Content, messageID: id, hash: toolSummaryHash(*m)})
}
}
}
@ -443,9 +464,9 @@ func (s *History) stripCold(ctx context.Context, b backend.Backend) (backend.Usa
// Build a single prompt listing each item.
var sb strings.Builder
sb.WriteString("Compress each of the following tool results to 1-3 sentences preserving key facts, file paths, values, and findings.\n\nReturn a JSON object where each key is the id and the value is the summary, for example:\n{\"0\": \"summarized result 0\", \"1\": \"summarized result 1\"}\n\nOnly include keys that were in the input.\n\n")
sb.WriteString("Compress each of the following tool results to 1-3 sentences preserving key facts, file paths, values, and findings.\n\nReturn ONLY a valid JSON object. Do not use Markdown fences, prose, or any text outside the JSON object. Each key must be exactly one of the supplied message IDs and each value must be a string summary, for example:\n{\"tool:call-123\": \"summarized result\", \"tool:call-456\": \"another summary\"}\n\nOnly include keys that were in the input.\n\n")
for _, it := range items {
fmt.Fprintf(&sb, "---\nid=%d:\n%s\n", it.id, it.orig)
fmt.Fprintf(&sb, "---\nid=%s:\n%s\n", it.messageID, it.orig)
}
ch, err := b.ChatStream(ctx, []backend.Message{{Role: "user", Content: sb.String()}}, nil, backend.GenerationParams{})
@ -478,8 +499,8 @@ func (s *History) stripCold(ctx context.Context, b backend.Backend) (backend.Usa
// Apply summaries back into history.
for _, it := range items {
key := fmt.Sprintf("%d", it.id)
if summary, ok := summaries[key]; ok && summary != "" {
if summary, ok := summaries[it.messageID]; ok && summary != "" {
s.summaryCache[it.hash] = summary
s.messages[it.idx].Content = summary
s.messages[it.idx].SummaryApplied = true
}

46
cmd/olliesrv/internal/agent/history_summary_test.go
View File

@ -54,12 +54,56 @@ func TestStripColdSkipsAlreadySummarized(t *testing.T) {
}
}
func TestStripColdReusesSummaryCache(t *testing.T) {
b := &summaryBackend{}
h := newHistory("test")
for i := 0; i < 9; i++ {
m := backend.Message{Role: "user", Content: "hot"}
if i < 1 {
m = backend.Message{Role: "tool", ToolCallID: "tc-0", Content: strings.Repeat("same result ", 30)}
}
h.messages = append(h.messages, m)
}
if _, _ = h.stripCold(context.Background(), b); b.calls != 1 {
t.Fatalf("first stripCold calls = %d, want 1", b.calls)
}
original := strings.Repeat("same result ", 30)
h.messages[0].Content = original
h.messages[0].SummaryApplied = false
if _, _ = h.stripCold(context.Background(), b); b.calls != 1 {
t.Fatalf("cached stripCold calls = %d, want 1", b.calls)
}
if !h.messages[0].SummaryApplied || h.messages[0].Content != "summary" {
t.Fatalf("cached summary was not applied: %#v", h.messages[0])
}
}
func TestStripColdSkipsAlreadySummarizedAfterRestore(t *testing.T) {
b := &summaryBackend{}
h := RestoreHistoryFromMessages([]backend.Message{
{Role: "tool", ToolCallID: "tc-0", Content: strings.Repeat("summary ", 30), SummaryApplied: true},
{Role: "user", Content: "hot"},
{Role: "user", Content: "hot"},
{Role: "user", Content: "hot"},
{Role: "user", Content: "hot"},
{Role: "user", Content: "hot"},
{Role: "user", Content: "hot"},
{Role: "user", Content: "hot"},
{Role: "user", Content: "hot"},
})
if _, _ = h.stripCold(context.Background(), b); b.calls != 0 {
t.Fatalf("restored summarized history calls = %d, want 0", b.calls)
}
}
type summaryBackend struct{ calls int }
func (b *summaryBackend) ChatStream(_ context.Context, messages []backend.Message, _ []backend.Tool, _ backend.GenerationParams) (<-chan backend.StreamEvent, error) {
b.calls++
if len(messages) != 1 || !strings.Contains(messages[0].Content, "id=tool:tc-0:") {
return nil, fmt.Errorf("summary request did not use stable message ID")
}
out := make(chan backend.StreamEvent, 1)
out <- backend.StreamEvent{Content: `{"0":"summary"}`, Done: true}
out <- backend.StreamEvent{Content: `{"tool:tc-0":"summary"}`, Done: true}
close(out)
return out, nil
}