134 lines
3.6 KiB
Go
134 lines
3.6 KiB
Go
// retry.go — Error tracking and transient retry logic.
|
|
//
|
|
// Tracks consecutive tool errors and repeated error messages. At the soft
|
|
// limit, nudges the model to try a different approach. At the hard limit,
|
|
// aborts the turn. Handles transient errors (rate limits, network) with
|
|
// exponential backoff.
|
|
|
|
package agent
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"ollie/cmd/olliesrv/internal/backend"
|
|
)
|
|
|
|
// errorState tracks consecutive and repeated tool errors across turns.
|
|
type errorState struct {
|
|
consecutive int
|
|
lastSig string
|
|
repeatCount int
|
|
}
|
|
|
|
// trackErrors updates error state and injects nudge messages when thresholds are hit.
|
|
func (ag *Agent) trackErrors(ctx context.Context, results []toolResult, es *errorState, step int) error {
|
|
h := ag.history
|
|
|
|
allErrors := len(results) > 0
|
|
allRateLimited := allErrors
|
|
for _, r := range results {
|
|
if !r.IsError {
|
|
allErrors = false
|
|
allRateLimited = false
|
|
break
|
|
}
|
|
if !strings.HasPrefix(r.Content, "error: shell is rate-limited") {
|
|
allRateLimited = false
|
|
}
|
|
}
|
|
|
|
if allRateLimited {
|
|
es.consecutive = 0
|
|
} else if allErrors {
|
|
es.consecutive++
|
|
} else {
|
|
es.consecutive = 0
|
|
}
|
|
|
|
if es.consecutive >= consecutiveErrorHardLimit {
|
|
ag.emit(Event{Role: "error", Name: backend.SeverityFatal, Content: fmt.Sprintf("%d consecutive tool errors — aborting\nremediation: the model is stuck; rephrase the task or intervene", es.consecutive)})
|
|
return fmt.Errorf("step %d: %d consecutive tool errors", step, es.consecutive)
|
|
}
|
|
if es.consecutive == consecutiveErrorSoftLimit {
|
|
h.update(backend.Message{
|
|
Role: "user",
|
|
Content: "<system-consecutive-errors>\nYour last several tool calls all failed. Try a different approach, or ask the user for help. Keep your plan current.\n</system-consecutive-errors>",
|
|
}, nil)
|
|
}
|
|
|
|
// Repeated identical error detection.
|
|
if allErrors && len(results) > 0 {
|
|
var errSig strings.Builder
|
|
for _, r := range results {
|
|
if r.IsError {
|
|
errSig.WriteString(r.Content)
|
|
errSig.WriteByte('\n')
|
|
}
|
|
}
|
|
sig := errSig.String()
|
|
if sig == es.lastSig {
|
|
es.repeatCount++
|
|
} else {
|
|
es.repeatCount = 1
|
|
es.lastSig = sig
|
|
}
|
|
if es.repeatCount == 3 {
|
|
h.update(backend.Message{
|
|
Role: "user",
|
|
Content: "<system-repeated-error>\nYou have made the same malformed tool call 3 times in a row. Read the error message carefully and fix the arguments. The error was:\n" + strings.TrimSpace(results[0].Content) + "\n</system-repeated-error>",
|
|
}, nil)
|
|
}
|
|
} else {
|
|
es.repeatCount = 0
|
|
es.lastSig = ""
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// transientWait returns the wait duration and whether the error is retryable.
|
|
func transientWait(err error, attempt int) (time.Duration, bool) {
|
|
var rlErr *backend.RateLimitError
|
|
if errors.As(err, &rlErr) {
|
|
wait := rlErr.RetryAfter
|
|
if wait == 0 {
|
|
wait = retryBaseDelay << attempt
|
|
}
|
|
return wait, true
|
|
}
|
|
var tErr *backend.TransientError
|
|
if errors.As(err, &tErr) {
|
|
return time.Duration(2<<attempt) * time.Second, true
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
// retryCountdown emits countdown events while waiting for rate limit to expire.
|
|
func (ag *Agent) retryCountdown(ctx context.Context, wait time.Duration) error {
|
|
deadline := time.Now().Add(wait)
|
|
for {
|
|
remaining := time.Until(deadline)
|
|
if remaining <= 0 {
|
|
return nil
|
|
}
|
|
totalSecs := int(remaining.Seconds()) + 1
|
|
h := totalSecs / 3600
|
|
m := (totalSecs % 3600) / 60
|
|
s := totalSecs % 60
|
|
ag.emit(Event{Role: "retry", Content: fmt.Sprintf("%02d:%02d:%02d", h, m, s)})
|
|
timer := time.NewTimer(min(remaining, time.Second))
|
|
select {
|
|
case <-ctx.Done():
|
|
if !timer.Stop() {
|
|
<-timer.C
|
|
}
|
|
return ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
}
|