agent: parse text-based tool calls instead of failing
Models that don't support native function calling emit tool calls as plain text. Instead of treating this as ToolUnsupportedError, parse the text and execute the tool calls normally. Handles both direct JSON and shorthand (key=value) formats with relaxed JSON support (unquoted keys).
This commit is contained in:
Levi Neely
parent
1485decdbb
commit
9d6800a51c
|
|
@ -249,15 +249,13 @@ func run(ctx context.Context, cfg agentConfig, state state) error {
|
|||
return fmt.Errorf("step %d: %s", step, stopReason)
|
||||
}
|
||||
|
||||
// Detect models that emit tool calls as plain text instead of using the
|
||||
// function-calling API. If the response contains no API-level tool calls
|
||||
// but the text matches a registered tool name followed by ':' or '(',
|
||||
// the model does not support function calling — treat as ToolUnsupportedError
|
||||
// so freeloader can blacklist and cycle to the next model.
|
||||
if len(toolCalls) == 0 && len(cfg.Tools) > 0 && hasTextToolCall(content.String(), cfg.Tools) {
|
||||
return fmt.Errorf("step %d: %w", step, &backend.ToolUnsupportedError{
|
||||
Message: "model emitted tool call as text (function calling API not supported)",
|
||||
})
|
||||
// Parse text-based tool calls: models that don't support the function
|
||||
// calling API emit tool invocations as plain text (e.g.
|
||||
// "call_tool: calls=[...]"). Parse these into proper ToolCall structs.
|
||||
if len(toolCalls) == 0 && len(cfg.Tools) > 0 {
|
||||
if parsed := parseTextToolCalls(content.String(), cfg.Tools); len(parsed) > 0 {
|
||||
toolCalls = parsed
|
||||
}
|
||||
}
|
||||
|
||||
// Execute tool calls, running consecutive parallel-read-safe tools concurrently.
|
||||
|
|
@ -638,16 +636,280 @@ func inferTaskStateUpdate(ts *TaskState, msg backend.Message, results []toolResu
|
|||
}
|
||||
}
|
||||
|
||||
// hasTextToolCall returns true when the assistant text contains a registered
|
||||
// tool name immediately followed by ':' or '(', indicating the model tried to
|
||||
// invoke a tool by writing the call as plain text rather than via the API.
|
||||
func hasTextToolCall(text string, tools []backend.Tool) bool {
|
||||
// parseTextToolCalls extracts tool calls from assistant text when the model
|
||||
// emits them as plain text instead of using the function calling API.
|
||||
// Recognizes formats:
|
||||
// - tool_name: {"key": ...} (JSON object)
|
||||
// - tool_name: key=[...], key2=... (shorthand: key=value pairs → JSON object)
|
||||
// Returns nil if no valid tool calls are found.
|
||||
func parseTextToolCalls(text string, tools []backend.Tool) []backend.ToolCall {
|
||||
toolNames := make(map[string]bool, len(tools))
|
||||
for _, t := range tools {
|
||||
if strings.Contains(text, t.Name+":") || strings.Contains(text, t.Name+"(") {
|
||||
return true
|
||||
toolNames[t.Name] = true
|
||||
}
|
||||
|
||||
var calls []backend.ToolCall
|
||||
for _, name := range []string{"execute_code", "call_tool", "pipe"} {
|
||||
if !toolNames[name] {
|
||||
continue
|
||||
}
|
||||
prefix := name + ":"
|
||||
idx := strings.Index(text, prefix)
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
rest := strings.TrimSpace(text[idx+len(prefix):])
|
||||
if rest == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
var argsJSON string
|
||||
if rest[0] == '{' {
|
||||
// Direct JSON object
|
||||
argsJSON = extractJSONObject(rest)
|
||||
if argsJSON == "" {
|
||||
argsJSON = extractJSONObject(relaxJSON(rest))
|
||||
}
|
||||
} else {
|
||||
// Shorthand format: key=[...], key2=value
|
||||
argsJSON = shorthandToJSON(rest)
|
||||
}
|
||||
if argsJSON == "" {
|
||||
continue
|
||||
}
|
||||
var check json.RawMessage
|
||||
if json.Unmarshal([]byte(argsJSON), &check) != nil {
|
||||
continue
|
||||
}
|
||||
calls = append(calls, backend.ToolCall{
|
||||
ID: fmt.Sprintf("text-%s-%d", name, len(calls)),
|
||||
Name: name,
|
||||
Arguments: json.RawMessage(argsJSON),
|
||||
})
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
// shorthandToJSON converts "key=[...], key2=value" format to a JSON object.
|
||||
// Handles: steps=[{code: "date"}], timeout=30
|
||||
func shorthandToJSON(s string) string {
|
||||
// Find the first key=value where value starts with [ or {
|
||||
eqIdx := strings.IndexByte(s, '=')
|
||||
if eqIdx < 0 {
|
||||
return ""
|
||||
}
|
||||
key := strings.TrimSpace(s[:eqIdx])
|
||||
if key == "" {
|
||||
return ""
|
||||
}
|
||||
valStr := s[eqIdx+1:]
|
||||
|
||||
// Extract the value (balanced brackets/braces)
|
||||
val := extractBalanced(strings.TrimSpace(valStr))
|
||||
if val == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Relax the value's JSON (unquoted keys → quoted)
|
||||
relaxed := relaxJSON(val)
|
||||
|
||||
// Build the JSON object: {"key": relaxed_value}
|
||||
keyJSON, _ := json.Marshal(key)
|
||||
result := "{" + string(keyJSON) + ":" + relaxed + "}"
|
||||
|
||||
// Check if there are more key=value pairs after the value
|
||||
afterVal := strings.TrimSpace(valStr[len(val):])
|
||||
if len(afterVal) > 0 && afterVal[0] == ',' {
|
||||
// Parse additional key=value pairs
|
||||
extra := parseExtraKV(afterVal[1:])
|
||||
if extra != "" {
|
||||
// Merge: strip trailing } from result, append extra
|
||||
result = result[:len(result)-1] + "," + extra + "}"
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// extractBalanced extracts a balanced [...] or {...} from the start of s.
|
||||
func extractBalanced(s string) string {
|
||||
if len(s) == 0 {
|
||||
return ""
|
||||
}
|
||||
open := s[0]
|
||||
var close byte
|
||||
switch open {
|
||||
case '[':
|
||||
close = ']'
|
||||
case '{':
|
||||
close = '}'
|
||||
default:
|
||||
// Simple value (number, string) — take until comma or newline
|
||||
end := strings.IndexAny(s, ",\n")
|
||||
if end < 0 {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
return strings.TrimSpace(s[:end])
|
||||
}
|
||||
|
||||
depth := 0
|
||||
inStr := false
|
||||
escaped := false
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if c == '\\' && inStr {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if c == '"' {
|
||||
inStr = !inStr
|
||||
continue
|
||||
}
|
||||
if inStr {
|
||||
continue
|
||||
}
|
||||
if c == open {
|
||||
depth++
|
||||
} else if c == close {
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return s[:i+1]
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseExtraKV parses "key=value, key2=value2" into JSON fields (without outer braces).
|
||||
func parseExtraKV(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
var parts []string
|
||||
for s != "" {
|
||||
eqIdx := strings.IndexByte(s, '=')
|
||||
if eqIdx < 0 {
|
||||
break
|
||||
}
|
||||
key := strings.TrimSpace(s[:eqIdx])
|
||||
s = strings.TrimSpace(s[eqIdx+1:])
|
||||
val := extractBalanced(s)
|
||||
if val == "" {
|
||||
break
|
||||
}
|
||||
s = strings.TrimSpace(s[len(val):])
|
||||
if len(s) > 0 && s[0] == ',' {
|
||||
s = s[1:]
|
||||
}
|
||||
keyJSON, _ := json.Marshal(key)
|
||||
parts = append(parts, string(keyJSON)+":"+relaxJSON(val))
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
// extractJSONObject finds the first balanced {...} in s.
|
||||
func extractJSONObject(s string) string {
|
||||
start := strings.IndexByte(s, '{')
|
||||
if start < 0 {
|
||||
return ""
|
||||
}
|
||||
depth := 0
|
||||
inStr := false
|
||||
escaped := false
|
||||
for i := start; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if c == '\\' && inStr {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if c == '"' {
|
||||
inStr = !inStr
|
||||
continue
|
||||
}
|
||||
if inStr {
|
||||
continue
|
||||
}
|
||||
if c == '{' {
|
||||
depth++
|
||||
} else if c == '}' {
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return s[start : i+1]
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// relaxJSON converts common relaxed-JSON patterns to valid JSON:
|
||||
// - unquoted keys (word: → "word":)
|
||||
// - single-quoted strings → double-quoted
|
||||
// This is a best-effort heuristic for model output.
|
||||
func relaxJSON(s string) string {
|
||||
var out strings.Builder
|
||||
out.Grow(len(s))
|
||||
i := 0
|
||||
for i < len(s) {
|
||||
c := s[i]
|
||||
// Single-quoted string → double-quoted
|
||||
if c == '\'' {
|
||||
out.WriteByte('"')
|
||||
i++
|
||||
for i < len(s) && s[i] != '\'' {
|
||||
if s[i] == '"' {
|
||||
out.WriteString(`\"`)
|
||||
} else if s[i] == '\\' && i+1 < len(s) {
|
||||
out.WriteByte(s[i])
|
||||
i++
|
||||
out.WriteByte(s[i])
|
||||
} else {
|
||||
out.WriteByte(s[i])
|
||||
}
|
||||
i++
|
||||
}
|
||||
out.WriteByte('"')
|
||||
if i < len(s) {
|
||||
i++ // skip closing '
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Unquoted key before colon: word followed by optional whitespace then ':'
|
||||
if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '_' {
|
||||
j := i
|
||||
for j < len(s) && (s[j] >= 'a' && s[j] <= 'z' || s[j] >= 'A' && s[j] <= 'Z' || s[j] >= '0' && s[j] <= '9' || s[j] == '_') {
|
||||
j++
|
||||
}
|
||||
// Check if followed by optional whitespace then ':'
|
||||
k := j
|
||||
for k < len(s) && (s[k] == ' ' || s[k] == '\t') {
|
||||
k++
|
||||
}
|
||||
if k < len(s) && s[k] == ':' {
|
||||
// It's an unquoted key — quote it
|
||||
out.WriteByte('"')
|
||||
out.WriteString(s[i:j])
|
||||
out.WriteByte('"')
|
||||
i = j
|
||||
continue
|
||||
}
|
||||
// Not a key, just copy the word
|
||||
out.WriteString(s[i:j])
|
||||
i = j
|
||||
continue
|
||||
}
|
||||
out.WriteByte(c)
|
||||
i++
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// classifyError returns a short string identifying the error type for the
|
||||
|
|
|
|||
|
|
@ -2,39 +2,88 @@ package agent
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"ollie/pkg/backend"
|
||||
)
|
||||
|
||||
// TestTextToolCall_DetectedAsUnsupported verifies that when a model emits a tool
|
||||
// call as plain text (no API-level tool calls) the loop returns a
|
||||
// ToolUnsupportedError so freeloader can blacklist and cycle.
|
||||
func TestTextToolCall_DetectedAsUnsupported(t *testing.T) {
|
||||
// TestTextToolCall_ParsedAndExecuted verifies that when a model emits a tool
|
||||
// call as plain text (no API-level tool calls) the loop parses it and executes
|
||||
// it as a normal tool call.
|
||||
func TestTextToolCall_ParsedAndExecuted(t *testing.T) {
|
||||
be := defaultBE()
|
||||
// Model responds with text that mimics a tool invocation instead of using
|
||||
// the function-calling API.
|
||||
be.respond = func(_ context.Context, _ []backend.Message, _ []backend.Tool, _ backend.GenerationParams) (<-chan backend.StreamEvent, error) {
|
||||
return textStream(`execute_code: steps=[{code: "print('hi')"}]`), nil
|
||||
be.respond = func(_ context.Context, msgs []backend.Message, _ []backend.Tool, _ backend.GenerationParams) (<-chan backend.StreamEvent, error) {
|
||||
// First call: model emits text tool call
|
||||
for _, m := range msgs {
|
||||
if m.Role == "tool" {
|
||||
// Second call after tool result: respond normally
|
||||
return textStream("done"), nil
|
||||
}
|
||||
}
|
||||
return textStream(`call_tool: {"calls":[{"tool":"file_read","args":["/tmp/test.go"]}]}`), nil
|
||||
}
|
||||
|
||||
var execCalled bool
|
||||
c := newCore(t, be, nil)
|
||||
c.runtime.Tools = []backend.Tool{{Name: "execute_code", Description: "run code"}}
|
||||
|
||||
evs := collectEvents(context.Background(), c, "run something")
|
||||
|
||||
errEvs := byRole(evs, "error")
|
||||
if len(errEvs) == 0 {
|
||||
t.Fatal("expected an error event; got none")
|
||||
c.runtime.Tools = []backend.Tool{
|
||||
{Name: "call_tool", Description: "run tools"},
|
||||
{Name: "execute_code", Description: "run code"},
|
||||
}
|
||||
// The error must mention tool_unsupported behaviour (not a plain backend error).
|
||||
if got := errEvs[0]; got == "" {
|
||||
t.Error("error event content is empty")
|
||||
c.runtime.Exec = func(_ context.Context, name string, args json.RawMessage) (string, error) {
|
||||
execCalled = true
|
||||
if name != "call_tool" {
|
||||
t.Errorf("expected tool name 'call_tool', got %q", name)
|
||||
}
|
||||
return "file contents here", nil
|
||||
}
|
||||
|
||||
evs := collectEvents(context.Background(), c, "read a file")
|
||||
|
||||
if !execCalled {
|
||||
t.Fatal("expected Exec to be called for text-based tool call")
|
||||
}
|
||||
errEvs := byRole(evs, "error")
|
||||
if len(errEvs) != 0 {
|
||||
t.Errorf("unexpected error events: %v", errEvs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTextToolCall_RelaxedJSON verifies parsing of relaxed JSON (unquoted keys).
|
||||
func TestTextToolCall_RelaxedJSON(t *testing.T) {
|
||||
be := defaultBE()
|
||||
be.respond = func(_ context.Context, msgs []backend.Message, _ []backend.Tool, _ backend.GenerationParams) (<-chan backend.StreamEvent, error) {
|
||||
for _, m := range msgs {
|
||||
if m.Role == "tool" {
|
||||
return textStream("done"), nil
|
||||
}
|
||||
}
|
||||
return textStream(`execute_code: steps=[{code: "date"}]`), nil
|
||||
}
|
||||
|
||||
var execCalled bool
|
||||
c := newCore(t, be, nil)
|
||||
c.runtime.Tools = []backend.Tool{
|
||||
{Name: "execute_code", Description: "run code"},
|
||||
{Name: "call_tool", Description: "run tools"},
|
||||
}
|
||||
c.runtime.Exec = func(_ context.Context, name string, args json.RawMessage) (string, error) {
|
||||
execCalled = true
|
||||
if name != "execute_code" {
|
||||
t.Errorf("expected tool name 'execute_code', got %q", name)
|
||||
}
|
||||
return "Sat May 23 11:00:00 UTC 2026", nil
|
||||
}
|
||||
|
||||
collectEvents(context.Background(), c, "what time is it")
|
||||
|
||||
if !execCalled {
|
||||
t.Fatal("expected Exec to be called for relaxed-JSON text tool call")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTextToolCall_NoFalsePositive verifies that a normal text response that
|
||||
// does not contain a tool-call pattern is not mistakenly flagged.
|
||||
// does not contain a tool-call pattern is not mistakenly parsed.
|
||||
func TestTextToolCall_NoFalsePositive(t *testing.T) {
|
||||
be := defaultBE()
|
||||
be.respond = func(_ context.Context, _ []backend.Message, _ []backend.Tool, _ backend.GenerationParams) (<-chan backend.StreamEvent, error) {
|
||||
|
|
|
|||
Reference in New Issue