diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 3469d00..9ebe9be 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -179,6 +179,17 @@ func run(ctx context.Context, cfg agentConfig, state state) error { return fmt.Errorf("step %d: %s", step, stopReason) } + // Detect models that emit tool calls as plain text instead of using the + // function-calling API. If the response contains no API-level tool calls + // but the text matches a registered tool name followed by ':' or '(', + // the model does not support function calling — treat as ToolUnsupportedError + // so freeloader can blacklist and cycle to the next model. + if len(toolCalls) == 0 && len(cfg.Tools) > 0 && hasTextToolCall(content.String(), cfg.Tools) { + return fmt.Errorf("step %d: %w", step, &backend.ToolUnsupportedError{ + Message: "model emitted tool call as text (function calling API not supported)", + }) + } + // Execute tool calls, running consecutive parallel-read-safe tools concurrently. msg := backend.Message{Role: "assistant", Content: content.String(), ToolCalls: toolCalls} results := make([]toolResult, 0, len(toolCalls)) @@ -369,6 +380,18 @@ func emit(cfg agentConfig, msg Event) { } } +// hasTextToolCall returns true when the assistant text contains a registered +// tool name immediately followed by ':' or '(', indicating the model tried to +// invoke a tool by writing the call as plain text rather than via the API. +func hasTextToolCall(text string, tools []backend.Tool) bool { + for _, t := range tools { + if strings.Contains(text, t.Name+":") || strings.Contains(text, t.Name+"(") { + return true + } + } + return false +} + // classifyError returns a short string identifying the error type for the // turnError hook payload. func classifyError(err error) string { diff --git a/pkg/agent/loop_texttool_test.go b/pkg/agent/loop_texttool_test.go new file mode 100644 index 0000000..9c9dda5 --- /dev/null +++ b/pkg/agent/loop_texttool_test.go @@ -0,0 +1,76 @@ +package agent + +import ( + "context" + "testing" + + "ollie/pkg/backend" +) + +// TestTextToolCall_DetectedAsUnsupported verifies that when a model emits a tool +// call as plain text (no API-level tool calls) the loop returns a +// ToolUnsupportedError so freeloader can blacklist and cycle. +func TestTextToolCall_DetectedAsUnsupported(t *testing.T) { + be := defaultBE() + // Model responds with text that mimics a tool invocation instead of using + // the function-calling API. + be.respond = func(_ context.Context, _ []backend.Message, _ []backend.Tool, _ backend.GenerationParams) (<-chan backend.StreamEvent, error) { + return textStream(`execute_code: steps=[{code: "print('hi')"}]`), nil + } + + c := newCore(t, be, nil) + c.cfg.Tools = []backend.Tool{{Name: "execute_code", Description: "run code"}} + + evs := collectEvents(context.Background(), c, "run something") + + errEvs := byRole(evs, "error") + if len(errEvs) == 0 { + t.Fatal("expected an error event; got none") + } + // The error must mention tool_unsupported behaviour (not a plain backend error). + if got := errEvs[0]; got == "" { + t.Error("error event content is empty") + } +} + +// TestTextToolCall_NoFalsePositive verifies that a normal text response that +// does not contain a tool-call pattern is not mistakenly flagged. +func TestTextToolCall_NoFalsePositive(t *testing.T) { + be := defaultBE() + be.respond = func(_ context.Context, _ []backend.Message, _ []backend.Tool, _ backend.GenerationParams) (<-chan backend.StreamEvent, error) { + return textStream("Here is how execute_code works in general."), nil + } + + c := newCore(t, be, nil) + c.cfg.Tools = []backend.Tool{{Name: "execute_code", Description: "run code"}} + + evs := collectEvents(context.Background(), c, "explain tools") + + errEvs := byRole(evs, "error") + if len(errEvs) != 0 { + t.Errorf("unexpected error events: %v", errEvs) + } + assistEvs := byRole(evs, "assistant") + if len(assistEvs) == 0 { + t.Error("expected assistant response; got none") + } +} + +// TestTextToolCall_NoToolsConfigured verifies that when no tools are configured +// the detection does not fire (nothing to detect against). +func TestTextToolCall_NoToolsConfigured(t *testing.T) { + be := defaultBE() + be.respond = func(_ context.Context, _ []backend.Message, _ []backend.Tool, _ backend.GenerationParams) (<-chan backend.StreamEvent, error) { + return textStream(`execute_code: steps=[{code: "print('hi')"}]`), nil + } + + c := newCore(t, be, nil) + // No tools configured. + + evs := collectEvents(context.Background(), c, "run something") + + errEvs := byRole(evs, "error") + if len(errEvs) != 0 { + t.Errorf("unexpected error with no tools configured: %v", errEvs) + } +}