lkn
/
ollie
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve workflow-aware tool ranking

This commit is contained in:
Ollie Agent 2026-08-20 19:01:21 +02:00
parent d33549eca1
commit 3f8c8df38a
8 changed files with 245 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
cmd/olliesrv/internal/agent/skill_match.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"os"
"path/filepath"
"sort"
"strings"
"sync"
@ -95,10 +96,22 @@ func matchTools(input string) string {
return ""
}
results, err := toolIndex.Match(input, toolMatchThreshold, toolMatchLimit)
workflowText := normalizeWorkflow(input).SearchText()
workflow := normalizeWorkflow(input)
searchInput := input
if workflowText != "" {
searchInput += " Workflow: " + workflowText
}
results, err := toolIndex.Match(searchInput, toolMatchThreshold, toolMatchLimit)
if err != nil {
return ""
}
for i := range results {
results[i].Score += workflowToolAdjustment(results[i].Skill.Name, workflow)
}
sort.SliceStable(results, func(i, j int) bool {
return results[i].Score > results[j].Score
})
if len(results) == 0 {
return ""
}
@ -154,15 +167,24 @@ func newToolIndex() (*skills.Index, error) {
continue
}
var meta struct {
Description string `json:"description"`
Description string `json:"description"`
Prompt string `json:"prompt"`
Keywords []string `json:"keywords"`
}
if json.Unmarshal(data, &meta) != nil || meta.Description == "" {
continue
}
toolName := strings.TrimSuffix(entry.Name(), ".meta")
searchText := meta.Description
if meta.Prompt != "" {
searchText += " Usage: " + meta.Prompt
}
if len(meta.Keywords) > 0 {
searchText += " Keywords: " + strings.Join(meta.Keywords, ", ")
}
toolSkills = append(toolSkills, skills.Skill{
Name: toolName,
Description: meta.Description,
Description: searchText,
})
}

120
cmd/olliesrv/internal/agent/workflow.go Normal file
View File

@ -0,0 +1,120 @@
package agent
import "strings"
// Workflow is the compact task shape used to guide capability discovery.
type Workflow struct {
Domain string
Operation string
Target string
Scope string
Needs []string
}
// normalizeWorkflow extracts high-confidence workflow labels from a terse
// request. It preserves the original request for semantic matching and does
// not attempt to generate a plan.
func normalizeWorkflow(input string) Workflow {
text := strings.ToLower(input)
workflow := Workflow{
Domain: "unknown",
Operation: "unknown",
Target: "unknown",
Scope: "unknown",
}
if containsAny(text, "documentation", "document", "docs", "markdown", "readme") {
workflow.Domain = "documentation"
}
if containsAny(text, "architecture", "design") {
workflow.Scope = "architecture"
}
if containsAny(text, "source code", "implementation", "code") {
workflow.Domain = "source"
if workflow.Scope == "unknown" {
workflow.Scope = "code"
}
}
if containsAny(text, "config", "configuration") {
workflow.Domain = "configuration"
if workflow.Scope == "unknown" {
workflow.Scope = "config"
}
}
switch {
case containsAny(text, "create", "new file", "new document", "from scratch", "entire file", "full-file"):
workflow.Operation = "create"
workflow.Target = "new"
case containsAny(text, "update", "edit", "revise", "modify", "change", "add ", "document a"):
workflow.Operation = "modify"
}
if workflow.Target != "new" && containsAny(text, "existing", "relevant", "the docs", "the documentation", "architecture docs", "architecture documentation") {
workflow.Target = "existing"
}
if containsAny(text, "read", "inspect", "review", "find", "locate", "understand", "examine") {
workflow.Needs = append(workflow.Needs, "inspect")
}
if workflow.Operation == "modify" && workflow.Target == "existing" {
workflow.Needs = append(workflow.Needs, "inspect", "edit")
}
if containsAny(text, "test", "verify", "check", "build") {
workflow.Needs = append(workflow.Needs, "verify")
}
return workflow
}
// SearchText returns normalized labels for semantic retrieval.
func (w Workflow) SearchText() string {
var parts []string
for _, value := range []string{w.Domain, w.Operation, w.Target, w.Scope} {
if value != "" && value != "unknown" {
parts = append(parts, value)
}
}
parts = append(parts, w.Needs...)
return strings.Join(parts, " ")
}
func workflowToolAdjustment(tool string, workflow Workflow) float32 {
var adjustment float32
if workflow.Operation == "modify" && workflow.Target == "existing" {
switch tool {
case "file_edit":
adjustment += 0.20
case "file_write":
adjustment -= 0.20
}
}
if workflow.Operation == "create" && workflow.Target == "new" {
switch tool {
case "file_write":
adjustment += 0.20
case "file_edit":
adjustment -= 0.20
}
}
if containsNeed(workflow.Needs, "inspect") && tool == "file_read" {
adjustment += 0.10
}
return adjustment
}
func containsNeed(needs []string, want string) bool {
for _, need := range needs {
if need == want {
return true
}
}
return false
}
func containsAny(text string, terms ...string) bool {
for _, term := range terms {
if strings.Contains(text, term) {
return true
}
}
return false
}

60
cmd/olliesrv/internal/agent/workflow_test.go Normal file
View File

@ -0,0 +1,60 @@
package agent
import "testing"
func TestNormalizeWorkflow(t *testing.T) {
tests := []struct {
name string
input string
want Workflow
}{
{
name: "update architecture docs",
input: "update the architecture docs",
want: Workflow{Domain: "documentation", Operation: "modify", Target: "existing", Scope: "architecture", Needs: []string{"inspect", "edit"}},
},
{
name: "add feature to relevant architecture documentation",
input: "add feature x to the relevant architecture documentation",
want: Workflow{Domain: "documentation", Operation: "modify", Target: "existing", Scope: "architecture", Needs: []string{"inspect", "edit"}},
},
{
name: "create documentation",
input: "create a new architecture document",
want: Workflow{Domain: "documentation", Operation: "create", Target: "new", Scope: "architecture"},
},
{
name: "search text",
input: "add feature x to the relevant architecture documentation",
want: Workflow{Domain: "documentation", Operation: "modify", Target: "existing", Scope: "architecture", Needs: []string{"inspect", "edit"}},
},
{
name: "verify implementation",
input: "inspect the implementation and run tests",
want: Workflow{Domain: "source", Operation: "unknown", Target: "unknown", Scope: "code", Needs: []string{"inspect", "verify"}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := normalizeWorkflow(tt.input)
if got.Domain != tt.want.Domain || got.Operation != tt.want.Operation || got.Target != tt.want.Target || got.Scope != tt.want.Scope {
t.Fatalf("normalizeWorkflow(%q) = %#v, want %#v", tt.input, got, tt.want)
}
if len(got.Needs) != len(tt.want.Needs) {
t.Fatalf("normalizeWorkflow(%q) needs = %#v, want %#v", tt.input, got.Needs, tt.want.Needs)
}
for i := range got.Needs {
if got.Needs[i] != tt.want.Needs[i] {
t.Fatalf("normalizeWorkflow(%q) needs = %#v, want %#v", tt.input, got.Needs, tt.want.Needs)
}
}
if tt.name == "search text" {
wantSearch := "documentation modify existing architecture inspect edit"
if got.SearchText() != wantSearch {
t.Fatalf("SearchText() = %q, want %q", got.SearchText(), wantSearch)
}
}
})
}
}

3
data/tools/file_edit.meta
View File

@ -1,5 +1,6 @@
{
"description": "Replace text in a file. Has fuzzy matching (whitespace/indent flexible).",
"description": "Edit or update an existing file by replacing a specific text block. Use for targeted changes to existing Markdown, documentation, architecture docs, source code, tests, configuration, and notes. Typical requests include update the docs, revise the architecture documentation, add a feature or section to existing documentation, add content to the relevant docs, correct a paragraph, change wording, modify a configuration entry, or apply a focused fix. Preserves all unrelated content. Does not create a new file or intentionally rewrite the entire file; use file_write for those cases.",
"keywords": ["update existing documentation", "edit existing documentation", "revise architecture documentation", "update architecture docs", "change architecture docs", "add feature to existing documentation", "add feature to existing architecture docs", "add feature to relevant architecture documentation", "add section to existing documentation", "add content to relevant docs", "document a new feature in existing docs", "document an implementation change in existing docs", "modify an existing Markdown file", "replace a paragraph in documentation", "correct a documentation section", "preserve existing file content", "targeted documentation edit", "in-place documentation change", "focused Markdown update", "source code targeted edit", "configuration targeted edit"],
"outputFormat": "diff",
"prompt": "## file_edit\n\nReplace text in a file. Has fuzzy matching (whitespace/indent flexible).\n\n**Args**: `path`, `old_string`, `new_string` (required); `replace_all` (optional boolean)\n\n```\nfile_edit(path=\"<ABSOLUTE_PATH_FROM_PWD>\", old_string=\"old text\", new_string=\"new text\")\nfile_edit(path=\"<ABSOLUTE_PATH_FROM_PWD>\", old_string=\"old text\", new_string=\"new text\", replace_all=true)\n```\n\n**Constraints**: Absolute paths only. Errors if `old_string` matches multiple locations \u2014 add surrounding context to disambiguate, or use `replace_all=true`.",
"args": {

3
data/tools/file_read.meta
View File

@ -1,5 +1,6 @@
{
"description": "Read a file with line numbers. Prefer segments over full reads.",
"description": "Read an existing file or documentation file with numbered lines before editing. Use to inspect source code, Markdown, architecture docs, configuration, notes, and existing content before an update. Typical requests include read the docs, inspect the architecture documentation, review a file, find the relevant section, or examine code before editing. Prefer focused line ranges for large files.",
"keywords": ["read an existing file", "read existing documentation", "read the architecture docs", "inspect architecture documentation", "review existing documentation", "find the relevant documentation", "find the relevant section", "locate existing content", "read before editing", "inspect before updating", "review before changing", "understand existing implementation", "examine source before editing", "read Markdown before updating", "inspect configuration before changing"],
"prompt": "## file_read\n\nRead a file with line numbers. Prefer segments over full reads.\n\n**Args**: `path` (required), optional `start` and `end` line numbers\n\n- `start`/`end`: 1-indexed line numbers, inclusive. Use integers. Max segment: 2000 lines.\n- No `start`/`end`: reads up to 500 lines from line 1.\n\n```\nfile_read(path=\"<ABSOLUTE_PATH_FROM_PWD>\")\nfile_read(path=\"<ABSOLUTE_PATH_FROM_PWD>\", start=100, end=200)\n```\n\n- Use `start`/`end` as integers even though they represent line numbers.\n\n**Constraints**: Absolute paths only.",
"args": {
"type": "object",

3
data/tools/file_write.meta
View File

@ -1,5 +1,6 @@
{
"description": "Create or overwrite a file. Use for new files or intentional full-file replacement; use file_edit for targeted changes.",
"description": "Create a new file or intentionally replace an entire existing file. Use when writing a new documentation page, creating a new Markdown note, generating a file from scratch, adding a new architecture document, or performing a deliberate full-file rewrite. Do not use for targeted updates, edits, revisions, or additions to existing documentation; use file_edit instead.",
"keywords": ["create a new file", "create a new document", "create new documentation", "write a new documentation page", "write documentation from scratch", "create a new Markdown file", "create a new architecture document", "generate a new document", "start a new file", "replace an entire existing file", "rewrite an entire file", "overwrite the complete file", "full-file replacement"],
"outputFormat": "diff",
"prompt": "## file_write\n\nCreate or overwrite a file. Use for new files or intentional full-file replacement. Use `file_edit` for targeted changes.\n\n**Args**: `path`, `content` (required)\n\n```\nfile_write(path=\"<ABSOLUTE_PATH_FROM_PWD>\", content=\"file content here\")\n```\n\n**Constraints**: Absolute paths only. Parent directory must exist.",
"args": {

7
doc/architecture-tools.md
View File

@ -60,6 +60,7 @@ The `.meta` file is the sole source of discovery and schema. The registry never
|---|---|---|
| `description` | string | One-liner shown in the tool listing. |
| `prompt` | string | Documentation injected when the tool is loaded. |
| `keywords` | array of strings | Additional user-language phrases used only for semantic discovery. Include synonyms, task verbs, file types, domains, and representative requests. |
| `args` | JSON Schema | Input schema exposed to the model. |
| `tier` | `hot`, `warm`, or `cold` | Context retention tier. |
| `cmd` | string | Shell command or executable path/name. |
@ -72,7 +73,11 @@ The `.meta` file is the sole source of discovery and schema. The registry never
Tool discovery remains metadata-driven: `toolsrv` scans `.meta` files and manages
which tools are loaded for an agent. Ollie adds a separate semantic ranking
layer before prompt assembly. See [`architecture-embedding.md`](architecture-embedding.md)
layer before prompt assembly. The ranking text combines `description`, `prompt`,
and optional `keywords`. Keywords are discovery-only vocabulary: they should
contain the words users use to request the operation, including synonyms,
workflow phrases, file types, and domain terms. They are not shown as a second
model-facing schema and do not change execution. See [`architecture-embedding.md`](architecture-embedding.md)
for the embedding model, index lifecycle, matching thresholds, and failure
behavior.

53
toolsrv/metadata/meta.go
View File

@ -12,29 +12,32 @@ import (
// MetaFile is the JSON structure of a .meta sidecar file.
type MetaFile struct {
Description string `json:"description"`
Prompt string `json:"prompt,omitempty"`
Args json.RawMessage `json:"args,omitempty"`
Tier string `json:"tier,omitempty"`
Scope string `json:"scope,omitempty"`
OutputFormat string `json:"outputFormat,omitempty"`
Timeout *int `json:"timeout,omitempty"`
Cmd string `json:"cmd,omitempty"`
Variants []Variant `json:"variants,omitempty"`
Description string `json:"description"`
Prompt string `json:"prompt,omitempty"`
// Keywords expands semantic discovery vocabulary without changing the
// model-facing tool description or protocol schema.
Keywords []string `json:"keywords,omitempty"`
Args json.RawMessage `json:"args,omitempty"`
Tier string `json:"tier,omitempty"`
Scope string `json:"scope,omitempty"`
OutputFormat string `json:"outputFormat,omitempty"`
Timeout *int `json:"timeout,omitempty"`
Cmd string `json:"cmd,omitempty"`
Variants []Variant `json:"variants,omitempty"`
}
// Variant is a conditional tool definition. The first variant whose
// match conditions are all satisfied determines the tool's schema,
// documentation, and executable.
type Variant struct {
Match map[string]any `json:"match"`
Description string `json:"description,omitempty"`
Prompt string `json:"prompt,omitempty"`
Args json.RawMessage `json:"args,omitempty"`
Tier string `json:"tier,omitempty"`
Scope string `json:"scope,omitempty"`
OutputFormat string `json:"outputFormat,omitempty"`
Cmd string `json:"cmd,omitempty"`
Match map[string]any `json:"match"`
Description string `json:"description,omitempty"`
Prompt string `json:"prompt,omitempty"`
Args json.RawMessage `json:"args,omitempty"`
Tier string `json:"tier,omitempty"`
Scope string `json:"scope,omitempty"`
OutputFormat string `json:"outputFormat,omitempty"`
Cmd string `json:"cmd,omitempty"`
}
// LoadMetaFile reads and parses a .meta sidecar file for the given tool name.
@ -181,14 +184,14 @@ func ToolInfoFromMeta(name string, m *MetaFile) protocol.ToolInfo {
tier = "hot"
}
return protocol.ToolInfo{
Name: name,
Description: resolved.Description,
InputSchema: args,
Prompt: resolved.Prompt,
Tier: tier,
Scope: resolved.Scope,
OutputFormat: resolved.OutputFormat,
Timeout: resolveTimeout(m.Timeout),
Name: name,
Description: resolved.Description,
InputSchema: args,
Prompt: resolved.Prompt,
Tier: tier,
Scope: resolved.Scope,
OutputFormat: resolved.OutputFormat,
Timeout: resolveTimeout(m.Timeout),
}
}