458 lines
14 KiB
Python
Executable File
458 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# args_json: {"type":"object","required":["action"],"properties":{"action":{"type":"string","description":"apps|tree|inspect|find|read|click|type|action"},"app":{"type":"string","description":"Application name"},"path":{"type":"string","description":"Element path"},"role":{"type":"string","description":"Element role (for find)"},"name":{"type":"string","description":"Element name (for find)"},"text":{"type":"string","description":"Text to type (for type action)"},"action_name":{"type":"string","description":"Action to perform (for action command)"},"depth":{"type":"integer","description":"Tree depth (for tree/inspect, default: 4/2)"}}}
|
|
# ollie:prompt
|
|
# ## gui_accessibility
|
|
#
|
|
# GUI automation via AT-SPI2 accessibility tree.
|
|
#
|
|
# **Args**: `[action, app, query...]`
|
|
# - Actions: `inspect`, `find`, `click`, `type`, `read`, `tree`, `action`
|
|
#
|
|
# ```
|
|
# gui_accessibility(action="find", app="Firefox", path="button", role="Close")
|
|
# gui_accessibility(action="tree", app="Kate")
|
|
# ```
|
|
# ollie:end
|
|
"""Interact with any GUI application through the accessibility (AT-SPI2) interface.
|
|
|
|
This uses the D-Bus-based accessibility tree to inspect, navigate, and interact
|
|
with UI elements in any application that supports AT-SPI (virtually all GTK, Qt,
|
|
and Electron apps).
|
|
|
|
Commands:
|
|
tree <app> - dump full widget tree
|
|
inspect <app> [depth] - brief overview of top-level widgets
|
|
find <app> <role> [name] - find elements by role and optional name
|
|
read <app> <path> - read text/value of element
|
|
click <app> <path> - click/activate element
|
|
type <app> <path> <text> - type text into element
|
|
action <app> <path> <action_name> - invoke named action on element
|
|
apps - list accessible applications
|
|
|
|
Paths are slash-separated indices (e.g., "0/2/1" = first child, third child, second child)
|
|
or you can use "role:name" patterns (e.g., "push button:OK").
|
|
|
|
Requires: python3 with gi (PyGObject) and Atspi typelib, OR the 'accerciser' package.
|
|
"""
|
|
import sys
|
|
import os
|
|
import time
|
|
|
|
|
|
def check_deps():
|
|
"""Check for AT-SPI2 dependencies."""
|
|
try:
|
|
import gi
|
|
gi.require_version('Atspi', '2.0')
|
|
from gi.repository import Atspi
|
|
return True
|
|
except (ImportError, ValueError):
|
|
return False
|
|
|
|
|
|
def get_atspi():
|
|
import gi
|
|
gi.require_version('Atspi', '2.0')
|
|
from gi.repository import Atspi
|
|
return Atspi
|
|
|
|
|
|
def list_apps():
|
|
"""List all accessible applications."""
|
|
Atspi = get_atspi()
|
|
desktop = Atspi.get_desktop(0)
|
|
count = desktop.get_child_count()
|
|
print(f"{'INDEX':<6} {'NAME':<30} {'CHILDREN'}")
|
|
for i in range(count):
|
|
app = desktop.get_child_at_index(i)
|
|
if app:
|
|
name = app.get_name() or "(unnamed)"
|
|
children = app.get_child_count()
|
|
print(f"{i:<6} {name:<30} {children}")
|
|
|
|
|
|
def find_app(name_substr):
|
|
"""Find application by name substring."""
|
|
Atspi = get_atspi()
|
|
desktop = Atspi.get_desktop(0)
|
|
count = desktop.get_child_count()
|
|
for i in range(count):
|
|
app = desktop.get_child_at_index(i)
|
|
if app:
|
|
app_name = app.get_name() or ""
|
|
if name_substr.lower() in app_name.lower():
|
|
return app
|
|
return None
|
|
|
|
|
|
def role_name(accessible):
|
|
"""Get human-readable role name."""
|
|
Atspi = get_atspi()
|
|
try:
|
|
role = accessible.get_role()
|
|
return accessible.get_role_name()
|
|
except Exception:
|
|
return "unknown"
|
|
|
|
|
|
def get_states(accessible):
|
|
"""Get state set as list of strings."""
|
|
Atspi = get_atspi()
|
|
try:
|
|
state_set = accessible.get_state_set()
|
|
states = []
|
|
for s in dir(Atspi.StateType):
|
|
if s.startswith('_'):
|
|
continue
|
|
try:
|
|
st = getattr(Atspi.StateType, s)
|
|
if state_set.contains(st):
|
|
states.append(s.lower())
|
|
except Exception:
|
|
pass
|
|
return states
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def resolve_path(root, path_str):
|
|
"""Resolve a path string to an accessible element.
|
|
|
|
Path can be:
|
|
- Numeric indices: "0/2/1"
|
|
- Role:name pattern: "push button:OK"
|
|
- Mixed: "0/frame:Main/push button:OK"
|
|
"""
|
|
if not path_str:
|
|
return root
|
|
|
|
parts = path_str.split('/')
|
|
current = root
|
|
|
|
for part in parts:
|
|
if ':' in part:
|
|
# Role:name pattern
|
|
target_role, target_name = part.split(':', 1)
|
|
found = False
|
|
count = current.get_child_count()
|
|
for i in range(count):
|
|
child = current.get_child_at_index(i)
|
|
if child:
|
|
r = role_name(child)
|
|
n = child.get_name() or ""
|
|
if (target_role.lower() in r.lower() and
|
|
target_name.lower() in n.lower()):
|
|
current = child
|
|
found = True
|
|
break
|
|
if not found:
|
|
print(f"ERROR: no child matching '{part}' under {role_name(current)}:{current.get_name()}",
|
|
file=sys.stderr)
|
|
sys.exit(1)
|
|
else:
|
|
# Numeric index
|
|
try:
|
|
idx = int(part)
|
|
child = current.get_child_at_index(idx)
|
|
if not child:
|
|
print(f"ERROR: no child at index {idx}", file=sys.stderr)
|
|
sys.exit(1)
|
|
current = child
|
|
except ValueError:
|
|
# Try as name match
|
|
found = False
|
|
count = current.get_child_count()
|
|
for i in range(count):
|
|
child = current.get_child_at_index(i)
|
|
if child and part.lower() in (child.get_name() or "").lower():
|
|
current = child
|
|
found = True
|
|
break
|
|
if not found:
|
|
print(f"ERROR: no child matching '{part}'", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
return current
|
|
|
|
|
|
def dump_tree(accessible, indent=0, max_depth=4, path=""):
|
|
"""Dump accessibility tree."""
|
|
if indent > max_depth:
|
|
return
|
|
|
|
name = accessible.get_name() or ""
|
|
role = role_name(accessible)
|
|
children = accessible.get_child_count()
|
|
|
|
# Get text content if available
|
|
text = ""
|
|
try:
|
|
ti = accessible.get_text()
|
|
if ti:
|
|
text_content = ti.get_text(0, min(ti.get_character_count(), 50))
|
|
if text_content and text_content != name:
|
|
text = f' text="{text_content}"'
|
|
except Exception:
|
|
pass
|
|
|
|
# Get value if available
|
|
value = ""
|
|
try:
|
|
vi = accessible.get_value()
|
|
if vi:
|
|
value = f' value={vi.get_current_value()}'
|
|
except Exception:
|
|
pass
|
|
|
|
prefix = " " * indent
|
|
path_display = path or "/"
|
|
|
|
# States of interest
|
|
states = get_states(accessible)
|
|
state_flags = ""
|
|
if "focused" in states:
|
|
state_flags += " [focused]"
|
|
if "checked" in states:
|
|
state_flags += " [checked]"
|
|
if "selected" in states:
|
|
state_flags += " [selected]"
|
|
if not ("visible" in states and "showing" in states):
|
|
state_flags += " [hidden]"
|
|
|
|
print(f"{prefix}[{path_display}] {role}: \"{name}\"{text}{value}{state_flags} ({children} children)")
|
|
|
|
for i in range(children):
|
|
child = accessible.get_child_at_index(i)
|
|
if child:
|
|
child_path = f"{path}/{i}" if path else str(i)
|
|
dump_tree(child, indent + 1, max_depth, child_path)
|
|
|
|
|
|
def find_elements(accessible, target_role, target_name="", results=None, path=""):
|
|
"""Find elements matching role and optional name."""
|
|
if results is None:
|
|
results = []
|
|
|
|
r = role_name(accessible)
|
|
n = accessible.get_name() or ""
|
|
|
|
role_match = target_role.lower() in r.lower() if target_role else True
|
|
name_match = target_name.lower() in n.lower() if target_name else True
|
|
|
|
if role_match and name_match:
|
|
results.append((path or "/", r, n, accessible))
|
|
|
|
count = accessible.get_child_count()
|
|
for i in range(min(count, 200)): # cap to avoid infinite trees
|
|
child = accessible.get_child_at_index(i)
|
|
if child:
|
|
child_path = f"{path}/{i}" if path else str(i)
|
|
find_elements(child, target_role, target_name, results, child_path)
|
|
|
|
return results
|
|
|
|
|
|
def read_element(accessible):
|
|
"""Read text/value from element."""
|
|
# Try text interface
|
|
try:
|
|
ti = accessible.get_text()
|
|
if ti:
|
|
count = ti.get_character_count()
|
|
if count > 0:
|
|
text = ti.get_text(0, count)
|
|
print(text)
|
|
return
|
|
except Exception:
|
|
pass
|
|
|
|
# Try value interface
|
|
try:
|
|
vi = accessible.get_value()
|
|
if vi:
|
|
print(vi.get_current_value())
|
|
return
|
|
except Exception:
|
|
pass
|
|
|
|
# Fall back to name
|
|
name = accessible.get_name()
|
|
if name:
|
|
print(name)
|
|
else:
|
|
print("(no readable content)")
|
|
|
|
|
|
def click_element(accessible):
|
|
"""Activate/click an element."""
|
|
Atspi = get_atspi()
|
|
|
|
# Try action interface
|
|
try:
|
|
ai = accessible.get_action()
|
|
if ai:
|
|
n_actions = ai.get_n_actions()
|
|
for i in range(n_actions):
|
|
action_name = ai.get_action_name(i)
|
|
if action_name in ('click', 'activate', 'press', 'invoke'):
|
|
ai.do_action(i)
|
|
print(f"performed: {action_name}")
|
|
return
|
|
# If no preferred action found, do the first one
|
|
if n_actions > 0:
|
|
action_name = ai.get_action_name(0)
|
|
ai.do_action(0)
|
|
print(f"performed: {action_name}")
|
|
return
|
|
except Exception as e:
|
|
pass
|
|
|
|
# Try generating a click event via component interface
|
|
try:
|
|
ci = accessible.get_component()
|
|
if ci:
|
|
pos = ci.get_position(Atspi.CoordType.SCREEN)
|
|
size = ci.get_size()
|
|
# Click center
|
|
x = pos.x + size.x // 2
|
|
y = pos.y + size.y // 2
|
|
Atspi.generate_mouse_event(x, y, 'b1c')
|
|
print(f"clicked at: {x},{y}")
|
|
return
|
|
except Exception:
|
|
pass
|
|
|
|
print("ERROR: no way to click this element", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
def type_text(accessible, text):
|
|
"""Type text into an element."""
|
|
Atspi = get_atspi()
|
|
|
|
# Try editable text interface
|
|
try:
|
|
ei = accessible.get_editable_text()
|
|
if ei:
|
|
# Clear existing and insert
|
|
ti = accessible.get_text()
|
|
if ti:
|
|
count = ti.get_character_count()
|
|
if count > 0:
|
|
ei.delete_text(0, count)
|
|
ei.insert_text(0, text, len(text))
|
|
print(f"typed: {len(text)} chars")
|
|
return
|
|
except Exception:
|
|
pass
|
|
|
|
# Fallback: focus and use keyboard events
|
|
try:
|
|
ci = accessible.get_component()
|
|
if ci:
|
|
ci.grab_focus()
|
|
time.sleep(0.1)
|
|
for ch in text:
|
|
Atspi.generate_keyboard_event(0, ch, Atspi.KeySynthType.STRING)
|
|
time.sleep(0.01)
|
|
print(f"typed via keyboard: {len(text)} chars")
|
|
return
|
|
except Exception:
|
|
pass
|
|
|
|
print("ERROR: cannot type into this element", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
def do_action(accessible, action_name):
|
|
"""Invoke a named action on element."""
|
|
try:
|
|
ai = accessible.get_action()
|
|
if ai:
|
|
n_actions = ai.get_n_actions()
|
|
for i in range(n_actions):
|
|
if ai.get_action_name(i) == action_name:
|
|
ai.do_action(i)
|
|
print(f"performed: {action_name}")
|
|
return
|
|
# List available actions
|
|
available = [ai.get_action_name(i) for i in range(n_actions)]
|
|
print(f"ERROR: action '{action_name}' not found. Available: {available}", file=sys.stderr)
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
print(f"ERROR: {e}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
def main():
|
|
if not check_deps():
|
|
print("ERROR: AT-SPI2 dependencies not available", file=sys.stderr)
|
|
print("Install: pip install PyGObject (or system package python3-gobject)", file=sys.stderr)
|
|
print("Also need: gir1.2-atspi-2.0 (Debian/Ubuntu) or gobject-introspection with Atspi typelib", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
from _lib.args import parse_args as _parse
|
|
_args = _parse()
|
|
cmd = _args.require("action")
|
|
|
|
if cmd == "apps":
|
|
list_apps()
|
|
return
|
|
|
|
app_name = _args.require("app")
|
|
app = find_app(app_name)
|
|
if not app:
|
|
print(f"ERROR: no accessible application matching '{app_name}'", file=sys.stderr)
|
|
print("Tip: run 'gui_accessibility apps' to see available applications", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
if cmd == "tree":
|
|
depth = _args.get_int('depth', 4)
|
|
dump_tree(app, max_depth=depth)
|
|
|
|
elif cmd == "inspect":
|
|
depth = _args.get_int('depth', 2)
|
|
dump_tree(app, max_depth=depth)
|
|
|
|
elif cmd == "find":
|
|
target_role = _args.require("role")
|
|
target_name = _args.get("name", "")
|
|
results = find_elements(app, target_role, target_name)
|
|
if not results:
|
|
print(f"no elements found matching role='{target_role}' name='{target_name}'")
|
|
else:
|
|
print(f"{'PATH':<20} {'ROLE':<20} {'NAME'}")
|
|
for path, role, name, _ in results[:50]:
|
|
print(f"{path:<20} {role:<20} {name}")
|
|
|
|
elif cmd == "read":
|
|
path = _args.require("path")
|
|
element = resolve_path(app, path)
|
|
read_element(element)
|
|
|
|
elif cmd == "click":
|
|
path = _args.require("path")
|
|
element = resolve_path(app, path)
|
|
click_element(element)
|
|
|
|
elif cmd == "type":
|
|
path = _args.require("path")
|
|
text = _args.require("text")
|
|
element = resolve_path(app, path)
|
|
type_text(element, text)
|
|
|
|
elif cmd == "action":
|
|
path = _args.require("path")
|
|
action_name = _args.require("action_name")
|
|
element = resolve_path(app, path)
|
|
do_action(element, action_name)
|
|
|
|
else:
|
|
print(f"unknown command: {cmd}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|