ollie/tools/gui_accessibility

#!/usr/bin/env python3
# description: GUI automation via AT-SPI2 accessibility tree. Args: <inspect|find|click|type|read|tree|action> [app] [query...]
"""Interact with any GUI application through the accessibility (AT-SPI2) interface.

This uses the D-Bus-based accessibility tree to inspect, navigate, and interact
with UI elements in any application that supports AT-SPI (virtually all GTK, Qt,
and Electron apps).

Commands:
  tree <app>                         - dump full widget tree
  inspect <app> [depth]              - brief overview of top-level widgets
  find <app> <role> [name]           - find elements by role and optional name
  read <app> <path>                  - read text/value of element
  click <app> <path>                 - click/activate element
  type <app> <path> <text>           - type text into element
  action <app> <path> <action_name>  - invoke named action on element
  apps                               - list accessible applications

Paths are slash-separated indices (e.g., "0/2/1" = first child, third child, second child)
or you can use "role:name" patterns (e.g., "push button:OK").

Requires: python3 with gi (PyGObject) and Atspi typelib, OR the 'accerciser' package.
"""
import sys
import os
import time


def check_deps():
    """Check for AT-SPI2 dependencies."""
    try:
        import gi
        gi.require_version('Atspi', '2.0')
        from gi.repository import Atspi
        return True
    except (ImportError, ValueError):
        return False


def get_atspi():
    import gi
    gi.require_version('Atspi', '2.0')
    from gi.repository import Atspi
    return Atspi


def list_apps():
    """List all accessible applications."""
    Atspi = get_atspi()
    desktop = Atspi.get_desktop(0)
    count = desktop.get_child_count()
    print(f"{'INDEX':<6} {'NAME':<30} {'CHILDREN'}")
    for i in range(count):
        app = desktop.get_child_at_index(i)
        if app:
            name = app.get_name() or "(unnamed)"
            children = app.get_child_count()
            print(f"{i:<6} {name:<30} {children}")


def find_app(name_substr):
    """Find application by name substring."""
    Atspi = get_atspi()
    desktop = Atspi.get_desktop(0)
    count = desktop.get_child_count()
    for i in range(count):
        app = desktop.get_child_at_index(i)
        if app:
            app_name = app.get_name() or ""
            if name_substr.lower() in app_name.lower():
                return app
    return None


def role_name(accessible):
    """Get human-readable role name."""
    Atspi = get_atspi()
    try:
        role = accessible.get_role()
        return accessible.get_role_name()
    except Exception:
        return "unknown"


def get_states(accessible):
    """Get state set as list of strings."""
    Atspi = get_atspi()
    try:
        state_set = accessible.get_state_set()
        states = []
        for s in dir(Atspi.StateType):
            if s.startswith('_'):
                continue
            try:
                st = getattr(Atspi.StateType, s)
                if state_set.contains(st):
                    states.append(s.lower())
            except Exception:
                pass
        return states
    except Exception:
        return []


def resolve_path(root, path_str):
    """Resolve a path string to an accessible element.
    
    Path can be:
    - Numeric indices: "0/2/1"
    - Role:name pattern: "push button:OK"
    - Mixed: "0/frame:Main/push button:OK"
    """
    if not path_str:
        return root
    
    parts = path_str.split('/')
    current = root
    
    for part in parts:
        if ':' in part:
            # Role:name pattern
            target_role, target_name = part.split(':', 1)
            found = False
            count = current.get_child_count()
            for i in range(count):
                child = current.get_child_at_index(i)
                if child:
                    r = role_name(child)
                    n = child.get_name() or ""
                    if (target_role.lower() in r.lower() and
                        target_name.lower() in n.lower()):
                        current = child
                        found = True
                        break
            if not found:
                print(f"ERROR: no child matching '{part}' under {role_name(current)}:{current.get_name()}",
                      file=sys.stderr)
                sys.exit(1)
        else:
            # Numeric index
            try:
                idx = int(part)
                child = current.get_child_at_index(idx)
                if not child:
                    print(f"ERROR: no child at index {idx}", file=sys.stderr)
                    sys.exit(1)
                current = child
            except ValueError:
                # Try as name match
                found = False
                count = current.get_child_count()
                for i in range(count):
                    child = current.get_child_at_index(i)
                    if child and part.lower() in (child.get_name() or "").lower():
                        current = child
                        found = True
                        break
                if not found:
                    print(f"ERROR: no child matching '{part}'", file=sys.stderr)
                    sys.exit(1)
    
    return current


def dump_tree(accessible, indent=0, max_depth=4, path=""):
    """Dump accessibility tree."""
    if indent > max_depth:
        return
    
    name = accessible.get_name() or ""
    role = role_name(accessible)
    children = accessible.get_child_count()
    
    # Get text content if available
    text = ""
    try:
        ti = accessible.get_text()
        if ti:
            text_content = ti.get_text(0, min(ti.get_character_count(), 50))
            if text_content and text_content != name:
                text = f' text="{text_content}"'
    except Exception:
        pass
    
    # Get value if available
    value = ""
    try:
        vi = accessible.get_value()
        if vi:
            value = f' value={vi.get_current_value()}'
    except Exception:
        pass
    
    prefix = "  " * indent
    path_display = path or "/"
    
    # States of interest
    states = get_states(accessible)
    state_flags = ""
    if "focused" in states:
        state_flags += " [focused]"
    if "checked" in states:
        state_flags += " [checked]"
    if "selected" in states:
        state_flags += " [selected]"
    if not ("visible" in states and "showing" in states):
        state_flags += " [hidden]"
    
    print(f"{prefix}[{path_display}] {role}: \"{name}\"{text}{value}{state_flags} ({children} children)")
    
    for i in range(children):
        child = accessible.get_child_at_index(i)
        if child:
            child_path = f"{path}/{i}" if path else str(i)
            dump_tree(child, indent + 1, max_depth, child_path)


def find_elements(accessible, target_role, target_name="", results=None, path=""):
    """Find elements matching role and optional name."""
    if results is None:
        results = []
    
    r = role_name(accessible)
    n = accessible.get_name() or ""
    
    role_match = target_role.lower() in r.lower() if target_role else True
    name_match = target_name.lower() in n.lower() if target_name else True
    
    if role_match and name_match:
        results.append((path or "/", r, n, accessible))
    
    count = accessible.get_child_count()
    for i in range(min(count, 200)):  # cap to avoid infinite trees
        child = accessible.get_child_at_index(i)
        if child:
            child_path = f"{path}/{i}" if path else str(i)
            find_elements(child, target_role, target_name, results, child_path)
    
    return results


def read_element(accessible):
    """Read text/value from element."""
    # Try text interface
    try:
        ti = accessible.get_text()
        if ti:
            count = ti.get_character_count()
            if count > 0:
                text = ti.get_text(0, count)
                print(text)
                return
    except Exception:
        pass
    
    # Try value interface
    try:
        vi = accessible.get_value()
        if vi:
            print(vi.get_current_value())
            return
    except Exception:
        pass
    
    # Fall back to name
    name = accessible.get_name()
    if name:
        print(name)
    else:
        print("(no readable content)")


def click_element(accessible):
    """Activate/click an element."""
    Atspi = get_atspi()
    
    # Try action interface
    try:
        ai = accessible.get_action()
        if ai:
            n_actions = ai.get_n_actions()
            for i in range(n_actions):
                action_name = ai.get_action_name(i)
                if action_name in ('click', 'activate', 'press', 'invoke'):
                    ai.do_action(i)
                    print(f"performed: {action_name}")
                    return
            # If no preferred action found, do the first one
            if n_actions > 0:
                action_name = ai.get_action_name(0)
                ai.do_action(0)
                print(f"performed: {action_name}")
                return
    except Exception as e:
        pass
    
    # Try generating a click event via component interface
    try:
        ci = accessible.get_component()
        if ci:
            pos = ci.get_position(Atspi.CoordType.SCREEN)
            size = ci.get_size()
            # Click center
            x = pos.x + size.x // 2
            y = pos.y + size.y // 2
            Atspi.generate_mouse_event(x, y, 'b1c')
            print(f"clicked at: {x},{y}")
            return
    except Exception:
        pass
    
    print("ERROR: no way to click this element", file=sys.stderr)
    sys.exit(1)


def type_text(accessible, text):
    """Type text into an element."""
    Atspi = get_atspi()
    
    # Try editable text interface
    try:
        ei = accessible.get_editable_text()
        if ei:
            # Clear existing and insert
            ti = accessible.get_text()
            if ti:
                count = ti.get_character_count()
                if count > 0:
                    ei.delete_text(0, count)
            ei.insert_text(0, text, len(text))
            print(f"typed: {len(text)} chars")
            return
    except Exception:
        pass
    
    # Fallback: focus and use keyboard events
    try:
        ci = accessible.get_component()
        if ci:
            ci.grab_focus()
            time.sleep(0.1)
            for ch in text:
                Atspi.generate_keyboard_event(0, ch, Atspi.KeySynthType.STRING)
                time.sleep(0.01)
            print(f"typed via keyboard: {len(text)} chars")
            return
    except Exception:
        pass
    
    print("ERROR: cannot type into this element", file=sys.stderr)
    sys.exit(1)


def do_action(accessible, action_name):
    """Invoke a named action on element."""
    try:
        ai = accessible.get_action()
        if ai:
            n_actions = ai.get_n_actions()
            for i in range(n_actions):
                if ai.get_action_name(i) == action_name:
                    ai.do_action(i)
                    print(f"performed: {action_name}")
                    return
            # List available actions
            available = [ai.get_action_name(i) for i in range(n_actions)]
            print(f"ERROR: action '{action_name}' not found. Available: {available}", file=sys.stderr)
            sys.exit(1)
    except Exception as e:
        print(f"ERROR: {e}", file=sys.stderr)
        sys.exit(1)


def main():
    if not check_deps():
        print("ERROR: AT-SPI2 dependencies not available", file=sys.stderr)
        print("Install: pip install PyGObject (or system package python3-gobject)", file=sys.stderr)
        print("Also need: gir1.2-atspi-2.0 (Debian/Ubuntu) or gobject-introspection with Atspi typelib", file=sys.stderr)
        sys.exit(1)
    
    if len(sys.argv) < 2:
        print("usage: gui_accessibility <apps|tree|inspect|find|read|click|type|action> [app] [args...]", file=sys.stderr)
        sys.exit(1)
    
    cmd = sys.argv[1]
    
    if cmd == "apps":
        list_apps()
        return
    
    if len(sys.argv) < 3:
        print(f"usage: gui_accessibility {cmd} <app_name> [args...]", file=sys.stderr)
        sys.exit(1)
    
    app_name = sys.argv[2]
    app = find_app(app_name)
    if not app:
        print(f"ERROR: no accessible application matching '{app_name}'", file=sys.stderr)
        print("Tip: run 'gui_accessibility apps' to see available applications", file=sys.stderr)
        sys.exit(1)
    
    if cmd == "tree":
        depth = int(sys.argv[3]) if len(sys.argv) > 3 else 4
        dump_tree(app, max_depth=depth)
    
    elif cmd == "inspect":
        depth = int(sys.argv[3]) if len(sys.argv) > 3 else 2
        dump_tree(app, max_depth=depth)
    
    elif cmd == "find":
        if len(sys.argv) < 4:
            print("usage: gui_accessibility find <app> <role> [name]", file=sys.stderr)
            sys.exit(1)
        target_role = sys.argv[3]
        target_name = sys.argv[4] if len(sys.argv) > 4 else ""
        results = find_elements(app, target_role, target_name)
        if not results:
            print(f"no elements found matching role='{target_role}' name='{target_name}'")
        else:
            print(f"{'PATH':<20} {'ROLE':<20} {'NAME'}")
            for path, role, name, _ in results[:50]:
                print(f"{path:<20} {role:<20} {name}")
    
    elif cmd == "read":
        if len(sys.argv) < 4:
            print("usage: gui_accessibility read <app> <path>", file=sys.stderr)
            sys.exit(1)
        path = sys.argv[3]
        element = resolve_path(app, path)
        read_element(element)
    
    elif cmd == "click":
        if len(sys.argv) < 4:
            print("usage: gui_accessibility click <app> <path>", file=sys.stderr)
            sys.exit(1)
        path = sys.argv[3]
        element = resolve_path(app, path)
        click_element(element)
    
    elif cmd == "type":
        if len(sys.argv) < 5:
            print("usage: gui_accessibility type <app> <path> <text>", file=sys.stderr)
            sys.exit(1)
        path = sys.argv[3]
        text = sys.argv[4]
        element = resolve_path(app, path)
        type_text(element, text)
    
    elif cmd == "action":
        if len(sys.argv) < 5:
            print("usage: gui_accessibility action <app> <path> <action_name>", file=sys.stderr)
            sys.exit(1)
        path = sys.argv[3]
        action_name = sys.argv[4]
        element = resolve_path(app, path)
        do_action(element, action_name)
    
    else:
        print(f"unknown command: {cmd}", file=sys.stderr)
        sys.exit(1)


if __name__ == "__main__":
    main()