#!/usr/bin/env python3 # description: GUI automation via AT-SPI2 accessibility tree. Args: [app] [query...] """Interact with any GUI application through the accessibility (AT-SPI2) interface. This uses the D-Bus-based accessibility tree to inspect, navigate, and interact with UI elements in any application that supports AT-SPI (virtually all GTK, Qt, and Electron apps). Commands: tree - dump full widget tree inspect [depth] - brief overview of top-level widgets find [name] - find elements by role and optional name read - read text/value of element click - click/activate element type - type text into element action - invoke named action on element apps - list accessible applications Paths are slash-separated indices (e.g., "0/2/1" = first child, third child, second child) or you can use "role:name" patterns (e.g., "push button:OK"). Requires: python3 with gi (PyGObject) and Atspi typelib, OR the 'accerciser' package. """ import sys import os import time def check_deps(): """Check for AT-SPI2 dependencies.""" try: import gi gi.require_version('Atspi', '2.0') from gi.repository import Atspi return True except (ImportError, ValueError): return False def get_atspi(): import gi gi.require_version('Atspi', '2.0') from gi.repository import Atspi return Atspi def list_apps(): """List all accessible applications.""" Atspi = get_atspi() desktop = Atspi.get_desktop(0) count = desktop.get_child_count() print(f"{'INDEX':<6} {'NAME':<30} {'CHILDREN'}") for i in range(count): app = desktop.get_child_at_index(i) if app: name = app.get_name() or "(unnamed)" children = app.get_child_count() print(f"{i:<6} {name:<30} {children}") def find_app(name_substr): """Find application by name substring.""" Atspi = get_atspi() desktop = Atspi.get_desktop(0) count = desktop.get_child_count() for i in range(count): app = desktop.get_child_at_index(i) if app: app_name = app.get_name() or "" if name_substr.lower() in app_name.lower(): return app return None def role_name(accessible): """Get human-readable role name.""" Atspi = get_atspi() try: role = accessible.get_role() return accessible.get_role_name() except Exception: return "unknown" def get_states(accessible): """Get state set as list of strings.""" Atspi = get_atspi() try: state_set = accessible.get_state_set() states = [] for s in dir(Atspi.StateType): if s.startswith('_'): continue try: st = getattr(Atspi.StateType, s) if state_set.contains(st): states.append(s.lower()) except Exception: pass return states except Exception: return [] def resolve_path(root, path_str): """Resolve a path string to an accessible element. Path can be: - Numeric indices: "0/2/1" - Role:name pattern: "push button:OK" - Mixed: "0/frame:Main/push button:OK" """ if not path_str: return root parts = path_str.split('/') current = root for part in parts: if ':' in part: # Role:name pattern target_role, target_name = part.split(':', 1) found = False count = current.get_child_count() for i in range(count): child = current.get_child_at_index(i) if child: r = role_name(child) n = child.get_name() or "" if (target_role.lower() in r.lower() and target_name.lower() in n.lower()): current = child found = True break if not found: print(f"ERROR: no child matching '{part}' under {role_name(current)}:{current.get_name()}", file=sys.stderr) sys.exit(1) else: # Numeric index try: idx = int(part) child = current.get_child_at_index(idx) if not child: print(f"ERROR: no child at index {idx}", file=sys.stderr) sys.exit(1) current = child except ValueError: # Try as name match found = False count = current.get_child_count() for i in range(count): child = current.get_child_at_index(i) if child and part.lower() in (child.get_name() or "").lower(): current = child found = True break if not found: print(f"ERROR: no child matching '{part}'", file=sys.stderr) sys.exit(1) return current def dump_tree(accessible, indent=0, max_depth=4, path=""): """Dump accessibility tree.""" if indent > max_depth: return name = accessible.get_name() or "" role = role_name(accessible) children = accessible.get_child_count() # Get text content if available text = "" try: ti = accessible.get_text() if ti: text_content = ti.get_text(0, min(ti.get_character_count(), 50)) if text_content and text_content != name: text = f' text="{text_content}"' except Exception: pass # Get value if available value = "" try: vi = accessible.get_value() if vi: value = f' value={vi.get_current_value()}' except Exception: pass prefix = " " * indent path_display = path or "/" # States of interest states = get_states(accessible) state_flags = "" if "focused" in states: state_flags += " [focused]" if "checked" in states: state_flags += " [checked]" if "selected" in states: state_flags += " [selected]" if not ("visible" in states and "showing" in states): state_flags += " [hidden]" print(f"{prefix}[{path_display}] {role}: \"{name}\"{text}{value}{state_flags} ({children} children)") for i in range(children): child = accessible.get_child_at_index(i) if child: child_path = f"{path}/{i}" if path else str(i) dump_tree(child, indent + 1, max_depth, child_path) def find_elements(accessible, target_role, target_name="", results=None, path=""): """Find elements matching role and optional name.""" if results is None: results = [] r = role_name(accessible) n = accessible.get_name() or "" role_match = target_role.lower() in r.lower() if target_role else True name_match = target_name.lower() in n.lower() if target_name else True if role_match and name_match: results.append((path or "/", r, n, accessible)) count = accessible.get_child_count() for i in range(min(count, 200)): # cap to avoid infinite trees child = accessible.get_child_at_index(i) if child: child_path = f"{path}/{i}" if path else str(i) find_elements(child, target_role, target_name, results, child_path) return results def read_element(accessible): """Read text/value from element.""" # Try text interface try: ti = accessible.get_text() if ti: count = ti.get_character_count() if count > 0: text = ti.get_text(0, count) print(text) return except Exception: pass # Try value interface try: vi = accessible.get_value() if vi: print(vi.get_current_value()) return except Exception: pass # Fall back to name name = accessible.get_name() if name: print(name) else: print("(no readable content)") def click_element(accessible): """Activate/click an element.""" Atspi = get_atspi() # Try action interface try: ai = accessible.get_action() if ai: n_actions = ai.get_n_actions() for i in range(n_actions): action_name = ai.get_action_name(i) if action_name in ('click', 'activate', 'press', 'invoke'): ai.do_action(i) print(f"performed: {action_name}") return # If no preferred action found, do the first one if n_actions > 0: action_name = ai.get_action_name(0) ai.do_action(0) print(f"performed: {action_name}") return except Exception as e: pass # Try generating a click event via component interface try: ci = accessible.get_component() if ci: pos = ci.get_position(Atspi.CoordType.SCREEN) size = ci.get_size() # Click center x = pos.x + size.x // 2 y = pos.y + size.y // 2 Atspi.generate_mouse_event(x, y, 'b1c') print(f"clicked at: {x},{y}") return except Exception: pass print("ERROR: no way to click this element", file=sys.stderr) sys.exit(1) def type_text(accessible, text): """Type text into an element.""" Atspi = get_atspi() # Try editable text interface try: ei = accessible.get_editable_text() if ei: # Clear existing and insert ti = accessible.get_text() if ti: count = ti.get_character_count() if count > 0: ei.delete_text(0, count) ei.insert_text(0, text, len(text)) print(f"typed: {len(text)} chars") return except Exception: pass # Fallback: focus and use keyboard events try: ci = accessible.get_component() if ci: ci.grab_focus() time.sleep(0.1) for ch in text: Atspi.generate_keyboard_event(0, ch, Atspi.KeySynthType.STRING) time.sleep(0.01) print(f"typed via keyboard: {len(text)} chars") return except Exception: pass print("ERROR: cannot type into this element", file=sys.stderr) sys.exit(1) def do_action(accessible, action_name): """Invoke a named action on element.""" try: ai = accessible.get_action() if ai: n_actions = ai.get_n_actions() for i in range(n_actions): if ai.get_action_name(i) == action_name: ai.do_action(i) print(f"performed: {action_name}") return # List available actions available = [ai.get_action_name(i) for i in range(n_actions)] print(f"ERROR: action '{action_name}' not found. Available: {available}", file=sys.stderr) sys.exit(1) except Exception as e: print(f"ERROR: {e}", file=sys.stderr) sys.exit(1) def main(): if not check_deps(): print("ERROR: AT-SPI2 dependencies not available", file=sys.stderr) print("Install: pip install PyGObject (or system package python3-gobject)", file=sys.stderr) print("Also need: gir1.2-atspi-2.0 (Debian/Ubuntu) or gobject-introspection with Atspi typelib", file=sys.stderr) sys.exit(1) if len(sys.argv) < 2: print("usage: gui_accessibility [app] [args...]", file=sys.stderr) sys.exit(1) cmd = sys.argv[1] if cmd == "apps": list_apps() return if len(sys.argv) < 3: print(f"usage: gui_accessibility {cmd} [args...]", file=sys.stderr) sys.exit(1) app_name = sys.argv[2] app = find_app(app_name) if not app: print(f"ERROR: no accessible application matching '{app_name}'", file=sys.stderr) print("Tip: run 'gui_accessibility apps' to see available applications", file=sys.stderr) sys.exit(1) if cmd == "tree": depth = int(sys.argv[3]) if len(sys.argv) > 3 else 4 dump_tree(app, max_depth=depth) elif cmd == "inspect": depth = int(sys.argv[3]) if len(sys.argv) > 3 else 2 dump_tree(app, max_depth=depth) elif cmd == "find": if len(sys.argv) < 4: print("usage: gui_accessibility find [name]", file=sys.stderr) sys.exit(1) target_role = sys.argv[3] target_name = sys.argv[4] if len(sys.argv) > 4 else "" results = find_elements(app, target_role, target_name) if not results: print(f"no elements found matching role='{target_role}' name='{target_name}'") else: print(f"{'PATH':<20} {'ROLE':<20} {'NAME'}") for path, role, name, _ in results[:50]: print(f"{path:<20} {role:<20} {name}") elif cmd == "read": if len(sys.argv) < 4: print("usage: gui_accessibility read ", file=sys.stderr) sys.exit(1) path = sys.argv[3] element = resolve_path(app, path) read_element(element) elif cmd == "click": if len(sys.argv) < 4: print("usage: gui_accessibility click ", file=sys.stderr) sys.exit(1) path = sys.argv[3] element = resolve_path(app, path) click_element(element) elif cmd == "type": if len(sys.argv) < 5: print("usage: gui_accessibility type ", file=sys.stderr) sys.exit(1) path = sys.argv[3] text = sys.argv[4] element = resolve_path(app, path) type_text(element, text) elif cmd == "action": if len(sys.argv) < 5: print("usage: gui_accessibility action ", file=sys.stderr) sys.exit(1) path = sys.argv[3] action_name = sys.argv[4] element = resolve_path(app, path) do_action(element, action_name) else: print(f"unknown command: {cmd}", file=sys.stderr) sys.exit(1) if __name__ == "__main__": main()