From 398d4678800c8db0b7d0be221701f1af6c01aa65 Mon Sep 17 00:00:00 2001 From: Levi Neely <141506390+lneely@users.noreply.github.com> Date: Mon, 11 May 2026 04:32:02 +0200 Subject: [PATCH] =?UTF-8?q?wayland:=20fix=20nested=20mode=20crash=20?= =?UTF-8?q?=E2=80=94=20null=20seat=20+=200=C3=970=20output=20on=20backend?= =?UTF-8?q?=5Fstart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs prevented nested Wayland mode (inside sway) from starting: 1. The wlr_wl_backend fires new_input during wlr_backend_start(), before wl_state.seat exists (seat is created later in wl_display_open). wlr_seat_set_keyboard(NULL, kb) → SIGSEGV. Guard with if (seat). 2. Sway's initial xdg-shell configure sends width=height=0 ("compositor chooses"). Without an explicit mode, wlr_output_commit_state tries to create a 0×0 GBM swapchain → crash in gbm_bo_create. Fall back to 1920×1080 when no preferred mode is available. Also install XSetErrorHandler at the top of wl_display_open so the monitor process's crash-dialog BadMatch (from sway's internal XWayland at :0) is suppressed rather than printed in Xlib default format. Co-Authored-By: Claude Sonnet 4.6 --- src/backend/wayland/wl_backend.c | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/backend/wayland/wl_backend.c b/src/backend/wayland/wl_backend.c index 588f48a..50e360c 100644 --- a/src/backend/wayland/wl_backend.c +++ b/src/backend/wayland/wl_backend.c @@ -468,6 +468,9 @@ frame_timer_cb(void *data) #define VIEW_FROM_LISTENER(ptr, field) \ ((struct wl_toplevel_view *)((char *)(ptr) - offsetof(struct wl_toplevel_view, field))) +/* X error handler — defined near the vtable at the bottom of this file. */ +static int wl_catch_xwayland_error(Display *edpy, XErrorEvent *error); + /* Phase 43: forward declaration — defined in the focus section below. */ static void wl_focus_set(WNativeWindow win, int revert_to, unsigned long timestamp); /* Phase 50: forward declaration — defined in client section below. */ @@ -1394,7 +1397,10 @@ handle_backend_new_input(struct wl_listener *listener, void *data) wlr_keyboard_set_keymap(kb, wl_state.xkb_keymap); wl_state.real_keyboard_dev = dev; wl_state.keyboard_dev = dev; - wlr_seat_set_keyboard(wl_state.seat, kb); + /* Seat may not exist yet if new_input fires during wlr_backend_start() + * before wlr_seat_create() is called later in wl_display_open(). */ + if (wl_state.seat) + wlr_seat_set_keyboard(wl_state.seat, kb); wl_state.real_keyboard_key_listener.notify = handle_keyboard_key; wl_signal_add(&kb->events.key, &wl_state.real_keyboard_key_listener); @@ -1484,13 +1490,24 @@ handle_backend_new_output(struct wl_listener *listener, void *data) if (wl_state.allocator && wl_state.renderer) wlr_output_init_render(output, wl_state.allocator, wl_state.renderer); - /* Enable the output: select preferred mode, then commit via state API. */ + /* Enable the output: select preferred mode, then commit via state API. + * Nested Wayland outputs: sway's initial xdg-shell configure sends 0×0 + * ("compositor defers to client"), so output->width/height stay 0 after + * the configure busy-loop. Without an explicit mode the swapchain tries + * to allocate a 0×0 GBM buffer which crashes. Fall back to 1920×1080 so + * the commit succeeds; sway will send a real configure once the window is + * visible and we handle it in the frame/configure signal path. */ { struct wlr_output_state state; wlr_output_state_init(&state); wlr_output_state_set_enabled(&state, true); struct wlr_output_mode *mode = wlr_output_preferred_mode(output); - if (mode) wlr_output_state_set_mode(&state, mode); + if (mode) { + wlr_output_state_set_mode(&state, mode); + } else { + /* Default size for nested mode (or headless with no preferred mode). */ + wlr_output_state_set_custom_mode(&state, 1920, 1080, 0); + } if (!wlr_output_commit_state(output, &state)) wwarning("wl_backend: initial output commit failed"); wlr_output_state_finish(&state); @@ -1557,6 +1574,14 @@ wl_display_open(const char *name) (void)name; /* Wayland compositor picks its own socket name */ + /* Install X error handler immediately: Mesa/EGL may open a connection to + * DISPLAY (e.g. the host compositor's XWayland) during wlr_backend_start, + * which happens before StartUp() calls register_error_handlers(). + * Installing here prevents Xlib's default handler from aborting the + * process on the benign BadMatch that XWayland generates on SetInputFocus + * during WM startup. */ + XSetErrorHandler(wl_catch_xwayland_error); + wlr_log_init(WLR_ERROR, NULL); wl_state.display = wl_display_create();