From 9104a1944f5b845beb6d6f4e6b377fae242f9f96 Mon Sep 17 00:00:00 2001 From: Levi Neely Date: Mon, 30 Mar 2026 11:23:47 +0200 Subject: [PATCH] wayland: auto-detect nested mode from WAYLAND_DISPLAY Previously nested compositor mode required WMAKER_WAYLAND_NESTED=1 to be set explicitly. Now wl_display_open() checks WAYLAND_DISPLAY: if it is set the process is already running inside a Wayland session, so wlr_wl_backend is used automatically and wmaker appears as a window on the host desktop. WMAKER_WAYLAND_NESTED=0 overrides the auto-detection and forces headless mode (used by the smoke test to avoid accidentally going nested when WAYLAND_DISPLAY is set in the test environment). Smoke test: add WMAKER_WAYLAND_NESTED=0 to the wmaker launch line so the headless path is always exercised regardless of the environment. Co-Authored-By: Claude Sonnet 4.5 --- src/backend/wayland/wl_backend.c | 13 ++++++++++++- tests/wl-smoke-test.sh | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/backend/wayland/wl_backend.c b/src/backend/wayland/wl_backend.c index a99f638..81bfce7 100644 --- a/src/backend/wayland/wl_backend.c +++ b/src/backend/wayland/wl_backend.c @@ -959,9 +959,20 @@ wl_display_open(const char *name) * is set (run inside an existing Wayland compositor), otherwise headless. * Both modes always include a headless backend (headless_inner) so that * wlr_headless_add_input_device works regardless of nesting. */ + /* Auto-detect nested mode: if WAYLAND_DISPLAY is set we are running inside + * an existing Wayland session and should use wlr_wl_backend so the + * compositor appears as a window on the host desktop. + * WMAKER_WAYLAND_NESTED=0 can be used to force headless even when + * WAYLAND_DISPLAY is present (e.g. for automated testing). */ { const char *nested_env = getenv("WMAKER_WAYLAND_NESTED"); - wl_state.is_nested = (nested_env && nested_env[0]) ? 1 : 0; + const char *wayland_display = getenv("WAYLAND_DISPLAY"); + if (nested_env && nested_env[0] == '0') + wl_state.is_nested = 0; /* explicit override: force headless */ + else if (nested_env && nested_env[0]) + wl_state.is_nested = 1; /* explicit override: force nested */ + else + wl_state.is_nested = (wayland_display && wayland_display[0]) ? 1 : 0; } /* Always create a headless backend for input devices. */ diff --git a/tests/wl-smoke-test.sh b/tests/wl-smoke-test.sh index 5f5aa87..1d05c71 100755 --- a/tests/wl-smoke-test.sh +++ b/tests/wl-smoke-test.sh @@ -179,7 +179,7 @@ if [ -z "$XDG_RUNTIME_DIR" ]; then export XDG_RUNTIME_DIR fi -WMAKER_USE_WAYLAND=1 src/wmaker \ +WMAKER_USE_WAYLAND=1 WMAKER_WAYLAND_NESTED=0 src/wmaker \ --no-dock --no-clip --no-autolaunch \ --dont-restore --static \ --for-real 2>/tmp/wl-smoke-stderr &