Fix ui being on incorrect monitor on gnome and sway

2 files changed, 293 insertions, 114 deletions

diff --git a/src/Overlay.cpp b/src/Overlay.cpp
index 732b0a4..aa14e3b 100644
--- a/src/Overlay.cpp
+++ b/src/Overlay.cpp
@@ -42,19 +42,6 @@ namespace gsr {
     static const double force_window_on_top_timeout_seconds = 1.0;
     static const double replay_status_update_check_timeout_seconds = 1.0;
 
-    static bool is_focused_application_wayland(Display *dpy) {
-        return get_focused_window(dpy, WindowCaptureType::FOCUSED) == 0;
-    }
-
-    static bool is_cursor_hovering_application_wayland(Display *dpy) {
-        Window root_window = None;
-        Window window = None;
-        int dummy_i;
-        unsigned int dummy_u;
-        XQueryPointer(dpy, DefaultRootWindow(dpy), &root_window, &window, &dummy_i, &dummy_i, &dummy_i, &dummy_i, &dummy_u);
-        return window == None;
-    }
-
     static mgl::Texture texture_from_ximage(XImage *img) {
         uint8_t *texture_data = (uint8_t*)malloc(img->width * img->height * 3);
         // TODO:
@@ -303,91 +290,6 @@ namespace gsr {
         return &win->monitors[0];
     }
 
-    static mgl::vec2i get_cursor_position(Display *dpy) {
-        Window root_window = None;
-        Window window = None;
-        int dummy_i;
-        unsigned int dummy_u;
-        mgl::vec2i root_pos;
-        XQueryPointer(dpy, DefaultRootWindow(dpy), &root_window, &window, &root_pos.x, &root_pos.y, &dummy_i, &dummy_i, &dummy_u);
-        return root_pos;
-    }
-
-    static mgl::vec2i create_window_get_center_position(Display *display) {
-        const int size = 16;
-        XSetWindowAttributes window_attr;
-        window_attr.event_mask = StructureNotifyMask;
-        window_attr.background_pixel = 0;
-        const Window window = XCreateWindow(display, DefaultRootWindow(display), 0, 0, size, size, 0, CopyFromParent, InputOutput, CopyFromParent, CWBackPixel | CWEventMask, &window_attr);
-        if(!window)
-            return {0, 0};
-
-        const Atom net_wm_window_type_atom = XInternAtom(display, "_NET_WM_WINDOW_TYPE", False);
-        const Atom net_wm_window_type_notification_atom = XInternAtom(display, "_NET_WM_WINDOW_TYPE_NOTIFICATION", False);
-        const Atom net_wm_window_type_utility = XInternAtom(display, "_NET_WM_WINDOW_TYPE_UTILITY", False);
-        const Atom net_wm_window_opacity = XInternAtom(display, "_NET_WM_WINDOW_OPACITY", False);
-
-        const Atom window_type_atoms[2] = {
-            net_wm_window_type_notification_atom,
-            net_wm_window_type_utility
-        };
-        XChangeProperty(display, window, net_wm_window_type_atom, XA_ATOM, 32, PropModeReplace, (const unsigned char*)window_type_atoms, 2L);
-
-        const double alpha = 0.0;
-        const unsigned long opacity = (unsigned long)(0xFFFFFFFFul * alpha);
-        XChangeProperty(display, window, net_wm_window_opacity, XA_CARDINAL, 32, PropModeReplace, (unsigned char *)&opacity, 1L);
-
-        XSizeHints *size_hints = XAllocSizeHints();
-        size_hints->width = size;
-        size_hints->min_width = size;
-        size_hints->max_width = size;
-        size_hints->height = size;
-        size_hints->min_height = size;
-        size_hints->max_height = size;
-        size_hints->flags = PSize | PMinSize | PMaxSize;
-        XSetWMNormalHints(display, window, size_hints);
-        XFree(size_hints);
-
-        XMapWindow(display, window);
-        XFlush(display);
-
-        const int x_fd = XConnectionNumber(display);
-        mgl::vec2i window_pos;
-        XEvent xev;
-        while(true) {
-            struct pollfd poll_fd;
-            poll_fd.fd = x_fd;
-            poll_fd.events = POLLIN;
-            poll_fd.revents = 0;
-            const int fds_ready = poll(&poll_fd, 1, 1000);
-            if(fds_ready == 0) {
-                fprintf(stderr, "Error: timed out waiting for ConfigureNotify after XCreateWindow\n");
-                break;
-            } else if(fds_ready == -1 || !(poll_fd.revents & POLLIN)) {
-                continue;
-            }
-
-            XNextEvent(display, &xev);
-            if(xev.type == ConfigureNotify) {
-                window_pos.x = xev.xconfigure.x + xev.xconfigure.width / 2;
-                window_pos.y = xev.xconfigure.y + xev.xconfigure.height / 2;
-                break;
-            }
-        }
-
-        XDestroyWindow(display, window);
-        XFlush(display);
-
-        return window_pos;
-    }
-
-    static bool is_compositor_running(Display *dpy, int screen) {
-        char prop_name[20];
-        snprintf(prop_name, sizeof(prop_name), "_NET_WM_CM_S%d", screen);
-        Atom prop_atom = XInternAtom(dpy, prop_name, False);
-        return XGetSelectionOwner(dpy, prop_atom) != None;
-    }
-
     static std::string get_power_supply_online_filepath() {
         std::string result;
         const char *paths[] = {
@@ -810,27 +712,37 @@ namespace gsr {
         mgl_context *context = mgl_get_context();
         Display *display = (Display*)context->connection;
 
+        const std::string wm_name = get_window_manager_name(display);
+        const bool is_kwin = wm_name == "KWin";
+
+        // The cursor position is wrong on wayland if an x11 window is not focused. On wayland we instead create a window and get the position where the wayland compositor puts it
+        Window x11_cursor_window = None;
+        const mgl::vec2i cursor_position = get_cursor_position(display, &x11_cursor_window);
+        const mgl::vec2i monitor_position_query_value = (x11_cursor_window || gsr_info.system_info.display_server != DisplayServer::WAYLAND) ? cursor_position : create_window_get_center_position(display);
+
         // Wayland doesn't allow XGrabPointer/XGrabKeyboard when a wayland application is focused.
         // If the focused window is a wayland application then don't use override redirect and instead create
         // a fullscreen window for the ui.
-        const bool prevent_game_minimizing = gsr_info.system_info.display_server != DisplayServer::WAYLAND || !is_focused_application_wayland(display);
+        const bool prevent_game_minimizing = gsr_info.system_info.display_server != DisplayServer::WAYLAND || x11_cursor_window;
 
-        window_size = { 1280, 720 };
+        window_size = { 32, 32 };
         window_pos = { 0, 0 };
 
         mgl::Window::CreateParams window_create_params;
         window_create_params.size = window_size;
-        window_create_params.min_size = window_size;
-        window_create_params.max_size = window_size;
+        if(prevent_game_minimizing) {
+            window_create_params.min_size = window_size;
+            window_create_params.max_size = window_size;
+        }
         window_create_params.position = window_pos;
-        window_create_params.hidden = true;
+        window_create_params.hidden = prevent_game_minimizing;
         window_create_params.override_redirect = prevent_game_minimizing;
         window_create_params.background_color = bg_color;
         window_create_params.support_alpha = true;
         window_create_params.hide_decorations = true;
         // MGL_WINDOW_TYPE_DIALOG is needed for kde plasma wayland in some cases, otherwise the window will pop up on another activity
         // or may not be visible at all
-        window_create_params.window_type = MGL_WINDOW_TYPE_DIALOG;
+        window_create_params.window_type = (is_kwin && gsr_info.system_info.display_server == DisplayServer::WAYLAND) ? MGL_WINDOW_TYPE_DIALOG : MGL_WINDOW_TYPE_NORMAL;
         window_create_params.render_api = MGL_RENDER_API_EGL;
 
         if(!window->create("gsr ui", window_create_params))
@@ -854,17 +766,16 @@ namespace gsr {
             return;
         }
 
-        // The cursor position is wrong on wayland if an x11 window is not focused. On wayland we instead create a window and get the position where the wayland compositor puts it
-        const mgl::vec2i cursor_position = get_cursor_position(display);
-        const mgl::vec2i monitor_position_query_value = gsr_info.system_info.display_server == DisplayServer::WAYLAND ? create_window_get_center_position(display) : cursor_position;
         const mgl_monitor *focused_monitor = find_monitor_at_position(*window, monitor_position_query_value);
         window_pos = {focused_monitor->pos.x, focused_monitor->pos.y};
         window_size = {focused_monitor->size.x, focused_monitor->size.y};
         get_theme().set_window_size(window_size);
 
-        window->set_size(window_size);
-        window->set_size_limits(window_size, window_size);
-        window->set_position(window_pos);
+        if(prevent_game_minimizing) {
+            window->set_size(window_size);
+            window->set_size_limits(window_size, window_size);
+            window->set_position(window_pos);
+        }
 
         win->cursor_position.x = cursor_position.x - window_pos.x;
         win->cursor_position.y = cursor_position.y - window_pos.y;
@@ -1047,7 +958,7 @@ namespace gsr {
 
         // The focused application can be an xwayland application but the cursor can hover over a wayland application.
         // This is even the case when hovering over the titlebar of the xwayland application.
-        if(!is_cursor_hovering_application_wayland(display))
+        if(prevent_game_minimizing)
             xi_setup();
 
         //window->set_fullscreen(true);
diff --git a/src/WindowUtils.cpp b/src/WindowUtils.cpp
index 057e95f..d245568 100644
--- a/src/WindowUtils.cpp
+++ b/src/WindowUtils.cpp
@@ -7,8 +7,31 @@
 #include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
+#include <poll.h>
+
+#define MAX_PROPERTY_VALUE_LEN 4096
 
 namespace gsr {
+    static unsigned char* window_get_property(Display *dpy, Window window, Atom property_type, const char *property_name, unsigned int *property_size) {
+        Atom ret_property_type = None;
+        int ret_format = 0;
+        unsigned long num_items = 0;
+        unsigned long num_remaining_bytes = 0;
+        unsigned char *data = nullptr;
+        const Atom atom = XInternAtom(dpy, property_name, False);
+        if(XGetWindowProperty(dpy, window, atom, 0, MAX_PROPERTY_VALUE_LEN / 4, False, property_type, &ret_property_type, &ret_format, &num_items, &num_remaining_bytes, &data) != Success || !data) {
+            return nullptr;
+        }
+
+        if(ret_property_type != property_type) {
+            XFree(data);
+            return nullptr;
+        }
+
+        *property_size = (ret_format / (32 / sizeof(long))) * num_items;
+        return data;
+    }
+
     static bool window_has_atom(Display *dpy, Window window, Atom atom) {
         Atom type;
         unsigned long len, bytes_left;
@@ -160,7 +183,7 @@ namespace gsr {
         return str;
     }
 
-    static std::string string_string(const char *str) {
+    static std::string strip_strip(const char *str) {
         int len = 0;
         str = strip(str, &len);
         return std::string(str, len);
@@ -175,7 +198,7 @@ namespace gsr {
         // Window title is not always ideal (for example for a browser), but for games its pretty much required
         char *window_title = get_window_title(dpy, focused_window);
         if(window_title) {
-            result = string_string(window_title);
+            result = strip_strip(window_title);
             XFree(window_title);
             return result;
         }
@@ -183,10 +206,255 @@ namespace gsr {
         XClassHint class_hint = {nullptr, nullptr};
         XGetClassHint(dpy, focused_window, &class_hint);
         if(class_hint.res_class) {
-            result = string_string(class_hint.res_class);
+            result = strip_strip(class_hint.res_class);
             return result;
         }
 
         return result;
     }
+
+    mgl::vec2i get_cursor_position(Display *dpy, Window *window) {
+        Window root_window = None;
+        *window = None;
+        int dummy_i;
+        unsigned int dummy_u;
+        mgl::vec2i root_pos;
+        XQueryPointer(dpy, DefaultRootWindow(dpy), &root_window, window, &root_pos.x, &root_pos.y, &dummy_i, &dummy_i, &dummy_u);
+
+        // This dumb shit is done to satisfy gnome wayland. Only set |window| if a valid x11 window is focused
+        if(window) {
+            XWindowAttributes attr;
+            if(XGetWindowAttributes(dpy, *window, &attr) && attr.override_redirect)
+                *window = None;
+
+            int revert_to = 0;
+            Window input_focus_window = None;
+            if(XGetInputFocus(dpy, &input_focus_window, &revert_to)) {
+                if(input_focus_window) {
+                    if(XGetWindowAttributes(dpy, input_focus_window, &attr) && attr.override_redirect)
+                        *window = None;
+                } else {
+                    *window = None;
+                }
+            }
+        }
+
+        return root_pos;
+    }
+
+    typedef struct {
+        unsigned long flags;
+        unsigned long functions;
+        unsigned long decorations;
+        long input_mode;
+        unsigned long status;
+    } MotifHints;
+
+    #define MWM_HINTS_DECORATIONS   2
+
+    #define MWM_DECOR_NONE          0
+    #define MWM_DECOR_ALL           1
+
+    static void window_set_decorations_visible(Display *display, Window window, bool visible) {
+        const Atom motif_wm_hints_atom = XInternAtom(display, "_MOTIF_WM_HINTS", False);
+        MotifHints motif_hints;
+        memset(&motif_hints, 0, sizeof(motif_hints));
+        motif_hints.flags = MWM_HINTS_DECORATIONS;
+        motif_hints.decorations = visible ? MWM_DECOR_ALL : MWM_DECOR_NONE;
+        XChangeProperty(display, window, motif_wm_hints_atom, motif_wm_hints_atom, 32, PropModeReplace, (unsigned char*)&motif_hints, sizeof(motif_hints) / sizeof(long));
+    }
+
+    static bool create_window_get_center_position_kde(Display *display, mgl::vec2i &position) {
+        const int size = 1;
+        XSetWindowAttributes window_attr;
+        window_attr.event_mask = StructureNotifyMask;
+        window_attr.background_pixel = 0;
+        const Window window = XCreateWindow(display, DefaultRootWindow(display), 0, 0, size, size, 0, CopyFromParent, InputOutput, CopyFromParent, CWBackPixel | CWEventMask, &window_attr);
+        if(!window)
+            return false;
+
+        const Atom net_wm_window_type_atom = XInternAtom(display, "_NET_WM_WINDOW_TYPE", False);
+        const Atom net_wm_window_type_notification_atom = XInternAtom(display, "_NET_WM_WINDOW_TYPE_NOTIFICATION", False);
+        const Atom net_wm_window_type_utility = XInternAtom(display, "_NET_WM_WINDOW_TYPE_UTILITY", False);
+        const Atom net_wm_window_opacity = XInternAtom(display, "_NET_WM_WINDOW_OPACITY", False);
+
+        const Atom window_type_atoms[2] = {
+            net_wm_window_type_notification_atom,
+            net_wm_window_type_utility
+        };
+        XChangeProperty(display, window, net_wm_window_type_atom, XA_ATOM, 32, PropModeReplace, (const unsigned char*)window_type_atoms, 2L);
+
+        const double alpha = 0.0;
+        const unsigned long opacity = (unsigned long)(0xFFFFFFFFul * alpha);
+        XChangeProperty(display, window, net_wm_window_opacity, XA_CARDINAL, 32, PropModeReplace, (unsigned char *)&opacity, 1L);
+
+        window_set_decorations_visible(display, window, false);
+
+        XSizeHints *size_hints = XAllocSizeHints();
+        size_hints->width = size;
+        size_hints->height = size;
+        size_hints->min_width = size;
+        size_hints->min_height = size;
+        size_hints->max_width = size;
+        size_hints->max_height = size;
+        size_hints->flags = PSize | PMinSize | PMaxSize;
+        XSetWMNormalHints(display, window, size_hints);
+        XFree(size_hints);
+
+        XMapWindow(display, window);
+        XFlush(display);
+
+        bool got_data = false;
+        const int x_fd = XConnectionNumber(display);
+        XEvent xev;
+        while(true) {
+            struct pollfd poll_fd;
+            poll_fd.fd = x_fd;
+            poll_fd.events = POLLIN;
+            poll_fd.revents = 0;
+            const int fds_ready = poll(&poll_fd, 1, 1000);
+            if(fds_ready == 0) {
+                fprintf(stderr, "Error: timed out waiting for ConfigureNotify after XCreateWindow\n");
+                break;
+            } else if(fds_ready == -1 || !(poll_fd.revents & POLLIN)) {
+                continue;
+            }
+
+            XNextEvent(display, &xev);
+            if(xev.type == ConfigureNotify) {
+                got_data = xev.xconfigure.x > 0 && xev.xconfigure.y > 0;
+                position.x = xev.xconfigure.x + xev.xconfigure.width / 2;
+                position.y = xev.xconfigure.y + xev.xconfigure.height / 2;
+                break;
+            }
+        }
+
+        XDestroyWindow(display, window);
+        XFlush(display);
+
+        return got_data;
+    }
+
+    static bool create_window_get_center_position_gnome(Display *display, mgl::vec2i &position) {
+        const int size = 32;
+        XSetWindowAttributes window_attr;
+        window_attr.event_mask = StructureNotifyMask | ExposureMask;
+        window_attr.background_pixel = 0;
+        const Window window = XCreateWindow(display, DefaultRootWindow(display), 0, 0, size, size, 0, CopyFromParent, InputOutput, CopyFromParent, CWBackPixel | CWEventMask, &window_attr);
+        if(!window)
+            return false;
+
+        const Atom net_wm_window_opacity = XInternAtom(display, "_NET_WM_WINDOW_OPACITY", False);
+        const double alpha = 0.0;
+        const unsigned long opacity = (unsigned long)(0xFFFFFFFFul * alpha);
+        XChangeProperty(display, window, net_wm_window_opacity, XA_CARDINAL, 32, PropModeReplace, (unsigned char *)&opacity, 1L);
+
+        window_set_decorations_visible(display, window, false);
+
+        XSizeHints *size_hints = XAllocSizeHints();
+        size_hints->width = size;
+        size_hints->height = size;
+        size_hints->min_width = size;
+        size_hints->min_height = size;
+        size_hints->max_width = size;
+        size_hints->max_height = size;
+        size_hints->flags = PSize | PMinSize | PMaxSize;
+        XSetWMNormalHints(display, window, size_hints);
+        XFree(size_hints);
+
+        XMapWindow(display, window);
+        XFlush(display);
+
+        bool got_data = false;
+        const int x_fd = XConnectionNumber(display);
+        XEvent xev;
+        while(true) {
+            struct pollfd poll_fd;
+            poll_fd.fd = x_fd;
+            poll_fd.events = POLLIN;
+            poll_fd.revents = 0;
+            const int fds_ready = poll(&poll_fd, 1, 1000);
+            if(fds_ready == 0) {
+                fprintf(stderr, "Error: timed out waiting for MapNotify/ConfigureNotify after XCreateWindow\n");
+                break;
+            } else if(fds_ready == -1 || !(poll_fd.revents & POLLIN)) {
+                continue;
+            }
+
+            XNextEvent(display, &xev);
+            if(xev.type == MapNotify) {
+                int x = 0;
+                int y = 0;
+                Window w = None;
+                XTranslateCoordinates(display, window, DefaultRootWindow(display), 0, 0, &x, &y, &w);
+
+                got_data = x > 0 && y > 0;
+                position.x = x + size / 2;
+                position.y = y + size / 2;
+                if(got_data)
+                    break;
+            } else if(xev.type == ConfigureNotify) {
+                got_data = xev.xconfigure.x > 0 && xev.xconfigure.y > 0;
+                position.x = xev.xconfigure.x + xev.xconfigure.width / 2;
+                position.y = xev.xconfigure.y + xev.xconfigure.height / 2;
+                if(got_data)
+                    break;
+            }
+        }
+
+        XDestroyWindow(display, window);
+        XFlush(display);
+
+        return got_data;
+    }
+
+    mgl::vec2i create_window_get_center_position(Display *display) {
+        mgl::vec2i pos;
+        if(!create_window_get_center_position_kde(display, pos)) {
+            pos.x = 0;
+            pos.y = 0;
+            create_window_get_center_position_gnome(display, pos);
+        }
+        return pos;
+    }
+
+    std::string get_window_manager_name(Display *display) {
+        std::string wm_name;
+        unsigned int property_size = 0;
+        Window window = None;
+
+        unsigned char *net_supporting_wm_check = window_get_property(display, DefaultRootWindow(display), XA_WINDOW, "_NET_SUPPORTING_WM_CHECK", &property_size);
+        if(net_supporting_wm_check) {
+            if(property_size == 8)
+                window = *(Window*)net_supporting_wm_check;
+            XFree(net_supporting_wm_check);
+        }
+
+        if(!window) {
+            unsigned char *win_supporting_wm_check = window_get_property(display, DefaultRootWindow(display), XA_WINDOW, "_WIN_SUPPORTING_WM_CHECK", &property_size);
+            if(win_supporting_wm_check) {
+                if(property_size == 8)
+                    window = *(Window*)win_supporting_wm_check;
+                XFree(win_supporting_wm_check);
+            }
+        }
+
+        if(!window)
+            return wm_name;
+
+        char *window_title = get_window_title(display, window);
+        if(window_title) {
+            wm_name = strip_strip(window_title);
+            XFree(window_title);
+        }
+
+        return wm_name;
+    }
+
+    bool is_compositor_running(Display *dpy, int screen) {
+        char prop_name[20];
+        snprintf(prop_name, sizeof(prop_name), "_NET_WM_CM_S%d", screen);
+        Atom prop_atom = XInternAtom(dpy, prop_name, False);
+        return XGetSelectionOwner(dpy, prop_atom) != None;
+    }
 }
\ No newline at end of file