Use window texture (xcomposite) for background if the window is fullscreen on the selected monitor

2 files changed, 246 insertions, 23 deletions

diff --git a/src/main.cpp b/src/main.cpp
index 6a0745f..f2cd629 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -12,6 +12,7 @@
 #include "../include/Process.hpp"
 #include "../include/Theme.hpp"
 #include "../include/GsrInfo.hpp"
+#include "../include/window_texture.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -40,10 +41,6 @@
 #include <mglpp/window/Event.hpp>
 #include <mglpp/system/Clock.hpp>
 
-// TODO: If no compositor is running but a fullscreen application is running (on the focused monitor)
-// then use xcomposite to get that window as a texture and use that as a background because then the background can update.
-// That case can also happen when using a compositor but when the compositor gets turned off when running a fullscreen application.
-// This can also happen after this overlay has started, in which case we want to update the background capture method.
 // TODO: Make keyboard controllable for steam deck (and other controllers).
 // TODO: Keep track of gpu screen recorder run by other programs to not allow recording at the same time, or something.
 // TODO: Remove gpu-screen-recorder-overlay-daemon and handle that alt+z global hotkey here instead, to show/hide the window
@@ -174,6 +171,54 @@ static std::string color_to_hex_str(mgl::Color color) {
     return result;
 }
 
+static Window get_window_at_cursor_position(Display *display) {
+    Window root_window = None;
+    Window window = None;
+    int dummy_i;
+    unsigned int dummy_u;
+    int cursor_pos_x = 0;
+    int cursor_pos_y = 0;
+    XQueryPointer(display, DefaultRootWindow(display), &root_window, &window, &dummy_i, &dummy_i, &cursor_pos_x, &cursor_pos_y, &dummy_u);
+    return window;
+}
+
+struct DrawableGeometry {
+    int x, y, width, height;
+};
+
+static bool get_drawable_geometry(Display *display, Drawable drawable, DrawableGeometry *geometry) {
+    geometry->x = 0;
+    geometry->y = 0;
+    geometry->width = 0;
+    geometry->height = 0;
+
+    Window root_window;
+    unsigned int w, h;
+    unsigned int dummy_border, dummy_depth;
+    Status s = XGetGeometry(display, drawable, &root_window, &geometry->x, &geometry->y, &w, &h, &dummy_border, &dummy_depth);
+
+    geometry->width = w;
+    geometry->height = h;
+    return s != Success;
+}
+
+static bool diff_int(int a, int b, int difference) {
+    return std::abs(a - b) <= difference;
+}
+
+static bool is_window_fullscreen_on_monitor(Display *display, Window window, const mgl_monitor *monitor) {
+    if(!window)
+        return false;
+
+    DrawableGeometry geometry;
+    if(!get_drawable_geometry(display, window, &geometry))
+        return false;
+
+    const int margin = 2;
+    return diff_int(geometry.x, monitor->pos.x, margin) && diff_int(geometry.y, monitor->pos.y, margin)
+        && diff_int(geometry.width, monitor->size.x, margin) && diff_int(geometry.height, monitor->size.y, margin);
+}
+
 // Returns the first monitor if not found. Assumes there is at least one monitor connected.
 static const mgl_monitor* find_monitor_by_cursor_position(mgl::Window &window) {
     const mgl_window *win = window.internal_window();
@@ -454,11 +499,21 @@ int main(int argc, char **argv) {
     }
 
     mgl::Init init;
-    Display *display = (Display*)mgl_get_context()->connection;
+    mgl_context *context = mgl_get_context();
+    Display *display = (Display*)context->connection;
+
+    egl_functions egl_funcs;
+    egl_funcs.eglGetError = (decltype(egl_funcs.eglGetError))context->gl.eglGetProcAddress("eglGetError");
+    egl_funcs.eglCreateImage = (decltype(egl_funcs.eglCreateImage))context->gl.eglGetProcAddress("eglCreateImage");
+    egl_funcs.eglDestroyImage = (decltype(egl_funcs.eglDestroyImage))context->gl.eglGetProcAddress("eglDestroyImage");
+    egl_funcs.glEGLImageTargetTexture2DOES = (decltype(egl_funcs.glEGLImageTargetTexture2DOES))context->gl.eglGetProcAddress("glEGLImageTargetTexture2DOES");
+
+    if(!egl_funcs.eglGetError || !egl_funcs.eglCreateImage || !egl_funcs.eglDestroyImage || !egl_funcs.glEGLImageTargetTexture2DOES) {
+        fprintf(stderr, "Error: required opengl functions not available on your system\n");
+        exit(1);
+    }
 
-    // TODO: Put window on the focused monitor right side and update when monitor changes resolution or other modes.
-    // Use monitor size instead of screen size.
-    // mgl now has monitor events so this can be handled directly with mgl.
+    const bool compositor_running = is_compositor_running(display, DefaultScreen(display));
 
     mgl::vec2i window_size = { 1280, 720 };
     mgl::vec2i window_pos = { 0, 0 };
@@ -516,16 +571,44 @@ int main(int argc, char **argv) {
     if(!stream_button_texture.load_from_file((resources_path + "images/stream.png").c_str()))
         startup_error("failed to load texture: images/stream.png");
 
+    WindowTexture window_texture;
+    bool window_texture_loaded = false;
+
+    mgl_texture window_texture_tex;
+    memset(&window_texture_tex, 0, sizeof(window_texture_tex));
+    mgl::Texture window_texture_texture;
+    mgl::Sprite window_texture_sprite;
+
     mgl::Texture screenshot_texture;
-    if(!is_compositor_running(display, 0)) {
-        XImage *img = XGetImage(display, DefaultRootWindow(display), window_pos.x, window_pos.y, window_size.x, window_size.y, AllPlanes, ZPixmap);
-        if(!img)
-            fprintf(stderr, "Error: failed to take a screenshot\n");
-
-        if(img) {
-            screenshot_texture = texture_from_ximage(img);
-            XDestroyImage(img);
-            img = NULL;
+    if(!compositor_running) {
+        const Window window_at_cursor_position = get_window_at_cursor_position(display);
+        if(is_window_fullscreen_on_monitor(display, window_at_cursor_position, focused_monitor) && window_at_cursor_position)
+            window_texture_loaded = window_texture_init(&window_texture, display, mgl_window_get_egl_display(window.internal_window()), window_at_cursor_position, egl_funcs) == 0;
+
+        if(window_texture_loaded && window_texture.texture_id) {
+            DrawableGeometry geometry;
+            get_drawable_geometry(display, (Drawable)window_texture.pixmap, &geometry);
+
+            window_texture_tex.id = window_texture.texture_id;
+            window_texture_tex.width = geometry.width;
+            window_texture_tex.height = geometry.height;
+            window_texture_tex.format = MGL_TEXTURE_FORMAT_RGBA;
+            window_texture_tex.max_width = 1 << 15;
+            window_texture_tex.max_height = 1 << 15;
+            window_texture_tex.pixel_coordinates = false;
+            window_texture_tex.mipmap = false;
+            window_texture_texture = mgl::Texture::reference(window_texture_tex);
+            window_texture_sprite.set_texture(&window_texture_texture);
+        } else {
+            XImage *img = XGetImage(display, DefaultRootWindow(display), window_pos.x, window_pos.y, window_size.x, window_size.y, AllPlanes, ZPixmap);
+            if(!img)
+                fprintf(stderr, "Error: failed to take a screenshot\n");
+
+            if(img) {
+                screenshot_texture = texture_from_ximage(img);
+                XDestroyImage(img);
+                img = NULL;
+            }
         }
     }
 
@@ -844,9 +927,6 @@ int main(int argc, char **argv) {
         top_bar_background.get_size().y * 0.5f - logo_sprite.get_size().y * 0.5f
     ).floor());
 
-    // mgl::Clock state_update_timer;
-    // const double state_update_timeout_sec = 2.0;
-
     mgl::Event event;
 
     event.type = mgl::Event::MouseMoved;
@@ -855,8 +935,11 @@ int main(int argc, char **argv) {
     page_stack.top()->on_event(event, window, mgl::vec2f(0.0f, 0.0f));
 
     const auto render = [&] {
-        window.clear(bg_color);
-        if(screenshot_texture.is_valid()) {
+        if(window_texture_loaded && window_texture.texture_id) {
+            window.clear(mgl::Color(0, 0, 0, 255));
+            window.draw(window_texture_sprite);
+        } else if(screenshot_texture.is_valid()) {
+            window.clear(bg_color);
             window.draw(screenshot_sprite);
             window.draw(bg_screenshot_overlay);
         }
@@ -876,7 +959,7 @@ int main(int argc, char **argv) {
 
         while(window.poll_event(event)) {
             page_stack.top()->on_event(event, window, mgl::vec2f(0.0f, 0.0f));
-            if(event.type == mgl::Event::KeyPressed) {
+            if(event.type == mgl::Event::KeyReleased) {
                 if(event.key.code == mgl::Keyboard::Escape && !page_stack.empty())
                     page_stack.pop();
             }
@@ -896,6 +979,8 @@ int main(int argc, char **argv) {
 
     quit:
     fprintf(stderr, "shutting down!\n");
+    if(window_texture_loaded)
+        window_texture_deinit(&window_texture);
     gsr::deinit_theme();
     window.close();
 
diff --git a/src/window_texture.c b/src/window_texture.c
new file mode 100644
index 0000000..6fe76a7
--- /dev/null
+++ b/src/window_texture.c
@@ -0,0 +1,138 @@
+#include "../include/window_texture.h"
+#include <X11/extensions/Xcomposite.h>
+#include <mgl/mgl.h>
+#include <stdio.h>
+
+#define EGL_IMAGE_PRESERVED_KHR 0x30D2
+#define EGL_TRUE 1
+#define EGL_NATIVE_PIXMAP_KHR 0x30B0
+
+static int x11_supports_composite_named_window_pixmap(Display *display) {
+    int extension_major;
+    int extension_minor;
+    if(!XCompositeQueryExtension(display, &extension_major, &extension_minor))
+        return 0;
+
+    int major_version;
+    int minor_version;
+    return XCompositeQueryVersion(display, &major_version, &minor_version) && (major_version > 0 || minor_version >= 2);
+}
+
+int window_texture_init(WindowTexture *window_texture, Display *display, void *egl_display, uint64_t window, egl_functions egl_funcs) {
+    window_texture->display = display;
+    window_texture->egl_display = egl_display;
+    window_texture->window = window;
+    window_texture->pixmap = None;
+    window_texture->texture_id = 0;
+    window_texture->redirected = 0;
+    window_texture->egl_funcs = egl_funcs;
+    
+    if(!x11_supports_composite_named_window_pixmap(display))
+        return 1;
+
+    XCompositeRedirectWindow(display, window, CompositeRedirectAutomatic);
+    window_texture->redirected = 1;
+    return window_texture_on_resize(window_texture);
+}
+
+static void window_texture_cleanup(WindowTexture *self, int delete_texture) {
+    mgl_context *context = mgl_get_context();
+
+    if(delete_texture && self->texture_id) {
+        context->gl.glDeleteTextures(1, &self->texture_id);
+        self->texture_id = 0;
+    }
+
+    if(self->pixmap) {
+        XFreePixmap(self->display, self->pixmap);
+        self->pixmap = None;
+    }
+}
+
+void window_texture_deinit(WindowTexture *self) {
+    if(self->redirected) {
+        XCompositeUnredirectWindow(self->display, self->window, CompositeRedirectAutomatic);
+        self->redirected = 0;
+    }
+    window_texture_cleanup(self, 1);
+}
+
+int window_texture_on_resize(WindowTexture *self) {
+    window_texture_cleanup(self, 0);
+    mgl_context *context = mgl_get_context();
+
+    int result = 0;
+    Pixmap pixmap = None;
+    unsigned int texture_id = 0;
+    void *image = NULL;
+
+    const intptr_t pixmap_attrs[] = {
+        EGL_IMAGE_PRESERVED_KHR, EGL_TRUE,
+        EGL_NONE,
+    };
+
+    pixmap = XCompositeNameWindowPixmap(self->display, self->window);
+    if(!pixmap) {
+        result = 2;
+        goto cleanup;
+    }
+
+    if(self->texture_id == 0) {
+        context->gl.glGenTextures(1, &texture_id);
+        if(texture_id == 0) {
+            result = 4;
+            goto cleanup;
+        }
+        context->gl.glBindTexture(GL_TEXTURE_2D, texture_id);
+    } else {
+        context->gl.glBindTexture(GL_TEXTURE_2D, self->texture_id);
+    }
+
+    context->gl.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    context->gl.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    context->gl.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    context->gl.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+ 
+    /*
+    float fLargest = 0.0f;
+    glGetFloatv(GL_MAX_TEXTURE_MAX_ANISOTROPY_EXT, &fLargest);
+    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, fLargest);
+    */
+
+    while(context->gl.glGetError()) {}
+    while(self->egl_funcs.eglGetError() != EGL_SUCCESS) {}
+
+    image = self->egl_funcs.eglCreateImage(self->egl_display, NULL, EGL_NATIVE_PIXMAP_KHR, (void*)pixmap, pixmap_attrs);
+    if(!image) {
+        result = 4;
+        goto cleanup;
+    }
+
+    self->egl_funcs.glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, image);
+    if(context->gl.glGetError() != 0 || self->egl_funcs.eglGetError() != EGL_SUCCESS) {
+        result = 5;
+        goto cleanup;
+    }
+
+    self->pixmap = pixmap;
+    self->texture_id = texture_id;
+
+    cleanup:
+    context->gl.glBindTexture(GL_TEXTURE_2D, 0);
+
+    if(image)
+        self->egl_funcs.eglDestroyImage(self->egl_display, image);
+
+    if(result != 0) {
+        if(texture_id != 0)
+            context->gl.glDeleteTextures(1, &texture_id);
+        if(pixmap)
+            XFreePixmap(self->display, pixmap);
+    }
+
+    return result;
+}
+
+unsigned int window_texture_get_opengl_texture_id(WindowTexture *self) {
+    return self->texture_id;
+}