GNOME: fix recording windows with client-side decorations by recording the compositor window and tracking the window

3 files changed, 168 insertions, 30 deletions

diff --git a/README.md b/README.md
index fc6c702..6a796f8 100644
--- a/README.md
+++ b/README.md
@@ -6,8 +6,6 @@ This screen recorder can be used for recording your desktop offline, for live st
 where only the last few seconds are saved.
 
 ## Note
-Might now work when using a compositor such as picom when using the glx backend. Using the xrender backend with picom fixes this issue.\
-Does not work when using gtk client side decorations (such as on Pop OS). Either disable those (if possible), install gtk-nocsd or record the whole monitor/screen if you have NvFBC.\
 For NvFBC to work with PRIME, you must set the primary GPU to your dedicated Nvidia graphics card. On Pop OS, you can select the 'NVIDIA Graphics' option in the power menu, or on Arch Linux you can use Optimus Manager.\
 If you are using a variable refresh rate monitor, then choose to record "screen-direct". This will allow variable refresh rate to work when recording fullscreen applications. Note that some applications such as mpv will not work in fullscreen mode. A fix is being developed for this.\
 
diff --git a/TODO b/TODO
index 030605d..a5739fe 100644
--- a/TODO
+++ b/TODO
@@ -8,7 +8,11 @@ Quickly changing workspace and back while recording under i3 breaks the screen r
 Remove hw_get_frame as it creates a new cuda device ptr which we dont use!
 Nvidia 515.57 supports nvfbc direct capture with mouse capture. Check if driver is equal or newer than this and use mouse capture in such situations (with direct capture) supports nvfbc direct capture with mouse capture.
 See https://trac.ffmpeg.org/wiki/EncodingForStreamingSites for optimizing streaming.
-Add -ma option to merge all audio tracks into one (muxing?).
+Add option to merge audio tracks into one (muxing?) by adding multiple audio streams in one -a arg separated by comma.
 Look at VK_EXT_external_memory_dma_buf.
 Allow setting a different output resolution than the input resolution.
 Use mov+faststart.
+Allow recording all monitors/selected monitor without nvfbc by recording the compositor proxy window and only recording the part that matches the monitor(s).
+Allow recording a region by recording the compositor proxy window / nvfbc window and copying part of it.
+Resizing the target window to be smaller than the initial size is buggy. The window texture ends up duplicated in the video.
+Handle frames (especially for applications with rounded client-side decorations, such as gnome applications. They are huge).
diff --git a/src/main.cpp b/src/main.cpp
index fb466c2..25b34d4 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -39,6 +39,7 @@
 #include <GLFW/glfw3.h>
 
 #include <X11/extensions/Xcomposite.h>
+//#include <X11/Xatom.h>
 
 extern "C" {
 #include <libavutil/pixfmt.h>
@@ -87,17 +88,18 @@ struct ScopedGLXFBConfig {
 };
 
 struct WindowPixmap {
-    WindowPixmap()
-        : pixmap(None), glx_pixmap(None), texture_id(0), target_texture_id(0),
-          texture_width(0), texture_height(0) {}
+    Pixmap pixmap = None;
+    GLXPixmap glx_pixmap = None;
+    GLuint texture_id = 0;
+    GLuint target_texture_id = 0;
 
-    Pixmap pixmap;
-    GLXPixmap glx_pixmap;
-    GLuint texture_id;
-    GLuint target_texture_id;
+    GLint texture_width = 0;
+    GLint texture_height = 0;
 
-    GLint texture_width;
-    GLint texture_height;
+    GLint texture_real_width = 0;
+    GLint texture_real_height = 0;
+
+    Window composite_window = None;
 };
 
 enum class VideoQuality {
@@ -153,6 +155,53 @@ static int x11_io_error_handler(Display *dpy) {
     return 0;
 }
 
+static Window get_compositor_window(Display *display) {
+    Window overlay_window = XCompositeGetOverlayWindow(display, DefaultRootWindow(display));
+    XCompositeReleaseOverlayWindow(display, DefaultRootWindow(display));
+
+    /*
+    Atom xdnd_proxy = XInternAtom(display, "XdndProxy", False);
+    if(!xdnd_proxy)
+        return None;
+
+    Atom type = None;
+    int format = 0;
+    unsigned long nitems = 0, after = 0;
+    unsigned char *data = nullptr;
+    if(XGetWindowProperty(display, overlay_window, xdnd_proxy, 0, 1, False, XA_WINDOW, &type, &format, &nitems, &after, &data) != Success)
+        return None;
+
+    fprintf(stderr, "type: %ld, format: %d, num items: %lu\n", type, format, nitems);
+    if(type == XA_WINDOW && format == 32 && nitems == 1)
+        fprintf(stderr, "Proxy window: %ld\n", *(Window*)data);
+
+    if(data)
+        XFree(data);
+    */
+
+    Window root_window, parent_window;
+    Window *children = nullptr;
+    unsigned int num_children = 0;
+    if(XQueryTree(display, overlay_window, &root_window, &parent_window, &children, &num_children) == 0)
+        return None;
+
+    Window compositor_window = None;
+    if(num_children == 1) {
+        compositor_window = children[0];
+        const int screen_width = XWidthOfScreen(DefaultScreenOfDisplay(display));
+        const int screen_height = XHeightOfScreen(DefaultScreenOfDisplay(display));
+
+        XWindowAttributes attr;
+        if(!XGetWindowAttributes(display, compositor_window, &attr) || attr.width != screen_width || attr.height != screen_height)
+            compositor_window = None;
+    }
+
+    if(children)
+        XFree(children);
+
+    return compositor_window;
+}
+
 static void cleanup_window_pixmap(Display *dpy, WindowPixmap &pixmap) {
     if (pixmap.target_texture_id) {
         glDeleteTextures(1, &pixmap.target_texture_id);
@@ -164,6 +213,8 @@ static void cleanup_window_pixmap(Display *dpy, WindowPixmap &pixmap) {
         pixmap.texture_id = 0;
         pixmap.texture_width = 0;
         pixmap.texture_height = 0;
+        pixmap.texture_real_width = 0;
+        pixmap.texture_real_height = 0;
     }
 
     if (pixmap.glx_pixmap) {
@@ -176,10 +227,15 @@ static void cleanup_window_pixmap(Display *dpy, WindowPixmap &pixmap) {
         XFreePixmap(dpy, pixmap.pixmap);
         pixmap.pixmap = None;
     }
+
+    if(pixmap.composite_window) {
+        XCompositeUnredirectWindow(dpy, pixmap.composite_window, CompositeRedirectAutomatic);
+        pixmap.composite_window = None;
+    }
 }
 
 static bool recreate_window_pixmap(Display *dpy, Window window_id,
-                                   WindowPixmap &pixmap) {
+                                   WindowPixmap &pixmap, bool fallback_composite_window = true) {
     cleanup_window_pixmap(dpy, pixmap);
 
     XWindowAttributes attr;
@@ -273,10 +329,41 @@ static bool recreate_window_pixmap(Display *dpy, Window window_id,
     glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT,
                              &pixmap.texture_height);
 
+    pixmap.texture_real_width = pixmap.texture_width;
+    pixmap.texture_real_height = pixmap.texture_height;
+
     if(pixmap.texture_width == 0 || pixmap.texture_height == 0) {
+        glBindTexture(GL_TEXTURE_2D, 0);        
         pixmap.texture_width = attr.width;
         pixmap.texture_height = attr.height;
-        fprintf(stderr, "Warning: failed to get texture size. You are probably running an unsupported compositor and recording the selected window doesn't work at the moment. This could also happen if you are trying to record a window with client-side decorations (GNOME issue). A black window will be displayed instead. A workaround is to record the whole monitor (which use NvFBC).\n");
+
+        pixmap.texture_real_width = pixmap.texture_width;
+        pixmap.texture_real_height = pixmap.texture_height;
+
+        if(fallback_composite_window) {
+            Window compositor_window = get_compositor_window(dpy);
+            if(!compositor_window) {
+                fprintf(stderr, "Warning: failed to get texture size. You are probably running an unsupported compositor and recording the selected window doesn't work at the moment. This could also happen if you are trying to record a window with client-side decorations. A black window will be displayed instead. A workaround is to record the whole monitor (which uses NvFBC).\n");
+                return false;
+            }
+
+            fprintf(stderr, "Warning: failed to get texture size. You are probably trying to record a window with client-side decorations (using GNOME?). Trying to fallback to recording the compositor proxy window\n");
+            XCompositeRedirectWindow(dpy, compositor_window, CompositeRedirectAutomatic);
+
+            if(recreate_window_pixmap(dpy, compositor_window, pixmap, false)) {
+                pixmap.composite_window = compositor_window;
+                pixmap.texture_width = attr.width;
+                pixmap.texture_height = attr.height;
+                return true;
+            }
+
+            pixmap.texture_width = attr.width;
+            pixmap.texture_height = attr.height;
+
+            return false;
+        } else {
+            fprintf(stderr, "Warning: failed to get texture size. You are probably running an unsupported compositor and recording the selected window doesn't work at the moment. This could also happen if you are trying to record a window with client-side decorations. A black window will be displayed instead. A workaround is to record the whole monitor (which uses NvFBC).\n");
+        }
     }
 
     fprintf(stderr, "texture width: %d, height: %d\n", pixmap.texture_width,
@@ -1004,6 +1091,8 @@ int main(int argc, char **argv) {
 
     uint32_t window_width = 0;
     uint32_t window_height = 0;
+    int window_x = 0;
+    int window_y = 0;
 
     NvFBCLibrary nv_fbc_library;
 
@@ -1086,8 +1175,12 @@ int main(int argc, char **argv) {
             return 1;
         }
 
-        window_width = attr.width;
-        window_height = attr.height;
+        window_width = std::max(0, attr.width);
+        window_height = std::max(0, attr.height);
+        window_x = attr.x;
+        window_y = attr.y;
+        Window c;    
+        XTranslateCoordinates(dpy, src_window_id, DefaultRootWindow(dpy), 0, 0, &window_x, &window_y, &c);
 
         XCompositeRedirectWindow(dpy, src_window_id, CompositeRedirectAutomatic);
 
@@ -1129,11 +1222,7 @@ int main(int argc, char **argv) {
         }
         glGetError(); // to clear the error caused deep in GLEW
 
-        if (!recreate_window_pixmap(dpy, src_window_id, window_pixmap)) {
-            fprintf(stderr, "Error: Failed to create glx pixmap for window: %lu\n",
-                    src_window_id);
-            return 1;
-        }
+        recreate_window_pixmap(dpy, src_window_id, window_pixmap);
 
         if(!record_area) {
             record_width = window_pixmap.texture_width;
@@ -1312,8 +1401,6 @@ int main(int argc, char **argv) {
     std::deque<AVPacket> frame_data_queue;
     bool frames_erased = false;
 
-    double prev_video_frame_time = clock_get_monotonic_seconds();
-
     const size_t audio_buffer_size = 1024 * 2 * 2;
     uint8_t *empty_audio = (uint8_t*)malloc(audio_buffer_size); // see sound.cpp
     if(!empty_audio) {
@@ -1418,6 +1505,7 @@ int main(int argc, char **argv) {
     handle_new_pid_file(replay_buffer_size_secs == -1 ? "record" : "replay");
     started = 1;
 
+    // Set update_fps to 24 to test if duplicate/delayed frames cause video/audio desync or too fast/slow video.
     const double update_fps = fps + 190;
     int64_t video_pts_counter = 0;
 
@@ -1442,10 +1530,13 @@ int main(int argc, char **argv) {
             }
 
             if (XCheckTypedWindowEvent(dpy, src_window_id, ConfigureNotify, &e) && e.xconfigure.window == src_window_id) {
+                while(XCheckTypedWindowEvent(dpy, src_window_id, ConfigureNotify, &e)) {}
+                window_x = e.xconfigure.x;
+                window_y = e.xconfigure.y;
                 // Window resize
-                if(e.xconfigure.width != window_width || e.xconfigure.height != window_height) {
-                    window_width = e.xconfigure.width;
-                    window_height = e.xconfigure.height;
+                if(e.xconfigure.width != (int)window_width || e.xconfigure.height != (int)window_height) {
+                    window_width = std::max(0, e.xconfigure.width);
+                    window_height = std::max(0, e.xconfigure.height);
                     window_resize_timer = glfwGetTime();
                     window_resized = true;
                 }
@@ -1533,10 +1624,56 @@ int main(int argc, char **argv) {
                 if(src_window_id) {
                     // TODO: Use a framebuffer instead. glCopyImageSubData requires
                     // opengl 4.2
+                    int source_x = 0;
+                    int source_y = 0;
+
+                    int source_width = window_pixmap.texture_width;
+                    int source_height = window_pixmap.texture_height;
+
+                    bool clamped = false;
+
+                    if(window_pixmap.composite_window) {
+                        source_x = window_x;
+                        source_y = window_y;
+
+                        int underflow_x = 0;
+                        int underflow_y = 0;
+
+                        if(source_x < 0) {
+                            underflow_x = -source_x;
+                            source_x = 0;
+                            source_width += source_x;
+                        }
+
+                        if(source_y < 0) {
+                            underflow_y = -source_y;
+                            source_y = 0;
+                            source_height += source_y;
+                        }
+
+                        const int clamped_source_width = std::max(0, window_pixmap.texture_real_width - source_x - underflow_x);
+                        const int clamped_source_height = std::max(0, window_pixmap.texture_real_height - source_y - underflow_y);
+
+                        if(clamped_source_width < source_width) {
+                            source_width = clamped_source_width;
+                            clamped = true;
+                        }
+
+                        if(clamped_source_height < source_height) {
+                            source_height = clamped_source_height;
+                            clamped = true;
+                        }
+                    }
+
+                    if(clamped) {
+                        // Requires opengl 4.4...
+                        glClearTexImage(window_pixmap.target_texture_id, 0, GL_RGB, GL_UNSIGNED_BYTE, nullptr);
+                    }
+
                     glCopyImageSubData(
-                        window_pixmap.texture_id, GL_TEXTURE_2D, 0, 0, 0, 0,
+                        window_pixmap.texture_id, GL_TEXTURE_2D, 0, source_x, source_y, 0,
                         window_pixmap.target_texture_id, GL_TEXTURE_2D, 0, 0, 0, 0,
-                        window_pixmap.texture_width, window_pixmap.texture_height, 1);
+                        source_width, source_height, 1);
                     int err = glGetError();
                     if(err != 0) {
                         static bool error_shown = false;
@@ -1583,7 +1720,7 @@ int main(int argc, char **argv) {
             }
 
             const double this_video_frame_time = clock_get_monotonic_seconds();
-            const int64_t expected_frames = (this_video_frame_time - start_time_pts) / target_fps;
+            const int64_t expected_frames = std::round((this_video_frame_time - start_time_pts) / target_fps);
 
             const int num_frames = std::max(0L, expected_frames - video_pts_counter);
             // TODO: Check if duplicate frame can be saved just by writing it with a different pts instead of sending it again
@@ -1596,7 +1733,6 @@ int main(int argc, char **argv) {
                     fprintf(stderr, "Error: avcodec_send_frame failed\n");
                 }
             }
-            prev_video_frame_time = this_video_frame_time;
             video_pts_counter += num_frames;
         }