From a3fedae32937f8965c9905e12633fa4de0c3658e Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 9 Mar 2024 16:59:09 +0100 Subject: Draw cursor in window capture --- README.md | 7 +-- TODO | 2 - build.sh | 5 +- include/capture/xcomposite.h | 4 ++ include/cursor.h | 26 +++++++++ project.conf | 1 + src/capture/nvfbc.c | 10 ++-- src/capture/xcomposite.c | 77 +++++++++++++++++-------- src/capture/xcomposite_cuda.c | 8 +++ src/capture/xcomposite_vaapi.c | 7 +++ src/cursor.c | 127 +++++++++++++++++++++++++++++++++++++++++ 11 files changed, 238 insertions(+), 36 deletions(-) create mode 100644 include/cursor.h create mode 100644 src/cursor.c diff --git a/README.md b/README.md index b921d52..447b4d3 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ If you install GPU Screen Recorder flatpak, which is the gtk gui version then yo libglvnd (which provides libgl and libegl)\ mesa\ ffmpeg (libavcodec, libavformat, libavutil, libswresample, libavfilter)\ -x11 (libx11, libxcomposite, libxrandr)\ +x11 (libx11, libxcomposite, libxrandr, xfixes)\ libpulse\ vaapi (libva, libva-mesa-driver)\ libdrm\ @@ -62,7 +62,7 @@ wayland-client libglvnd (which provides libgl and libegl)\ mesa\ ffmpeg (libavcodec, libavformat, libavutil, libswresample, libavfilter)\ -x11 (libx11, libxcomposite, libxrandr)\ +x11 (libx11, libxcomposite, libxrandr, xfixes)\ libpulse\ vaapi (libva, libva-intel-driver)\ libdrm\ @@ -71,7 +71,7 @@ wayland-client ## NVIDIA libglvnd (which provides libgl and libegl)\ ffmpeg (libavcodec, libavformat, libavutil, libswresample, libavfilter)\ -x11 (libx11, libxcomposite, libxrandr)\ +x11 (libx11, libxcomposite, libxrandr, xfixes)\ libpulse\ cuda runtime (libcuda.so.1) (libnvidia-compute)\ nvenc (libnvidia-encode)\ @@ -163,6 +163,5 @@ If you want to donate you can donate via bitcoin or monero. # TODO * Dynamically change bitrate/resolution to match desired fps. This would be helpful when streaming for example, where the encode output speed also depends on upload speed to the streaming service. -* Show cursor when recording a window. Currently the cursor is only visible when recording a monitor. * Implement opengl injection to capture texture. This fixes VRR without having to use NvFBC direct capture. * Always use direct capture with NvFBC once the capture issue in mpv fullscreen has been resolved (maybe detect if direct capture fails in nvfbc and switch to non-direct recording. NvFBC says if direct capture fails). diff --git a/TODO b/TODO index 8c0f567..f83d728 100644 --- a/TODO +++ b/TODO @@ -118,8 +118,6 @@ Use CAP_SYS_NICE in flatpak too on the main gpu screen recorder binary. It makes Show error when using compressed kms plane which isn't supported. Also do that in the gui. -Use video_codec_context->width/height instead of frame->width/height in capture. - Modify ffmpeg to accept opengl texture for nvenc encoding. Removes extra buffers and copies. When vulkan encode is added, mention minimum nvidia driver required. (550.54.14?). diff --git a/build.sh b/build.sh index f415472..b353406 100755 --- a/build.sh +++ b/build.sh @@ -20,7 +20,7 @@ build_gsr_kms_server() { } build_gsr() { - dependencies="libavcodec libavformat libavutil x11 xcomposite xrandr libpulse libswresample libavfilter libva libcap libdrm wayland-egl wayland-client" + dependencies="libavcodec libavformat libavutil x11 xcomposite xrandr xfixes libpulse libswresample libavfilter libva libcap libdrm wayland-egl wayland-client" includes="$(pkg-config --cflags $dependencies)" libs="$(pkg-config --libs $dependencies) -ldl -pthread -lm" $CC -c src/capture/capture.c $opts $includes @@ -41,10 +41,11 @@ build_gsr() { $CC -c src/color_conversion.c $opts $includes $CC -c src/utils.c $opts $includes $CC -c src/library_loader.c $opts $includes + $CC -c src/cursor.c $opts $includes $CXX -c src/sound.cpp $opts $includes $CXX -c src/main.cpp $opts $includes $CXX -o gpu-screen-recorder capture.o nvfbc.o kms_client.o egl.o cuda.o xnvctrl.o overclock.o window_texture.o shader.o \ - color_conversion.o utils.o library_loader.o xcomposite.o xcomposite_cuda.o xcomposite_vaapi.o kms_vaapi.o kms_cuda.o kms.o sound.o main.o $libs $opts + color_conversion.o utils.o library_loader.o cursor.o xcomposite.o xcomposite_cuda.o xcomposite_vaapi.o kms_vaapi.o kms_cuda.o kms.o sound.o main.o $libs $opts } build_gsr_kms_server diff --git a/include/capture/xcomposite.h b/include/capture/xcomposite.h index 222ff5b..2a9f3f7 100644 --- a/include/capture/xcomposite.h +++ b/include/capture/xcomposite.h @@ -6,6 +6,7 @@ #include "../vec2.h" #include "../color_conversion.h" #include "../window_texture.h" +#include "../cursor.h" typedef struct { gsr_egl *egl; @@ -33,6 +34,9 @@ typedef struct { WindowTexture window_texture; Atom net_active_window_atom; + + gsr_cursor cursor; + bool clear_next_frame; } gsr_capture_xcomposite; void gsr_capture_xcomposite_init(gsr_capture_xcomposite *self, const gsr_capture_xcomposite_params *params); diff --git a/include/cursor.h b/include/cursor.h new file mode 100644 index 0000000..b1ec6bd --- /dev/null +++ b/include/cursor.h @@ -0,0 +1,26 @@ +#ifndef GSR_CURSOR_H +#define GSR_CURSOR_H + +#include "egl.h" +#include "vec2.h" + +typedef struct { + gsr_egl *egl; + Display *display; + int x_fixes_event_base; + + unsigned int texture_id; + vec2i size; + vec2i hotspot; + vec2i position; + + bool cursor_image_set; +} gsr_cursor; + +int gsr_cursor_init(gsr_cursor *self, gsr_egl *egl, Display *display); +void gsr_cursor_deinit(gsr_cursor *self); + +void gsr_cursor_update(gsr_cursor *self, XEvent *xev); +void gsr_cursor_tick(gsr_cursor *self, Window relative_to); + +#endif /* GSR_CURSOR_H */ diff --git a/project.conf b/project.conf index 23092af..a046427 100644 --- a/project.conf +++ b/project.conf @@ -15,6 +15,7 @@ libavutil = ">=56.2" x11 = ">=1" xcomposite = ">=0.2" xrandr = ">=1" +xfixes = ">=2" libpulse = ">=13" libswresample = ">=3" libavfilter = ">=5" diff --git a/src/capture/nvfbc.c b/src/capture/nvfbc.c index d909e1e..a9a7cc8 100644 --- a/src/capture/nvfbc.c +++ b/src/capture/nvfbc.c @@ -397,8 +397,8 @@ static int gsr_capture_nvfbc_capture(gsr_capture *cap, AVFrame *frame) { cap_nvfbc->params.egl->glClear(0); gsr_color_conversion_draw(&cap_nvfbc->base.color_conversion, cap_nvfbc->setup_params.dwTextures[grab_params.dwTextureIndex], - (vec2i){0, 0}, (vec2i){cap_nvfbc->base.video_codec_context->width, cap_nvfbc->base.video_codec_context->height}, - (vec2i){0, 0}, (vec2i){cap_nvfbc->base.video_codec_context->width, cap_nvfbc->base.video_codec_context->height}, + (vec2i){0, 0}, (vec2i){frame->width, frame->height}, + (vec2i){0, 0}, (vec2i){frame->width, frame->height}, 0.0f, false); cap_nvfbc->params.egl->glXSwapBuffers(cap_nvfbc->params.egl->x11.dpy, cap_nvfbc->params.egl->x11.window); @@ -416,11 +416,11 @@ static int gsr_capture_nvfbc_capture(gsr_capture *cap, AVFrame *frame) { memcpy_struct.dstMemoryType = CU_MEMORYTYPE_DEVICE; memcpy_struct.srcArray = cap_nvfbc->mapped_arrays[i]; - memcpy_struct.srcPitch = cap_nvfbc->base.video_codec_context->width / div[i]; + memcpy_struct.srcPitch = frame->width / div[i]; memcpy_struct.dstDevice = (CUdeviceptr)frame->data[i]; memcpy_struct.dstPitch = frame->linesize[i]; - memcpy_struct.WidthInBytes = cap_nvfbc->base.video_codec_context->width * (cap_nvfbc->params.hdr ? 2 : 1); - memcpy_struct.Height = cap_nvfbc->base.video_codec_context->height / div[i]; + memcpy_struct.WidthInBytes = frame->width * (cap_nvfbc->params.hdr ? 2 : 1); + memcpy_struct.Height = frame->height / div[i]; // TODO: Remove this copy if possible cap_nvfbc->cuda.cuMemcpy2DAsync_v2(&memcpy_struct, cap_nvfbc->cuda_stream); } diff --git a/src/capture/xcomposite.c b/src/capture/xcomposite.c index dbe841d..73da951 100644 --- a/src/capture/xcomposite.c +++ b/src/capture/xcomposite.c @@ -57,8 +57,8 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v /* TODO: Do these in tick, and allow error if follow_focused */ XWindowAttributes attr; - if(!XGetWindowAttributes(self->params.egl->x11.dpy, self->params.window, &attr) && !self->params.follow_focused) { - fprintf(stderr, "gsr error: gsr_capture_xcomposite_start failed: invalid window id: %lu\n", self->params.window); + if(!XGetWindowAttributes(self->params.egl->x11.dpy, self->window, &attr) && !self->params.follow_focused) { + fprintf(stderr, "gsr error: gsr_capture_xcomposite_start failed: invalid window id: %lu\n", self->window); return -1; } @@ -69,7 +69,7 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v XSelectInput(self->params.egl->x11.dpy, DefaultRootWindow(self->params.egl->x11.dpy), PropertyChangeMask); // TODO: Get select and add these on top of it and then restore at the end. Also do the same in other xcomposite - XSelectInput(self->params.egl->x11.dpy, self->params.window, StructureNotifyMask | ExposureMask); + XSelectInput(self->params.egl->x11.dpy, self->window, StructureNotifyMask | ExposureMask); if(!self->params.egl->eglExportDMABUFImageQueryMESA) { fprintf(stderr, "gsr error: gsr_capture_xcomposite_start: could not find eglExportDMABUFImageQueryMESA\n"); @@ -83,8 +83,13 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v /* Disable vsync */ self->params.egl->eglSwapInterval(self->params.egl->egl_display, 0); - if(window_texture_init(&self->window_texture, self->params.egl->x11.dpy, self->params.window, self->params.egl) != 0 && !self->params.follow_focused) { - fprintf(stderr, "gsr error: gsr_capture_xcomposite_start: failed to get window texture for window %ld\n", self->params.window); + if(window_texture_init(&self->window_texture, self->params.egl->x11.dpy, self->window, self->params.egl) != 0 && !self->params.follow_focused) { + fprintf(stderr, "gsr error: gsr_capture_xcomposite_start: failed to get window texture for window %ld\n", self->window); + return -1; + } + + if(gsr_cursor_init(&self->cursor, self->params.egl, self->params.egl->x11.dpy) != 0) { + gsr_capture_xcomposite_stop(self, video_codec_context); return -1; } @@ -115,14 +120,9 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v } void gsr_capture_xcomposite_stop(gsr_capture_xcomposite *self, AVCodecContext *video_codec_context) { + (void)video_codec_context; window_texture_deinit(&self->window_texture); - - if(video_codec_context->hw_device_ctx) - av_buffer_unref(&video_codec_context->hw_device_ctx); - if(video_codec_context->hw_frames_ctx) - av_buffer_unref(&video_codec_context->hw_frames_ctx); - - gsr_capture_base_stop(&self->base, self->params.egl); + gsr_cursor_deinit(&self->cursor); } void gsr_capture_xcomposite_tick(gsr_capture_xcomposite *self, AVCodecContext *video_codec_context) { @@ -168,6 +168,8 @@ void gsr_capture_xcomposite_tick(gsr_capture_xcomposite *self, AVCodecContext *v break; } } + + gsr_cursor_update(&self->cursor, &self->xev); } if(self->params.follow_focused && !self->follow_focused_initialized) { @@ -194,17 +196,6 @@ void gsr_capture_xcomposite_tick(gsr_capture_xcomposite *self, AVCodecContext *v window_texture_deinit(&self->window_texture); window_texture_init(&self->window_texture, self->params.egl->x11.dpy, self->window, self->params.egl); // TODO: Do not do the below window_texture_on_resize after this - - self->texture_size.x = 0; - self->texture_size.y = 0; - - self->params.egl->glBindTexture(GL_TEXTURE_2D, window_texture_get_opengl_texture_id(&self->window_texture)); - self->params.egl->glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &self->texture_size.x); - self->params.egl->glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &self->texture_size.y); - self->params.egl->glBindTexture(GL_TEXTURE_2D, 0); - - self->texture_size.x = min_int(video_codec_context->width, max_int(2, even_number_ceil(self->texture_size.x))); - self->texture_size.y = min_int(video_codec_context->height, max_int(2, even_number_ceil(self->texture_size.y))); } } @@ -252,11 +243,51 @@ int gsr_capture_xcomposite_capture(gsr_capture_xcomposite *self, AVFrame *frame) const int target_x = max_int(0, frame->width / 2 - self->texture_size.x / 2); const int target_y = max_int(0, frame->height / 2 - self->texture_size.y / 2); + // TODO: Can we do this a better way than to call it every capture? + gsr_cursor_tick(&self->cursor, self->window); + + const vec2i cursor_pos = { + target_x + self->cursor.position.x - self->cursor.hotspot.x, + target_y + self->cursor.position.y - self->cursor.hotspot.y + }; + + const bool cursor_completely_inside_window = + cursor_pos.x >= target_x && + cursor_pos.x <= target_x + self->texture_size.x && + cursor_pos.y >= target_y && + cursor_pos.y <= target_y + self->texture_size.x; + + const bool cursor_inside_window = + cursor_pos.x + self->cursor.size.x >= target_x && + cursor_pos.x <= target_x + self->texture_size.x && + cursor_pos.y + self->cursor.size.y >= target_y && + cursor_pos.y <= target_y + self->texture_size.x; + + if(self->clear_next_frame) { + self->clear_next_frame = false; + gsr_color_conversion_clear(&self->base.color_conversion); + } + + /* + We dont draw the cursor if it's outside the window but if it's partially inside the window then the cursor area that is outside the window + will not get overdrawn the next frame causing a cursor trail to be visible since we dont clear the background. + To fix this we detect if the cursor is partially inside the window and clear the background only in that case. + */ + if(!cursor_completely_inside_window && cursor_inside_window) + self->clear_next_frame = true; + gsr_color_conversion_draw(&self->base.color_conversion, window_texture_get_opengl_texture_id(&self->window_texture), (vec2i){target_x, target_y}, self->texture_size, (vec2i){0, 0}, self->texture_size, 0.0f, false); + if(cursor_inside_window) { + gsr_color_conversion_draw(&self->base.color_conversion, self->cursor.texture_id, + cursor_pos, self->cursor.size, + (vec2i){0, 0}, self->cursor.size, + 0.0f, false); + } + self->params.egl->eglSwapBuffers(self->params.egl->egl_display, self->params.egl->egl_surface); //self->params.egl->glFlush(); //self->params.egl->glFinish(); diff --git a/src/capture/xcomposite_cuda.c b/src/capture/xcomposite_cuda.c index 12cea8c..0a324bd 100644 --- a/src/capture/xcomposite_cuda.c +++ b/src/capture/xcomposite_cuda.c @@ -3,6 +3,7 @@ #include #include #include +#include typedef struct { gsr_capture_xcomposite xcomposite; @@ -70,6 +71,13 @@ static void gsr_capture_xcomposite_unload_cuda_graphics(gsr_capture_xcomposite_c static void gsr_capture_xcomposite_cuda_stop(gsr_capture *cap, AVCodecContext *video_codec_context) { gsr_capture_xcomposite_cuda *cap_xcomp = cap->priv; + + if(video_codec_context->hw_device_ctx) + av_buffer_unref(&video_codec_context->hw_device_ctx); + if(video_codec_context->hw_frames_ctx) + av_buffer_unref(&video_codec_context->hw_frames_ctx); + + gsr_capture_base_stop(&cap_xcomp->xcomposite.base, cap_xcomp->xcomposite.params.egl); gsr_capture_xcomposite_stop(&cap_xcomp->xcomposite, video_codec_context); gsr_capture_xcomposite_unload_cuda_graphics(cap_xcomp); gsr_cuda_unload(&cap_xcomp->cuda); diff --git a/src/capture/xcomposite_vaapi.c b/src/capture/xcomposite_vaapi.c index 570901b..a19956d 100644 --- a/src/capture/xcomposite_vaapi.c +++ b/src/capture/xcomposite_vaapi.c @@ -5,6 +5,7 @@ #include #include #include +#include typedef struct { gsr_capture_xcomposite xcomposite; @@ -62,6 +63,12 @@ static void gsr_capture_xcomposite_vaapi_stop(gsr_capture *cap, AVCodecContext * } } + if(video_codec_context->hw_device_ctx) + av_buffer_unref(&video_codec_context->hw_device_ctx); + if(video_codec_context->hw_frames_ctx) + av_buffer_unref(&video_codec_context->hw_frames_ctx); + + gsr_capture_base_stop(&cap_xcomp->xcomposite.base, cap_xcomp->xcomposite.params.egl); gsr_capture_xcomposite_stop(&cap_xcomp->xcomposite, video_codec_context); } diff --git a/src/cursor.c b/src/cursor.c new file mode 100644 index 0000000..737c33b --- /dev/null +++ b/src/cursor.c @@ -0,0 +1,127 @@ +#include "../include/cursor.h" + +#include +#include +#include +#include + +#include + +static bool gsr_cursor_set_from_x11_cursor_image(gsr_cursor *self, XFixesCursorImage *x11_cursor_image) { + uint8_t *cursor_data = NULL; + uint8_t *out = NULL; + + if(!x11_cursor_image) + goto err; + + if(!x11_cursor_image->pixels) + goto err; + + self->hotspot.x = x11_cursor_image->xhot; + self->hotspot.y = x11_cursor_image->yhot; + self->egl->glBindTexture(GL_TEXTURE_2D, self->texture_id); + + self->size.x = x11_cursor_image->width; + self->size.y = x11_cursor_image->height; + const unsigned long *pixels = x11_cursor_image->pixels; + cursor_data = malloc(self->size.x * self->size.y * 4); + if(!cursor_data) + goto err; + out = cursor_data; + /* Un-premultiply alpha */ + for(int y = 0; y < self->size.y; ++y) { + for(int x = 0; x < self->size.x; ++x) { + uint32_t pixel = *pixels++; + uint8_t *in = (uint8_t*)&pixel; + uint8_t alpha = in[3]; + if(alpha == 0) + alpha = 1; + + *out++ = (unsigned)*in++ * 255/alpha; + *out++ = (unsigned)*in++ * 255/alpha; + *out++ = (unsigned)*in++ * 255/alpha; + *out++ = *in++; + } + } + + self->egl->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, self->size.x, self->size.y, 0, GL_RGBA, GL_UNSIGNED_BYTE, cursor_data); + free(cursor_data); + + self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + + self->egl->glBindTexture(GL_TEXTURE_2D, 0); + XFree(x11_cursor_image); + return true; + + err: + self->egl->glBindTexture(GL_TEXTURE_2D, 0); + if(x11_cursor_image) + XFree(x11_cursor_image); + return false; +} + +int gsr_cursor_init(gsr_cursor *self, gsr_egl *egl, Display *display) { + int x_fixes_error_base = 0; + + assert(egl); + assert(display); + memset(self, 0, sizeof(*self)); + self->egl = egl; + self->display = display; + + self->x_fixes_event_base = 0; + if(!XFixesQueryExtension(self->display, &self->x_fixes_event_base, &x_fixes_error_base)) { + fprintf(stderr, "gsr error: gsr_cursor_init: your X11 server is missing the XFixes extension\n"); + gsr_cursor_deinit(self); + return -1; + } + + self->egl->glGenTextures(1, &self->texture_id); + + XFixesSelectCursorInput(self->display, DefaultRootWindow(self->display), XFixesDisplayCursorNotifyMask); + gsr_cursor_set_from_x11_cursor_image(self, XFixesGetCursorImage(self->display)); + self->cursor_image_set = true; + + return 0; +} + +void gsr_cursor_deinit(gsr_cursor *self) { + if(!self->egl) + return; + + if(self->texture_id) { + self->egl->glDeleteTextures(1, &self->texture_id); + self->texture_id = 0; + } + + XFixesSelectCursorInput(self->display, DefaultRootWindow(self->display), 0); + + self->display = NULL; + self->egl = NULL; +} + +void gsr_cursor_update(gsr_cursor *self, XEvent *xev) { + if(xev->type == self->x_fixes_event_base + XFixesCursorNotify) { + XFixesCursorNotifyEvent *cursor_notify_event = (XFixesCursorNotifyEvent*)xev; + if(cursor_notify_event->subtype == XFixesDisplayCursorNotify && cursor_notify_event->window == DefaultRootWindow(self->display)) { + self->cursor_image_set = true; + gsr_cursor_set_from_x11_cursor_image(self, XFixesGetCursorImage(self->display)); + } + } + + if(!self->cursor_image_set) { + self->cursor_image_set = true; + gsr_cursor_set_from_x11_cursor_image(self, XFixesGetCursorImage(self->display)); + } +} + +void gsr_cursor_tick(gsr_cursor *self, Window relative_to) { + /* TODO: Use XInput2 instead. However that doesn't work when the pointer is grabbed. Maybe check for focused window change and XSelectInput PointerMask */ + Window dummy_window; + int dummy_i; + unsigned int dummy_u; + XQueryPointer(self->display, relative_to, &dummy_window, &dummy_window, &dummy_i, &dummy_i, &self->position.x, &self->position.y, &dummy_u); +} -- cgit v1.2.3