From f3c32a880a426d8365f01ba7d61d81b75c04d9bc Mon Sep 17 00:00:00 2001
From: dec05eba <dec05eba@protonmail.com>
Date: Sun, 10 Mar 2024 20:59:17 +0100
Subject: Fix cursor capture in nvidia wayland, hdr, clear background
 immediately in window capture

---
 src/capture/capture.c         | 5 +++--
 src/capture/kms.c             | 2 +-
 src/capture/kms_cuda.c        | 2 +-
 src/capture/nvfbc.c           | 2 +-
 src/capture/xcomposite.c      | 1 +
 src/capture/xcomposite_cuda.c | 2 +-
 6 files changed, 8 insertions(+), 6 deletions(-)

(limited to 'src/capture')

diff --git a/src/capture/capture.c b/src/capture/capture.c
index 0917356..a1d2913 100644
--- a/src/capture/capture.c
+++ b/src/capture/capture.c
@@ -276,6 +276,7 @@ bool gsr_capture_base_setup_cuda_textures(gsr_capture_base *self, AVFrame *frame
     color_conversion_params.destination_textures[0] = self->target_textures[0];
     color_conversion_params.destination_textures[1] = self->target_textures[1];
     color_conversion_params.num_destination_textures = 2;
+    color_conversion_params.load_external_image_shader = true;
 
     if(gsr_color_conversion_init(&self->color_conversion, &color_conversion_params) != 0) {
         fprintf(stderr, "gsr error: gsr_capture_kms_setup_cuda_textures: failed to create color conversion\n");
@@ -357,7 +358,7 @@ bool drm_create_codec_context(const char *card_path, AVCodecContext *video_codec
     return true;
 }
 
-bool cuda_create_codec_context(CUcontext cu_ctx, AVCodecContext *video_codec_context, int width, int height, CUstream *cuda_stream) {
+bool cuda_create_codec_context(CUcontext cu_ctx, AVCodecContext *video_codec_context, int width, int height, bool hdr, CUstream *cuda_stream) {
     AVBufferRef *device_ctx = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_CUDA);
     if(!device_ctx) {
         fprintf(stderr, "gsr error: cuda_create_codec_context failed: failed to create hardware device context\n");
@@ -383,7 +384,7 @@ bool cuda_create_codec_context(CUcontext cu_ctx, AVCodecContext *video_codec_con
     AVHWFramesContext *hw_frame_context = (AVHWFramesContext*)frame_context->data;
     hw_frame_context->width = width;
     hw_frame_context->height = height;
-    hw_frame_context->sw_format = AV_PIX_FMT_NV12;
+    hw_frame_context->sw_format = hdr ? AV_PIX_FMT_P010LE : AV_PIX_FMT_NV12;
     hw_frame_context->format = video_codec_context->pix_fmt;
     hw_frame_context->device_ref = device_ctx;
     hw_frame_context->device_ctx = (AVHWDeviceContext*)device_ctx->data;
diff --git a/src/capture/kms.c b/src/capture/kms.c
index 292bd13..0b75b1f 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -349,7 +349,7 @@ bool gsr_capture_kms_capture(gsr_capture_kms *self, AVFrame *frame, bool hdr, bo
         gsr_color_conversion_draw(&self->base.color_conversion, self->base.cursor_texture,
             cursor_pos, cursor_size,
             (vec2i){0, 0}, cursor_size,
-            texture_rotation, false);
+            texture_rotation, cursor_texture_is_external);
     }
 
     self->base.egl->eglSwapBuffers(self->base.egl->egl_display, self->base.egl->egl_surface);
diff --git a/src/capture/kms_cuda.c b/src/capture/kms_cuda.c
index 7d6d9b4..c206fde 100644
--- a/src/capture/kms_cuda.c
+++ b/src/capture/kms_cuda.c
@@ -38,7 +38,7 @@ static int gsr_capture_kms_cuda_start(gsr_capture *cap, AVCodecContext *video_co
         return -1;
     }
 
-    if(!cuda_create_codec_context(cap_kms->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, &cap_kms->cuda_stream)) {
+    if(!cuda_create_codec_context(cap_kms->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, cap_kms->params.hdr, &cap_kms->cuda_stream)) {
         gsr_capture_kms_cuda_stop(cap, video_codec_context);
         return -1;
     }
diff --git a/src/capture/nvfbc.c b/src/capture/nvfbc.c
index 6317e07..7ba0438 100644
--- a/src/capture/nvfbc.c
+++ b/src/capture/nvfbc.c
@@ -310,7 +310,7 @@ static int gsr_capture_nvfbc_start(gsr_capture *cap, AVCodecContext *video_codec
     frame->width = video_codec_context->width;
     frame->height = video_codec_context->height;
 
-    if(!cuda_create_codec_context(cap_nvfbc->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, &cap_nvfbc->cuda_stream))
+    if(!cuda_create_codec_context(cap_nvfbc->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, false, &cap_nvfbc->cuda_stream))
         goto error_cleanup;
 
     gsr_cuda_context cuda_context = {
diff --git a/src/capture/xcomposite.c b/src/capture/xcomposite.c
index eeed2ab..f1d620c 100644
--- a/src/capture/xcomposite.c
+++ b/src/capture/xcomposite.c
@@ -117,6 +117,7 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v
     frame->height = video_codec_context->height;
 
     self->window_resize_timer = clock_get_monotonic_seconds();
+    self->clear_next_frame = true;
     return 0;
 }
 
diff --git a/src/capture/xcomposite_cuda.c b/src/capture/xcomposite_cuda.c
index 27d3864..e8beb4e 100644
--- a/src/capture/xcomposite_cuda.c
+++ b/src/capture/xcomposite_cuda.c
@@ -32,7 +32,7 @@ static int gsr_capture_xcomposite_cuda_start(gsr_capture *cap, AVCodecContext *v
         return -1;
     }
 
-    if(!cuda_create_codec_context(cap_xcomp->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, &cap_xcomp->cuda_stream)) {
+    if(!cuda_create_codec_context(cap_xcomp->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, false, &cap_xcomp->cuda_stream)) {
         gsr_capture_xcomposite_cuda_stop(cap, video_codec_context);
         return -1;
     }
-- 
cgit v1.2.3