Fix cursor capture in nvidia wayland, hdr, clear background immediately in window capture

9 files changed, 99 insertions, 149 deletions

diff --git a/include/capture/capture.h b/include/capture/capture.h
index 148da16..2eb8e42 100644
--- a/include/capture/capture.h
+++ b/include/capture/capture.h
@@ -63,6 +63,6 @@ bool gsr_capture_base_setup_cuda_textures(gsr_capture_base *self, AVFrame *frame
 void gsr_capture_base_stop(gsr_capture_base *self);
 
 bool drm_create_codec_context(const char *card_path, AVCodecContext *video_codec_context, int width, int height, bool hdr, VADisplay *va_dpy);
-bool cuda_create_codec_context(CUcontext cu_ctx, AVCodecContext *video_codec_context, int width, int height, CUstream *cuda_stream);
+bool cuda_create_codec_context(CUcontext cu_ctx, AVCodecContext *video_codec_context, int width, int height, bool hdr, CUstream *cuda_stream);
 
 #endif /* GSR_CAPTURE_CAPTURE_H */
diff --git a/include/color_conversion.h b/include/color_conversion.h
index 1848715..d05df6a 100644
--- a/include/color_conversion.h
+++ b/include/color_conversion.h
@@ -16,7 +16,6 @@ typedef enum {
 } gsr_source_color;
 
 typedef enum {
-    GSR_DESTINATION_COLOR_BGR, // TODO: remove
     GSR_DESTINATION_COLOR_NV12, /* YUV420, BT709, 8-bit */
     GSR_DESTINATION_COLOR_P010  /* YUV420, BT2020, 10-bit */
 } gsr_destination_color;
@@ -36,12 +35,13 @@ typedef struct {
     int num_destination_textures;
 
     gsr_color_range color_range;
+    bool load_external_image_shader;
 } gsr_color_conversion_params;
 
 typedef struct {
     gsr_color_conversion_params params;
-    gsr_color_uniforms uniforms[2];
-    gsr_shader shaders[2];
+    gsr_color_uniforms uniforms[4];
+    gsr_shader shaders[4];
 
     unsigned int framebuffers[2];
 
diff --git a/src/capture/capture.c b/src/capture/capture.c
index 0917356..a1d2913 100644
--- a/src/capture/capture.c
+++ b/src/capture/capture.c
@@ -276,6 +276,7 @@ bool gsr_capture_base_setup_cuda_textures(gsr_capture_base *self, AVFrame *frame
     color_conversion_params.destination_textures[0] = self->target_textures[0];
     color_conversion_params.destination_textures[1] = self->target_textures[1];
     color_conversion_params.num_destination_textures = 2;
+    color_conversion_params.load_external_image_shader = true;
 
     if(gsr_color_conversion_init(&self->color_conversion, &color_conversion_params) != 0) {
         fprintf(stderr, "gsr error: gsr_capture_kms_setup_cuda_textures: failed to create color conversion\n");
@@ -357,7 +358,7 @@ bool drm_create_codec_context(const char *card_path, AVCodecContext *video_codec
     return true;
 }
 
-bool cuda_create_codec_context(CUcontext cu_ctx, AVCodecContext *video_codec_context, int width, int height, CUstream *cuda_stream) {
+bool cuda_create_codec_context(CUcontext cu_ctx, AVCodecContext *video_codec_context, int width, int height, bool hdr, CUstream *cuda_stream) {
     AVBufferRef *device_ctx = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_CUDA);
     if(!device_ctx) {
         fprintf(stderr, "gsr error: cuda_create_codec_context failed: failed to create hardware device context\n");
@@ -383,7 +384,7 @@ bool cuda_create_codec_context(CUcontext cu_ctx, AVCodecContext *video_codec_con
     AVHWFramesContext *hw_frame_context = (AVHWFramesContext*)frame_context->data;
     hw_frame_context->width = width;
     hw_frame_context->height = height;
-    hw_frame_context->sw_format = AV_PIX_FMT_NV12;
+    hw_frame_context->sw_format = hdr ? AV_PIX_FMT_P010LE : AV_PIX_FMT_NV12;
     hw_frame_context->format = video_codec_context->pix_fmt;
     hw_frame_context->device_ref = device_ctx;
     hw_frame_context->device_ctx = (AVHWDeviceContext*)device_ctx->data;
diff --git a/src/capture/kms.c b/src/capture/kms.c
index 292bd13..0b75b1f 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -349,7 +349,7 @@ bool gsr_capture_kms_capture(gsr_capture_kms *self, AVFrame *frame, bool hdr, bo
         gsr_color_conversion_draw(&self->base.color_conversion, self->base.cursor_texture,
             cursor_pos, cursor_size,
             (vec2i){0, 0}, cursor_size,
-            texture_rotation, false);
+            texture_rotation, cursor_texture_is_external);
     }
 
     self->base.egl->eglSwapBuffers(self->base.egl->egl_display, self->base.egl->egl_surface);
diff --git a/src/capture/kms_cuda.c b/src/capture/kms_cuda.c
index 7d6d9b4..c206fde 100644
--- a/src/capture/kms_cuda.c
+++ b/src/capture/kms_cuda.c
@@ -38,7 +38,7 @@ static int gsr_capture_kms_cuda_start(gsr_capture *cap, AVCodecContext *video_co
         return -1;
     }
 
-    if(!cuda_create_codec_context(cap_kms->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, &cap_kms->cuda_stream)) {
+    if(!cuda_create_codec_context(cap_kms->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, cap_kms->params.hdr, &cap_kms->cuda_stream)) {
         gsr_capture_kms_cuda_stop(cap, video_codec_context);
         return -1;
     }
diff --git a/src/capture/nvfbc.c b/src/capture/nvfbc.c
index 6317e07..7ba0438 100644
--- a/src/capture/nvfbc.c
+++ b/src/capture/nvfbc.c
@@ -310,7 +310,7 @@ static int gsr_capture_nvfbc_start(gsr_capture *cap, AVCodecContext *video_codec
     frame->width = video_codec_context->width;
     frame->height = video_codec_context->height;
 
-    if(!cuda_create_codec_context(cap_nvfbc->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, &cap_nvfbc->cuda_stream))
+    if(!cuda_create_codec_context(cap_nvfbc->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, false, &cap_nvfbc->cuda_stream))
         goto error_cleanup;
 
     gsr_cuda_context cuda_context = {
diff --git a/src/capture/xcomposite.c b/src/capture/xcomposite.c
index eeed2ab..f1d620c 100644
--- a/src/capture/xcomposite.c
+++ b/src/capture/xcomposite.c
@@ -117,6 +117,7 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v
     frame->height = video_codec_context->height;
 
     self->window_resize_timer = clock_get_monotonic_seconds();
+    self->clear_next_frame = true;
     return 0;
 }
 
diff --git a/src/capture/xcomposite_cuda.c b/src/capture/xcomposite_cuda.c
index 27d3864..e8beb4e 100644
--- a/src/capture/xcomposite_cuda.c
+++ b/src/capture/xcomposite_cuda.c
@@ -32,7 +32,7 @@ static int gsr_capture_xcomposite_cuda_start(gsr_capture *cap, AVCodecContext *v
         return -1;
     }
 
-    if(!cuda_create_codec_context(cap_xcomp->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, &cap_xcomp->cuda_stream)) {
+    if(!cuda_create_codec_context(cap_xcomp->cuda.cu_ctx, video_codec_context, video_codec_context->width, video_codec_context->height, false, &cap_xcomp->cuda_stream)) {
         gsr_capture_xcomposite_cuda_stop(cap, video_codec_context);
         return -1;
     }
diff --git a/src/color_conversion.c b/src/color_conversion.c
index dad58ff..d837413 100644
--- a/src/color_conversion.c
+++ b/src/color_conversion.c
@@ -7,7 +7,7 @@
 
 /* TODO: highp instead of mediump? */
 
-#define MAX_SHADERS 2
+#define MAX_SHADERS 4
 #define MAX_FRAMEBUFFERS 2
 
 static float abs_f(float v) {
@@ -75,83 +75,7 @@ static const char* color_format_range_get_transform_matrix(gsr_destination_color
     return NULL;
 }
 
-static int load_shader_bgr(gsr_shader *shader, gsr_egl *egl, gsr_color_uniforms *uniforms) {
-    char vertex_shader[2048];
-    snprintf(vertex_shader, sizeof(vertex_shader),
-        "#version 300 es                                   \n"
-        "in vec2 pos;                                      \n"
-        "in vec2 texcoords;                                \n"
-        "out vec2 texcoords_out;                           \n"
-        "uniform vec2 offset;                              \n"
-        "uniform float rotation;                           \n"
-        ROTATE_Z
-        "void main()                                       \n"
-        "{                                                 \n"
-        "  texcoords_out = (vec4(texcoords.x - 0.5, texcoords.y - 0.5, 0.0, 0.0) * rotate_z(rotation)).xy + vec2(0.5, 0.5);  \n"
-        "  gl_Position = vec4(offset.x, offset.y, 0.0, 0.0) + vec4(pos.x, pos.y, 0.0, 1.0);    \n"
-        "}                                                 \n");
-
-    char fragment_shader[] =
-        "#version 300 es                                                                 \n"
-        "precision mediump float;                                                        \n"
-        "in vec2 texcoords_out;                                                          \n"
-        "uniform sampler2D tex1;                                                         \n"
-        "out vec4 FragColor;                                                             \n"
-        "void main()                                                                     \n"
-        "{                                                                               \n"
-        "  FragColor = texture(tex1, texcoords_out).bgra;                                 \n"
-        "}                                                                               \n";
-
-    if(gsr_shader_init(shader, egl, vertex_shader, fragment_shader) != 0)
-        return -1;
-
-    gsr_shader_bind_attribute_location(shader, "pos", 0);
-    gsr_shader_bind_attribute_location(shader, "texcoords", 1);
-    uniforms->offset = egl->glGetUniformLocation(shader->program_id, "offset");
-    uniforms->rotation = egl->glGetUniformLocation(shader->program_id, "rotation");
-    return 0;
-}
-
-static int load_shader_bgr_external_texture(gsr_shader *shader, gsr_egl *egl, gsr_color_uniforms *uniforms) {
-    char vertex_shader[2048];
-    snprintf(vertex_shader, sizeof(vertex_shader),
-        "#version 300 es                                   \n"
-        "in vec2 pos;                                      \n"
-        "in vec2 texcoords;                                \n"
-        "out vec2 texcoords_out;                           \n"
-        "uniform vec2 offset;                              \n"
-        "uniform float rotation;                           \n"
-        ROTATE_Z
-        "void main()                                       \n"
-        "{                                                 \n"
-        "  texcoords_out = (vec4(texcoords.x - 0.5, texcoords.y - 0.5, 0.0, 0.0) * rotate_z(rotation)).xy + vec2(0.5, 0.5);  \n"
-        "  gl_Position = vec4(offset.x, offset.y, 0.0, 0.0) + vec4(pos.x, pos.y, 0.0, 1.0);    \n"
-        "}                                                 \n");
-
-    char fragment_shader[] =
-        "#version 300 es                                                                 \n"
-        "#extension GL_OES_EGL_image_external : enable                                   \n"
-        "#extension GL_OES_EGL_image_external_essl3 : require                            \n"
-        "precision mediump float;                                                        \n"
-        "in vec2 texcoords_out;                                                          \n"
-        "uniform samplerExternalOES tex1;                                                \n"
-        "out vec4 FragColor;                                                             \n"
-        "void main()                                                                     \n"
-        "{                                                                               \n"
-        "  FragColor = texture(tex1, texcoords_out).bgra;                                \n"
-        "}                                                                               \n";
-
-    if(gsr_shader_init(shader, egl, vertex_shader, fragment_shader) != 0)
-        return -1;
-
-    gsr_shader_bind_attribute_location(shader, "pos", 0);
-    gsr_shader_bind_attribute_location(shader, "texcoords", 1);
-    uniforms->offset = egl->glGetUniformLocation(shader->program_id, "offset");
-    uniforms->rotation = egl->glGetUniformLocation(shader->program_id, "rotation");
-    return 0;
-}
-
-static int load_shader_y(gsr_shader *shader, gsr_egl *egl, gsr_color_uniforms *uniforms, gsr_destination_color color_format, gsr_color_range color_range) {
+static int load_shader_y(gsr_shader *shader, gsr_egl *egl, gsr_color_uniforms *uniforms, gsr_destination_color color_format, gsr_color_range color_range, bool external_texture) {
     const char *color_transform_matrix = color_format_range_get_transform_matrix(color_format, color_range);
 
     char vertex_shader[2048];
@@ -170,19 +94,37 @@ static int load_shader_y(gsr_shader *shader, gsr_egl *egl, gsr_color_uniforms *u
         "}                                                 \n");
 
     char fragment_shader[2048];
-    snprintf(fragment_shader, sizeof(fragment_shader),
-        "#version 300 es                                                                 \n"
-        "precision mediump float;                                                        \n"
-        "in vec2 texcoords_out;                                                          \n"
-        "uniform sampler2D tex1;                                                         \n"
-        "out vec4 FragColor;                                                             \n"
-        "%s"
-        "void main()                                                                     \n"
-        "{                                                                               \n"
-        "  vec4 pixel = texture(tex1, texcoords_out);                                    \n"
-        "  FragColor.x = (RGBtoYUV * vec4(pixel.rgb, 1.0)).x;                            \n"
-        "  FragColor.w = pixel.a;                                                        \n"
-        "}                                                                               \n", color_transform_matrix);
+    if(external_texture) {
+        snprintf(fragment_shader, sizeof(fragment_shader),
+            "#version 300 es                                                                 \n"
+            "#extension GL_OES_EGL_image_external : enable                                   \n"
+            "#extension GL_OES_EGL_image_external_essl3 : require                            \n"
+            "precision mediump float;                                                        \n"
+            "in vec2 texcoords_out;                                                          \n"
+            "uniform samplerExternalOES tex1;                                                \n"
+            "out vec4 FragColor;                                                             \n"
+            "%s"
+            "void main()                                                                     \n"
+            "{                                                                               \n"
+            "  vec4 pixel = texture(tex1, texcoords_out);                                    \n"
+            "  FragColor.x = (RGBtoYUV * vec4(pixel.rgb, 1.0)).x;                            \n"
+            "  FragColor.w = pixel.a;                                                        \n"
+            "}                                                                               \n", color_transform_matrix);
+    } else {
+        snprintf(fragment_shader, sizeof(fragment_shader),
+            "#version 300 es                                                                 \n"
+            "precision mediump float;                                                        \n"
+            "in vec2 texcoords_out;                                                          \n"
+            "uniform sampler2D tex1;                                                         \n"
+            "out vec4 FragColor;                                                             \n"
+            "%s"
+            "void main()                                                                     \n"
+            "{                                                                               \n"
+            "  vec4 pixel = texture(tex1, texcoords_out);                                    \n"
+            "  FragColor.x = (RGBtoYUV * vec4(pixel.rgb, 1.0)).x;                            \n"
+            "  FragColor.w = pixel.a;                                                        \n"
+            "}                                                                               \n", color_transform_matrix);
+    }
 
     if(gsr_shader_init(shader, egl, vertex_shader, fragment_shader) != 0)
         return -1;
@@ -194,7 +136,7 @@ static int load_shader_y(gsr_shader *shader, gsr_egl *egl, gsr_color_uniforms *u
     return 0;
 }
 
-static unsigned int load_shader_uv(gsr_shader *shader, gsr_egl *egl, gsr_color_uniforms *uniforms, gsr_destination_color color_format, gsr_color_range color_range) {
+static unsigned int load_shader_uv(gsr_shader *shader, gsr_egl *egl, gsr_color_uniforms *uniforms, gsr_destination_color color_format, gsr_color_range color_range, bool external_texture) {
     const char *color_transform_matrix = color_format_range_get_transform_matrix(color_format, color_range);
 
     char vertex_shader[2048];
@@ -213,19 +155,37 @@ static unsigned int load_shader_uv(gsr_shader *shader, gsr_egl *egl, gsr_color_u
         "}                                               \n");
 
     char fragment_shader[2048];
-    snprintf(fragment_shader, sizeof(fragment_shader),
-        "#version 300 es                                                                       \n"
-        "precision mediump float;                                                              \n"
-        "in vec2 texcoords_out;                                                                \n"
-        "uniform sampler2D tex1;                                                               \n"
-        "out vec4 FragColor;                                                                   \n"
-        "%s"
-        "void main()                                                                           \n"
-        "{                                                                                     \n"
-        "  vec4 pixel = texture(tex1, texcoords_out);                                          \n"
-        "  FragColor.xy = (RGBtoYUV * vec4(pixel.rgb, 1.0)).yz;                                \n"
-        "  FragColor.w = pixel.a;                                                              \n"
-        "}                                                                                     \n", color_transform_matrix);
+    if(external_texture) {
+        snprintf(fragment_shader, sizeof(fragment_shader),
+            "#version 300 es                                                                       \n"
+            "#extension GL_OES_EGL_image_external : enable                                         \n"
+            "#extension GL_OES_EGL_image_external_essl3 : require                                  \n"
+            "precision mediump float;                                                              \n"
+            "in vec2 texcoords_out;                                                                \n"
+            "uniform samplerExternalOES tex1;                                                      \n"
+            "out vec4 FragColor;                                                                   \n"
+            "%s"
+            "void main()                                                                           \n"
+            "{                                                                                     \n"
+            "  vec4 pixel = texture(tex1, texcoords_out);                                          \n"
+            "  FragColor.xy = (RGBtoYUV * vec4(pixel.rgb, 1.0)).yz;                                \n"
+            "  FragColor.w = pixel.a;                                                              \n"
+            "}                                                                                     \n", color_transform_matrix);
+    } else {
+        snprintf(fragment_shader, sizeof(fragment_shader),
+            "#version 300 es                                                                       \n"
+            "precision mediump float;                                                              \n"
+            "in vec2 texcoords_out;                                                                \n"
+            "uniform sampler2D tex1;                                                               \n"
+            "out vec4 FragColor;                                                                   \n"
+            "%s"
+            "void main()                                                                           \n"
+            "{                                                                                     \n"
+            "  vec4 pixel = texture(tex1, texcoords_out);                                          \n"
+            "  FragColor.xy = (RGBtoYUV * vec4(pixel.rgb, 1.0)).yz;                                \n"
+            "  FragColor.w = pixel.a;                                                              \n"
+            "}                                                                                     \n", color_transform_matrix);
+    }
 
     if(gsr_shader_init(shader, egl, vertex_shader, fragment_shader) != 0)
         return -1;
@@ -294,23 +254,6 @@ int gsr_color_conversion_init(gsr_color_conversion *self, const gsr_color_conver
     self->params = *params;
 
     switch(params->destination_color) {
-        case GSR_DESTINATION_COLOR_BGR: {
-            if(self->params.num_destination_textures != 1) {
-                fprintf(stderr, "gsr error: gsr_color_conversion_init: expected 1 destination texture for destination color BGR, got %d destination texture(s)\n", self->params.num_destination_textures);
-                return -1;
-            }
-
-            if(load_shader_bgr(&self->shaders[0], self->params.egl, &self->uniforms[0]) != 0) {
-                fprintf(stderr, "gsr error: gsr_color_conversion_init: failed to load bgr shader\n");
-                goto err;
-            }
-
-            if(load_shader_bgr_external_texture(&self->shaders[1], self->params.egl, &self->uniforms[1]) != 0) {
-                fprintf(stderr, "gsr error: gsr_color_conversion_init: failed to load bgr shader (external texture)\n");
-                goto err;
-            }
-            break;
-        }
         case GSR_DESTINATION_COLOR_NV12:
         case GSR_DESTINATION_COLOR_P010: {
             if(self->params.num_destination_textures != 2) {
@@ -318,15 +261,27 @@ int gsr_color_conversion_init(gsr_color_conversion *self, const gsr_color_conver
                 return -1;
             }
 
-            if(load_shader_y(&self->shaders[0], self->params.egl, &self->uniforms[0], params->destination_color, params->color_range) != 0) {
+            if(load_shader_y(&self->shaders[0], self->params.egl, &self->uniforms[0], params->destination_color, params->color_range, false) != 0) {
                 fprintf(stderr, "gsr error: gsr_color_conversion_init: failed to load Y shader\n");
                 goto err;
             }
 
-            if(load_shader_uv(&self->shaders[1], self->params.egl, &self->uniforms[1], params->destination_color, params->color_range) != 0) {
+            if(load_shader_uv(&self->shaders[1], self->params.egl, &self->uniforms[1], params->destination_color, params->color_range, false) != 0) {
                 fprintf(stderr, "gsr error: gsr_color_conversion_init: failed to load UV shader\n");
                 goto err;
             }
+
+            if(self->params.load_external_image_shader) {
+                if(load_shader_y(&self->shaders[2], self->params.egl, &self->uniforms[2], params->destination_color, params->color_range, true) != 0) {
+                    fprintf(stderr, "gsr error: gsr_color_conversion_init: failed to load Y shader\n");
+                    goto err;
+                }
+
+                if(load_shader_uv(&self->shaders[3], self->params.egl, &self->uniforms[3], params->destination_color, params->color_range, true) != 0) {
+                    fprintf(stderr, "gsr error: gsr_color_conversion_init: failed to load UV shader\n");
+                    goto err;
+                }
+            }
             break;
         }
     }
@@ -459,15 +414,10 @@ void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_
         self->params.egl->glBindFramebuffer(GL_FRAMEBUFFER, self->framebuffers[0]);
         //cap_xcomp->params.egl->glClear(GL_COLOR_BUFFER_BIT); // TODO: Do this in a separate clear_ function. We want to do that when using multiple drm to create the final image (multiple monitors for example)
 
-        if(external_texture) {
-            gsr_shader_use(&self->shaders[1]);
-            self->params.egl->glUniform1f(self->uniforms[1].rotation, rotation);
-            self->params.egl->glUniform2f(self->uniforms[1].offset, pos_norm.x, pos_norm.y);
-        } else {
-            gsr_shader_use(&self->shaders[0]);
-            self->params.egl->glUniform1f(self->uniforms[0].rotation, rotation);
-            self->params.egl->glUniform2f(self->uniforms[0].offset, pos_norm.x, pos_norm.y);
-        }
+        const int shader_index = external_texture ? 2 : 0;
+        gsr_shader_use(&self->shaders[shader_index]);
+        self->params.egl->glUniform1f(self->uniforms[shader_index].rotation, rotation);
+        self->params.egl->glUniform2f(self->uniforms[shader_index].offset, pos_norm.x, pos_norm.y);
         self->params.egl->glDrawArrays(GL_TRIANGLES, 0, 6);
     }
 
@@ -475,9 +425,10 @@ void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_
         self->params.egl->glBindFramebuffer(GL_FRAMEBUFFER, self->framebuffers[1]);
         //cap_xcomp->params.egl->glClear(GL_COLOR_BUFFER_BIT);
 
-        gsr_shader_use(&self->shaders[1]);
-        self->params.egl->glUniform1f(self->uniforms[1].rotation, rotation);
-        self->params.egl->glUniform2f(self->uniforms[1].offset, pos_norm.x, pos_norm.y);
+        const int shader_index = external_texture ? 3 : 1;
+        gsr_shader_use(&self->shaders[shader_index]);
+        self->params.egl->glUniform1f(self->uniforms[shader_index].rotation, rotation);
+        self->params.egl->glUniform2f(self->uniforms[shader_index].offset, pos_norm.x, pos_norm.y);
         self->params.egl->glDrawArrays(GL_TRIANGLES, 0, 6);
     }
 
@@ -494,9 +445,6 @@ void gsr_color_conversion_clear(gsr_color_conversion *self) {
     float color2[4] = {0.0f, 0.0f, 0.0f, 1.0f};
 
     switch(self->params.destination_color) {
-        case GSR_DESTINATION_COLOR_BGR: {
-            break;
-        }
         case GSR_DESTINATION_COLOR_NV12:
         case GSR_DESTINATION_COLOR_P010: {
             color2[0] = 0.5f;