Fix incorrect color format in x11 cursor (bgr instead of rgb)

11 files changed, 27 insertions, 55 deletions

diff --git a/TODO b/TODO
index df85d7b..de803f8 100644
--- a/TODO
+++ b/TODO
@@ -184,4 +184,6 @@ Support pipewire audio capture which also allows capturing audio from a single a
 
 Support recording/replay/livestreaming at the same time by allowing commands to be run on an existing gpu screen recorder instance.
 
-Test if `xrandr --output DP-1 --scale 1.5` captures correct size on nvidia.
\ No newline at end of file
+Test if `xrandr --output DP-1 --scale 1.5` captures correct size on nvidia.
+
+Fix cursor position and scale when scaling x11 display.
\ No newline at end of file
diff --git a/include/capture/capture.h b/include/capture/capture.h
index 7c8887d..dc5b7ac 100644
--- a/include/capture/capture.h
+++ b/include/capture/capture.h
@@ -9,9 +9,9 @@
 typedef struct AVCodecContext AVCodecContext;
 typedef struct AVStream AVStream;
 typedef struct AVFrame AVFrame;
-typedef struct gsr_capture gsr_capture;
 typedef struct AVMasteringDisplayMetadata AVMasteringDisplayMetadata;
 typedef struct AVContentLightMetadata AVContentLightMetadata;
+typedef struct gsr_capture gsr_capture;
 
 struct gsr_capture {
     /* These methods should not be called manually. Call gsr_capture_* instead */
@@ -20,7 +20,6 @@ struct gsr_capture {
     void (*tick)(gsr_capture *cap); /* can be NULL. If there is an event then |on_event| is called before this */
     bool (*should_stop)(gsr_capture *cap, bool *err); /* can be NULL. If NULL, return false */
     int (*capture)(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *color_conversion);
-    gsr_source_color (*get_source_color)(gsr_capture *cap);
     bool (*uses_external_image)(gsr_capture *cap); /* can be NULL. If NULL, return false */
     bool (*set_hdr_metadata)(gsr_capture *cap, AVMasteringDisplayMetadata *mastering_display_metadata, AVContentLightMetadata *light_metadata); /* can be NULL. If NULL, return false */
     uint64_t (*get_window_id)(gsr_capture *cap); /* can be NULL. Returns 0 if unknown */
@@ -37,7 +36,6 @@ void gsr_capture_on_event(gsr_capture *cap, gsr_egl *egl);
 void gsr_capture_tick(gsr_capture *cap);
 bool gsr_capture_should_stop(gsr_capture *cap, bool *err);
 int gsr_capture_capture(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *color_conversion);
-gsr_source_color gsr_capture_get_source_color(gsr_capture *cap);
 bool gsr_capture_uses_external_image(gsr_capture *cap);
 bool gsr_capture_set_hdr_metadata(gsr_capture *cap, AVMasteringDisplayMetadata *mastering_display_metadata, AVContentLightMetadata *light_metadata);
 void gsr_capture_destroy(gsr_capture *cap, AVCodecContext *video_codec_context);
diff --git a/include/color_conversion.h b/include/color_conversion.h
index 236bfbd..c079edd 100644
--- a/include/color_conversion.h
+++ b/include/color_conversion.h
@@ -33,7 +33,6 @@ typedef struct {
 typedef struct {
     gsr_egl *egl;
 
-    gsr_source_color source_color;
     gsr_destination_color destination_color;
 
     unsigned int destination_textures[2];
@@ -57,7 +56,7 @@ typedef struct {
 int gsr_color_conversion_init(gsr_color_conversion *self, const gsr_color_conversion_params *params);
 void gsr_color_conversion_deinit(gsr_color_conversion *self);
 
-void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i source_pos, vec2i source_size, vec2i texture_pos, vec2i texture_size, float rotation, bool external_texture);
+void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i source_pos, vec2i source_size, vec2i texture_pos, vec2i texture_size, float rotation, bool external_texture, gsr_source_color source_color);
 void gsr_color_conversion_clear(gsr_color_conversion *self);
 
 #endif /* GSR_COLOR_CONVERSION_H */
diff --git a/src/capture/capture.c b/src/capture/capture.c
index ec10854..2a4a689 100644
--- a/src/capture/capture.c
+++ b/src/capture/capture.c
@@ -34,10 +34,6 @@ int gsr_capture_capture(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *
     return cap->capture(cap, frame, color_conversion);
 }
 
-gsr_source_color gsr_capture_get_source_color(gsr_capture *cap) {
-    return cap->get_source_color(cap);
-}
-
 bool gsr_capture_uses_external_image(gsr_capture *cap) {
     if(cap->uses_external_image)
         return cap->uses_external_image(cap);
diff --git a/src/capture/kms.c b/src/capture/kms.c
index 6f021b9..c85811e 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -507,7 +507,7 @@ static void render_drm_cursor(gsr_capture_kms *self, gsr_color_conversion *color
     gsr_color_conversion_draw(color_conversion, self->cursor_texture_id,
         cursor_pos, (vec2i){cursor_size.x * scale.x, cursor_size.y * scale.y},
         (vec2i){0, 0}, cursor_size,
-        texture_rotation, cursor_texture_id_is_external);
+        texture_rotation, cursor_texture_id_is_external, GSR_SOURCE_COLOR_RGB);
 
     self->params.egl->glDisable(GL_SCISSOR_TEST);
 }
@@ -534,7 +534,7 @@ static void render_x11_cursor(gsr_capture_kms *self, gsr_color_conversion *color
     gsr_color_conversion_draw(color_conversion, self->x11_cursor.texture_id,
         cursor_pos, (vec2i){self->x11_cursor.size.x * scale.x, self->x11_cursor.size.y * scale.y},
         (vec2i){0, 0}, self->x11_cursor.size,
-        0.0f, false);
+        0.0f, false, GSR_SOURCE_COLOR_RGB);
 
     self->params.egl->glDisable(GL_SCISSOR_TEST);
 }
@@ -629,7 +629,7 @@ static int gsr_capture_kms_capture(gsr_capture *cap, AVFrame *frame, gsr_color_c
         gsr_color_conversion_draw(color_conversion, self->external_texture_fallback ? self->external_input_texture_id : self->input_texture_id,
             target_pos, output_size,
             capture_pos, self->capture_size,
-            texture_rotation, self->external_texture_fallback);
+            texture_rotation, self->external_texture_fallback, GSR_SOURCE_COLOR_RGB);
     }
 
     if(self->params.record_cursor) {
@@ -661,11 +661,6 @@ static bool gsr_capture_kms_should_stop(gsr_capture *cap, bool *err) {
     return false;
 }
 
-static gsr_source_color gsr_capture_kms_get_source_color(gsr_capture *cap) {
-    (void)cap;
-    return GSR_SOURCE_COLOR_RGB;
-}
-
 static bool gsr_capture_kms_uses_external_image(gsr_capture *cap) {
     (void)cap;
     return true;
@@ -752,7 +747,6 @@ gsr_capture* gsr_capture_kms_create(const gsr_capture_kms_params *params) {
         //.tick = gsr_capture_kms_tick,
         .should_stop = gsr_capture_kms_should_stop,
         .capture = gsr_capture_kms_capture,
-        .get_source_color = gsr_capture_kms_get_source_color,
         .uses_external_image = gsr_capture_kms_uses_external_image,
         .set_hdr_metadata = gsr_capture_kms_set_hdr_metadata,
         //.is_damaged = gsr_capture_kms_is_damaged,
diff --git a/src/capture/nvfbc.c b/src/capture/nvfbc.c
index 96f3894..d5a270e 100644
--- a/src/capture/nvfbc.c
+++ b/src/capture/nvfbc.c
@@ -434,7 +434,7 @@ static int gsr_capture_nvfbc_capture(gsr_capture *cap, AVFrame *frame, gsr_color
     gsr_color_conversion_draw(color_conversion, self->setup_params.dwTextures[grab_params.dwTextureIndex],
         target_pos, (vec2i){output_size.x, output_size.y},
         (vec2i){0, 0}, frame_size,
-        0.0f, false);
+        0.0f, false, GSR_SOURCE_COLOR_BGR);
 
     self->params.egl->glFlush();
     self->params.egl->glFinish();
@@ -442,11 +442,6 @@ static int gsr_capture_nvfbc_capture(gsr_capture *cap, AVFrame *frame, gsr_color
     return 0;
 }
 
-static gsr_source_color gsr_capture_nvfbc_get_source_color(gsr_capture *cap) {
-    (void)cap;
-    return GSR_SOURCE_COLOR_BGR;
-}
-
 static void gsr_capture_nvfbc_destroy(gsr_capture *cap, AVCodecContext *video_codec_context) {
     (void)video_codec_context;
     gsr_capture_nvfbc *self = cap->priv;
@@ -492,7 +487,6 @@ gsr_capture* gsr_capture_nvfbc_create(const gsr_capture_nvfbc_params *params) {
         .tick = NULL,
         .should_stop = NULL,
         .capture = gsr_capture_nvfbc_capture,
-        .get_source_color = gsr_capture_nvfbc_get_source_color,
         .uses_external_image = NULL,
         .destroy = gsr_capture_nvfbc_destroy,
         .priv = cap_nvfbc
diff --git a/src/capture/portal.c b/src/capture/portal.c
index a42bac0..d68e86f 100644
--- a/src/capture/portal.c
+++ b/src/capture/portal.c
@@ -377,7 +377,7 @@ static int gsr_capture_portal_capture(gsr_capture *cap, AVFrame *frame, gsr_colo
         gsr_color_conversion_draw(color_conversion, using_external_image ? self->texture_map.external_texture_id : self->texture_map.texture_id,
             target_pos, output_size,
             (vec2i){region.x, region.y}, self->capture_size,
-            0.0f, using_external_image);
+            0.0f, using_external_image, GSR_SOURCE_COLOR_RGB);
     }
 
     if(self->params.record_cursor && self->texture_map.cursor_texture_id > 0 && cursor_region.width > 0) {
@@ -396,7 +396,7 @@ static int gsr_capture_portal_capture(gsr_capture *cap, AVFrame *frame, gsr_colo
         gsr_color_conversion_draw(color_conversion, self->texture_map.cursor_texture_id,
             (vec2i){cursor_pos.x, cursor_pos.y}, (vec2i){cursor_region.width * scale.x, cursor_region.height * scale.y},
             (vec2i){0, 0}, (vec2i){cursor_region.width, cursor_region.height},
-            0.0f, false);
+            0.0f, false, GSR_SOURCE_COLOR_RGB);
         self->params.egl->glDisable(GL_SCISSOR_TEST);
     }
 
@@ -408,11 +408,6 @@ static int gsr_capture_portal_capture(gsr_capture *cap, AVFrame *frame, gsr_colo
     return 0;
 }
 
-static gsr_source_color gsr_capture_portal_get_source_color(gsr_capture *cap) {
-    (void)cap;
-    return GSR_SOURCE_COLOR_RGB;
-}
-
 static bool gsr_capture_portal_uses_external_image(gsr_capture *cap) {
     (void)cap;
     return true;
@@ -462,7 +457,6 @@ gsr_capture* gsr_capture_portal_create(const gsr_capture_portal_params *params)
         .tick = NULL,
         .should_stop = NULL,
         .capture = gsr_capture_portal_capture,
-        .get_source_color = gsr_capture_portal_get_source_color,
         .uses_external_image = gsr_capture_portal_uses_external_image,
         .is_damaged = gsr_capture_portal_is_damaged,
         .clear_damage = gsr_capture_portal_clear_damage,
diff --git a/src/capture/xcomposite.c b/src/capture/xcomposite.c
index 6a3be16..2867b45 100644
--- a/src/capture/xcomposite.c
+++ b/src/capture/xcomposite.c
@@ -281,7 +281,7 @@ static int gsr_capture_xcomposite_capture(gsr_capture *cap, AVFrame *frame, gsr_
         gsr_color_conversion_draw(color_conversion, window_texture_get_opengl_texture_id(&self->window_texture),
             target_pos, output_size,
             (vec2i){0, 0}, self->texture_size,
-            0.0f, false);
+            0.0f, false, GSR_SOURCE_COLOR_RGB);
     }
 
     if(self->params.record_cursor && self->cursor.visible) {
@@ -303,7 +303,7 @@ static int gsr_capture_xcomposite_capture(gsr_capture *cap, AVFrame *frame, gsr_
         gsr_color_conversion_draw(color_conversion, self->cursor.texture_id,
             cursor_pos, (vec2i){self->cursor.size.x * scale.x, self->cursor.size.y * scale.y},
             (vec2i){0, 0}, self->cursor.size,
-            0.0f, false);
+            0.0f, false, GSR_SOURCE_COLOR_RGB);
 
         self->params.egl->glDisable(GL_SCISSOR_TEST);
     }
@@ -314,11 +314,6 @@ static int gsr_capture_xcomposite_capture(gsr_capture *cap, AVFrame *frame, gsr_
     return 0;
 }
 
-static gsr_source_color gsr_capture_xcomposite_get_source_color(gsr_capture *cap) {
-    (void)cap;
-    return GSR_SOURCE_COLOR_RGB;
-}
-
 static uint64_t gsr_capture_xcomposite_get_window_id(gsr_capture *cap) {
     gsr_capture_xcomposite *self = cap->priv;
     return self->window;
@@ -358,7 +353,6 @@ gsr_capture* gsr_capture_xcomposite_create(const gsr_capture_xcomposite_params *
         .tick = gsr_capture_xcomposite_tick,
         .should_stop = gsr_capture_xcomposite_should_stop,
         .capture = gsr_capture_xcomposite_capture,
-        .get_source_color = gsr_capture_xcomposite_get_source_color,
         .uses_external_image = NULL,
         .get_window_id = gsr_capture_xcomposite_get_window_id,
         .destroy = gsr_capture_xcomposite_destroy,
diff --git a/src/color_conversion.c b/src/color_conversion.c
index 3962dd4..f34512c 100644
--- a/src/color_conversion.c
+++ b/src/color_conversion.c
@@ -325,22 +325,22 @@ void gsr_color_conversion_deinit(gsr_color_conversion *self) {
     self->params.egl = NULL;
 }
 
-static void gsr_color_conversion_swizzle_texture_source(gsr_color_conversion *self) {
-    if(self->params.source_color == GSR_SOURCE_COLOR_BGR) {
-        const int swizzle_mask[] = { GL_BLUE, GL_GREEN, GL_RED, 1 };
+static void gsr_color_conversion_swizzle_texture_source(gsr_color_conversion *self, gsr_source_color source_color) {
+    if(source_color == GSR_SOURCE_COLOR_BGR) {
+        const int swizzle_mask[] = { GL_BLUE, GL_GREEN, GL_RED, GL_ALPHA };
         self->params.egl->glTexParameteriv(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_RGBA, swizzle_mask);
     }
 }
 
-static void gsr_color_conversion_swizzle_reset(gsr_color_conversion *self) {
-    if(self->params.source_color == GSR_SOURCE_COLOR_BGR) {
+static void gsr_color_conversion_swizzle_reset(gsr_color_conversion *self, gsr_source_color source_color) {
+    if(source_color == GSR_SOURCE_COLOR_BGR) {
         const int swizzle_mask[] = { GL_RED, GL_GREEN, GL_BLUE, GL_ALPHA };
         self->params.egl->glTexParameteriv(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_RGBA, swizzle_mask);
     }
 }
 
 /* |source_pos| is in pixel coordinates and |source_size|  */
-void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i source_pos, vec2i source_size, vec2i texture_pos, vec2i texture_size, float rotation, bool external_texture) {
+void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i source_pos, vec2i source_size, vec2i texture_pos, vec2i texture_size, float rotation, bool external_texture, gsr_source_color source_color) {
     // TODO: Remove this crap
     rotation = M_PI*2.0f - rotation;
 
@@ -402,7 +402,7 @@ void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_
         -1.0f + 0.0f + size_norm.x, -1.0f + 0.0f + size_norm.y, texture_pos_norm.x + texture_size_norm.x, texture_pos_norm.y + texture_size_norm.y
     };
 
-    gsr_color_conversion_swizzle_texture_source(self);
+    gsr_color_conversion_swizzle_texture_source(self, source_color);
 
     self->params.egl->glBindVertexArray(self->vertex_array_object_id);
     self->params.egl->glViewport(0, 0, dest_texture_size.x, dest_texture_size.y);
@@ -438,7 +438,7 @@ void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_
     self->params.egl->glBindTexture(texture_target, 0);
     self->params.egl->glBindFramebuffer(GL_FRAMEBUFFER, 0);
 
-    gsr_color_conversion_swizzle_reset(self);
+    gsr_color_conversion_swizzle_reset(self, source_color);
 }
 
 void gsr_color_conversion_clear(gsr_color_conversion *self) {
diff --git a/src/cursor.c b/src/cursor.c
index 3dca0c6..56b9694 100644
--- a/src/cursor.c
+++ b/src/cursor.c
@@ -43,10 +43,12 @@ static bool gsr_cursor_set_from_x11_cursor_image(gsr_cursor *self, XFixesCursorI
                 *visible = true;
             }
 
-            *out++ = (unsigned)*in++ * 255/alpha;
-            *out++ = (unsigned)*in++ * 255/alpha;
-            *out++ = (unsigned)*in++ * 255/alpha;
-            *out++ = *in++;
+            out[0] = (float)in[2] * 255.0/(float)alpha;
+            out[1] = (float)in[1] * 255.0/(float)alpha;
+            out[2] = (float)in[0] * 255.0/(float)alpha;
+            out[3] = in[3];
+            out += 4;
+            in += 4;
         }
     }
 
diff --git a/src/main.cpp b/src/main.cpp
index 35b5119..21425d9 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -3318,7 +3318,6 @@ int main(int argc, char **argv) {
     memset(&color_conversion_params, 0, sizeof(color_conversion_params));
     color_conversion_params.color_range = color_range;
     color_conversion_params.egl = &egl;
-    color_conversion_params.source_color = gsr_capture_get_source_color(capture);
     color_conversion_params.load_external_image_shader = gsr_capture_uses_external_image(capture);
     gsr_video_encoder_get_textures(video_encoder, color_conversion_params.destination_textures, &color_conversion_params.num_destination_textures, &color_conversion_params.destination_color);