Fix incorrect region when monitor is rotated

8 files changed, 21 insertions, 17 deletions

diff --git a/TODO b/TODO
index fa2b4f0..347bb96 100644
--- a/TODO
+++ b/TODO
@@ -253,4 +253,6 @@ When webcam support is added also support v4l2loopback? this is done by using av
 
 Do proper exit, to call gsr_capture_destroy which will properly stop gsr-kms-server. Otherwise there can be zombie gsr-kms-server on error.
 
-Replace all scissors with clearing textures if the cursor hits the outside of the frame image
\ No newline at end of file
+Replace all scissors with clearing textures if the cursor hits the outside of the frame image.
+
+Cursor position might be slightly wrong on rotated monitor.
diff --git a/include/color_conversion.h b/include/color_conversion.h
index f1eaad8..1c067e2 100644
--- a/include/color_conversion.h
+++ b/include/color_conversion.h
@@ -72,7 +72,7 @@ typedef struct {
 int gsr_color_conversion_init(gsr_color_conversion *self, const gsr_color_conversion_params *params);
 void gsr_color_conversion_deinit(gsr_color_conversion *self);
 
-void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i destination_pos, vec2i destination_size, vec2i texture_pos, vec2i texture_size, gsr_rotation rotation, bool external_texture, gsr_source_color source_color);
+void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i destination_pos, vec2i destination_size, vec2i source_pos, vec2i source_size, vec2i texture_size, gsr_rotation rotation, bool external_texture, gsr_source_color source_color);
 void gsr_color_conversion_clear(gsr_color_conversion *self);
 
 gsr_rotation gsr_monitor_rotation_to_rotation(gsr_monitor_rotation monitor_rotation);
diff --git a/src/capture/kms.c b/src/capture/kms.c
index d834090..2cbd757 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -502,7 +502,7 @@ static void render_drm_cursor(gsr_capture_kms *self, gsr_color_conversion *color
 
     gsr_color_conversion_draw(color_conversion, self->cursor_texture_id,
         cursor_pos, (vec2i){cursor_size.x * scale.x, cursor_size.y * scale.y},
-        (vec2i){0, 0}, cursor_size,
+        (vec2i){0, 0}, cursor_size, cursor_size,
         gsr_monitor_rotation_to_rotation(self->monitor_rotation), cursor_texture_id_is_external, GSR_SOURCE_COLOR_RGB);
 
     self->params.egl->glDisable(GL_SCISSOR_TEST);
@@ -530,7 +530,7 @@ static void render_x11_cursor(gsr_capture_kms *self, gsr_color_conversion *color
 
     gsr_color_conversion_draw(color_conversion, self->x11_cursor.texture_id,
         cursor_pos, (vec2i){self->x11_cursor.size.x * scale.x, self->x11_cursor.size.y * scale.y},
-        (vec2i){0, 0}, self->x11_cursor.size,
+        (vec2i){0, 0}, self->x11_cursor.size, self->x11_cursor.size,
         GSR_ROT_0, false, GSR_SOURCE_COLOR_RGB);
 
     self->params.egl->glDisable(GL_SCISSOR_TEST);
@@ -616,6 +616,7 @@ static int gsr_capture_kms_capture(gsr_capture *cap, gsr_capture_metadata *captu
         gsr_kms_set_hdr_metadata(self, drm_fd);
 
     self->capture_size = rotate_capture_size_if_rotated(self, (vec2i){ drm_fd->src_w, drm_fd->src_h });
+    const vec2i original_frame_size = self->capture_size;
     if(self->params.region_size.x > 0 && self->params.region_size.y > 0)
         self->capture_size = self->params.region_size;
 
@@ -644,7 +645,7 @@ static int gsr_capture_kms_capture(gsr_capture *cap, gsr_capture_metadata *captu
 
     gsr_color_conversion_draw(color_conversion, self->external_texture_fallback ? self->external_input_texture_id : self->input_texture_id,
         target_pos, output_size,
-        capture_pos, self->capture_size,
+        capture_pos, self->capture_size, original_frame_size,
         gsr_monitor_rotation_to_rotation(self->monitor_rotation), self->external_texture_fallback, GSR_SOURCE_COLOR_RGB);
 
     if(self->params.record_cursor) {
diff --git a/src/capture/nvfbc.c b/src/capture/nvfbc.c
index 5f47b00..4ed19b3 100644
--- a/src/capture/nvfbc.c
+++ b/src/capture/nvfbc.c
@@ -363,6 +363,7 @@ static int gsr_capture_nvfbc_capture(gsr_capture *cap, gsr_capture_metadata *cap
     }
 
     vec2i frame_size = (vec2i){self->width, self->height};
+    const vec2i original_frame_size = frame_size;
     if(self->params.region_size.x > 0 && self->params.region_size.y > 0)
         frame_size = self->params.region_size;
 
@@ -395,7 +396,7 @@ static int gsr_capture_nvfbc_capture(gsr_capture *cap, gsr_capture_metadata *cap
 
     gsr_color_conversion_draw(color_conversion, self->setup_params.dwTextures[grab_params.dwTextureIndex],
         target_pos, (vec2i){output_size.x, output_size.y},
-        self->params.region_position, frame_size,
+        self->params.region_position, frame_size, original_frame_size,
         GSR_ROT_0, false, GSR_SOURCE_COLOR_BGR);
 
     //self->params.egl->glFlush();
diff --git a/src/capture/portal.c b/src/capture/portal.c
index 6b25829..ec87ab6 100644
--- a/src/capture/portal.c
+++ b/src/capture/portal.c
@@ -347,7 +347,7 @@ static int gsr_capture_portal_capture(gsr_capture *cap, gsr_capture_metadata *ca
 
     gsr_color_conversion_draw(color_conversion, using_external_image ? self->texture_map.external_texture_id : self->texture_map.texture_id,
         target_pos, output_size,
-        (vec2i){region.x, region.y}, self->capture_size,
+        (vec2i){region.x, region.y}, self->capture_size, self->capture_size,
         GSR_ROT_0, using_external_image, GSR_SOURCE_COLOR_RGB);
 
     if(self->params.record_cursor && self->texture_map.cursor_texture_id > 0 && cursor_region.width > 0) {
@@ -365,7 +365,7 @@ static int gsr_capture_portal_capture(gsr_capture *cap, gsr_capture_metadata *ca
         self->params.egl->glScissor(target_pos.x, target_pos.y, output_size.x, output_size.y);
         gsr_color_conversion_draw(color_conversion, self->texture_map.cursor_texture_id,
             (vec2i){cursor_pos.x, cursor_pos.y}, (vec2i){cursor_region.width * scale.x, cursor_region.height * scale.y},
-            (vec2i){0, 0}, (vec2i){cursor_region.width, cursor_region.height},
+            (vec2i){0, 0}, (vec2i){cursor_region.width, cursor_region.height}, (vec2i){cursor_region.width, cursor_region.height},
             GSR_ROT_0, false, GSR_SOURCE_COLOR_RGB);
         self->params.egl->glDisable(GL_SCISSOR_TEST);
     }
diff --git a/src/capture/xcomposite.c b/src/capture/xcomposite.c
index 86f7388..2d0574c 100644
--- a/src/capture/xcomposite.c
+++ b/src/capture/xcomposite.c
@@ -258,7 +258,7 @@ static int gsr_capture_xcomposite_capture(gsr_capture *cap, gsr_capture_metadata
 
     gsr_color_conversion_draw(color_conversion, window_texture_get_opengl_texture_id(&self->window_texture),
         target_pos, output_size,
-        (vec2i){0, 0}, self->texture_size,
+        (vec2i){0, 0}, self->texture_size, self->texture_size,
         GSR_ROT_0, false, GSR_SOURCE_COLOR_RGB);
 
     if(self->params.record_cursor && self->cursor.visible) {
@@ -279,7 +279,7 @@ static int gsr_capture_xcomposite_capture(gsr_capture *cap, gsr_capture_metadata
 
         gsr_color_conversion_draw(color_conversion, self->cursor.texture_id,
             cursor_pos, (vec2i){self->cursor.size.x * scale.x, self->cursor.size.y * scale.y},
-            (vec2i){0, 0}, self->cursor.size,
+            (vec2i){0, 0}, self->cursor.size, self->cursor.size,
             GSR_ROT_0, false, GSR_SOURCE_COLOR_RGB);
     }
 
diff --git a/src/capture/ximage.c b/src/capture/ximage.c
index ac00d72..1f86d93 100644
--- a/src/capture/ximage.c
+++ b/src/capture/ximage.c
@@ -159,7 +159,7 @@ static int gsr_capture_ximage_capture(gsr_capture *cap, gsr_capture_metadata *ca
 
     gsr_color_conversion_draw(color_conversion, self->texture_id,
         target_pos, output_size,
-        (vec2i){0, 0}, self->capture_size,
+        (vec2i){0, 0}, self->capture_size, self->capture_size,
         GSR_ROT_0, false, GSR_SOURCE_COLOR_RGB);
 
     if(self->params.record_cursor && self->cursor.visible) {
@@ -180,7 +180,7 @@ static int gsr_capture_ximage_capture(gsr_capture *cap, gsr_capture_metadata *ca
 
         gsr_color_conversion_draw(color_conversion, self->cursor.texture_id,
             cursor_pos, (vec2i){self->cursor.size.x * scale.x, self->cursor.size.y * scale.y},
-            (vec2i){0, 0}, self->cursor.size,
+            (vec2i){0, 0}, self->cursor.size, self->cursor.size,
             GSR_ROT_0, false, GSR_SOURCE_COLOR_RGB);
 
         self->params.egl->glDisable(GL_SCISSOR_TEST);
diff --git a/src/color_conversion.c b/src/color_conversion.c
index 3cc9006..27ef488 100644
--- a/src/color_conversion.c
+++ b/src/color_conversion.c
@@ -462,17 +462,17 @@ static void gsr_color_conversion_dispatch_compute_shader(gsr_color_conversion *s
     self->params.egl->glDispatchCompute(max_int(1, num_groups_x), max_int(1, num_groups_y), 1);
 }
 
-void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i destination_pos, vec2i destination_size, vec2i texture_pos, vec2i texture_size, gsr_rotation rotation, bool external_texture, gsr_source_color source_color) {
+void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_id, vec2i destination_pos, vec2i destination_size, vec2i source_pos, vec2i source_size, vec2i texture_size, gsr_rotation rotation, bool external_texture, gsr_source_color source_color) {
     vec2f scale = {0.0f, 0.0f};
-    if(texture_size.x > 0 && texture_size.y > 0)
-        scale = (vec2f){ (double)destination_size.x/(double)texture_size.x, (double)destination_size.y/(double)texture_size.y };
+    if(source_size.x > 0 && source_size.y > 0)
+        scale = (vec2f){ (double)destination_size.x/(double)source_size.x, (double)destination_size.y/(double)source_size.y };
 
     vec2i source_position = {0, 0};
     float rotation_matrix[2][2] = {{0, 0}, {0, 0}};
     gsr_color_conversion_apply_rotation(rotation, rotation_matrix, &source_position, texture_size, scale);
 
-    source_position.x -= (texture_pos.x * scale.x + 0.5);
-    source_position.y -= (texture_pos.y * scale.y + 0.5);
+    source_position.x -= (source_pos.x * scale.x + 0.5);
+    source_position.y -= (source_pos.y * scale.y + 0.5);
 
     const int texture_target = external_texture ? GL_TEXTURE_EXTERNAL_OES : GL_TEXTURE_2D;
     self->params.egl->glBindTexture(texture_target, texture_id);