From 3e2e2444d9fa65ae404604542689cf7829c99cfa Mon Sep 17 00:00:00 2001
From: dec05eba <dec05eba@protonmail.com>
Date: Fri, 14 Jun 2024 02:05:54 +0200
Subject: Properly cut out cursor outside video area when dealing with hevc amd
 padding

---
 src/capture/kms.c        |  5 +++++
 src/capture/xcomposite.c | 25 +++++--------------------
 src/egl.c                |  2 ++
 src/main.cpp             |  1 +
 4 files changed, 13 insertions(+), 20 deletions(-)

(limited to 'src')

diff --git a/src/capture/kms.c b/src/capture/kms.c
index e479019..2d1cea3 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -358,10 +358,15 @@ bool gsr_capture_kms_capture(gsr_capture_kms *self, AVFrame *frame, bool hdr, bo
         self->base.egl->eglDestroyImage(self->base.egl->egl_display, cursor_image);
         self->base.egl->glBindTexture(target, 0);
 
+        self->base.egl->glEnable(GL_SCISSOR_TEST);
+        self->base.egl->glScissor(target_x, target_y, self->capture_size.x, self->capture_size.y);
+
         gsr_color_conversion_draw(&self->base.color_conversion, self->base.cursor_texture,
             cursor_pos, cursor_size,
             (vec2i){0, 0}, cursor_size,
             texture_rotation, cursor_texture_is_external);
+
+        self->base.egl->glDisable(GL_SCISSOR_TEST);
     }
 
     self->base.egl->eglSwapBuffers(self->base.egl->egl_display, self->base.egl->egl_surface);
diff --git a/src/capture/xcomposite.c b/src/capture/xcomposite.c
index 47daa99..236cfc0 100644
--- a/src/capture/xcomposite.c
+++ b/src/capture/xcomposite.c
@@ -116,7 +116,6 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v
     frame->height = video_codec_context->height;
 
     self->window_resize_timer = clock_get_monotonic_seconds();
-    self->clear_next_frame = true;
     return 0;
 }
 
@@ -251,41 +250,27 @@ int gsr_capture_xcomposite_capture(gsr_capture_xcomposite *self, AVFrame *frame)
         target_y + self->cursor.position.y - self->cursor.hotspot.y
     };
 
-    const bool cursor_completely_inside_window =
-        cursor_pos.x >= target_x &&
-        cursor_pos.x + self->cursor.size.x <= target_x + self->texture_size.x &&
-        cursor_pos.y >= target_y &&
-        cursor_pos.y + self->cursor.size.y <= target_y + self->texture_size.y;
-
     const bool cursor_inside_window =
         cursor_pos.x + self->cursor.size.x >= target_x &&
         cursor_pos.x <= target_x + self->texture_size.x &&
         cursor_pos.y + self->cursor.size.y >= target_y &&
         cursor_pos.y <= target_y + self->texture_size.y;
 
-    if(self->clear_next_frame) {
-        self->clear_next_frame = false;
-        gsr_color_conversion_clear(&self->base.color_conversion);
-    }
-
-    /*
-        We dont draw the cursor if it's outside the window but if it's partially inside the window then the cursor area that is outside the window
-        will not get overdrawn the next frame causing a cursor trail to be visible since we dont clear the background.
-        To fix this we detect if the cursor is partially inside the window and clear the background only in that case.
-    */
-    if(!cursor_completely_inside_window && cursor_inside_window && self->params.record_cursor)
-        self->clear_next_frame = true;
-
     gsr_color_conversion_draw(&self->base.color_conversion, window_texture_get_opengl_texture_id(&self->window_texture),
         (vec2i){target_x, target_y}, self->texture_size,
         (vec2i){0, 0}, self->texture_size,
         0.0f, false);
 
     if(cursor_inside_window && self->params.record_cursor) {
+        self->base.egl->glEnable(GL_SCISSOR_TEST);
+        self->base.egl->glScissor(target_x, target_y, self->texture_size.x, self->texture_size.y);
+
         gsr_color_conversion_draw(&self->base.color_conversion, self->cursor.texture_id,
             cursor_pos, self->cursor.size,
             (vec2i){0, 0}, self->cursor.size,
             0.0f, false);
+
+        self->base.egl->glDisable(GL_SCISSOR_TEST);
     }
 
     self->params.egl->eglSwapBuffers(self->params.egl->egl_display, self->params.egl->egl_surface);
diff --git a/src/egl.c b/src/egl.c
index 2c139e6..48234ac 100644
--- a/src/egl.c
+++ b/src/egl.c
@@ -452,11 +452,13 @@ static bool gsr_egl_load_gl(gsr_egl *self, void *library) {
         { (void**)&self->glEnableVertexAttribArray, "glEnableVertexAttribArray" },
         { (void**)&self->glDrawArrays, "glDrawArrays" },
         { (void**)&self->glEnable, "glEnable" },
+        { (void**)&self->glDisable, "glDisable" },
         { (void**)&self->glBlendFunc, "glBlendFunc" },
         { (void**)&self->glGetUniformLocation, "glGetUniformLocation" },
         { (void**)&self->glUniform1f, "glUniform1f" },
         { (void**)&self->glUniform2f, "glUniform2f" },
         { (void**)&self->glDebugMessageCallback, "glDebugMessageCallback" },
+        { (void**)&self->glScissor, "glScissor" },
 
         { NULL, NULL }
     };
diff --git a/src/main.cpp b/src/main.cpp
index ce56373..afc60d1 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -1504,6 +1504,7 @@ static gsr_capture* create_capture_impl(const char *window_str, const char *scre
                     window_str = first_output.output_name;
                 } else {
                     fprintf(stderr, "Error: no available output found\n");
+                    _exit(1);
                 }
             }
 
-- 
cgit v1.2.3