Fix incorrect compute size when scaling image

11 files changed, 54 insertions, 32 deletions

diff --git a/include/egl.h b/include/egl.h
index f7b0cc1..859f8e3 100644
--- a/include/egl.h
+++ b/include/egl.h
@@ -141,6 +141,8 @@ typedef void(*__GLXextFuncPtr)(void);
 #define GL_MAX_COMPUTE_FIXED_GROUP_INVOCATIONS  0x90EB
 #define GL_TEXTURE0                             0x84C0
 #define GL_TEXTURE1                             0x84C1
+#define GL_CLAMP_TO_BORDER                      0x812D
+#define GL_TEXTURE_BORDER_COLOR                 0x1004
 
 #define GL_VENDOR                               0x1F00
 #define GL_RENDERER                             0x1F01
@@ -243,6 +245,7 @@ struct gsr_egl {
     void (*glBindImageTexture)(unsigned int unit, unsigned int texture, int level, unsigned char layered, int layer, unsigned int access, unsigned int format);
     void (*glTexParameteri)(unsigned int target, unsigned int pname, int param);
     void (*glTexParameteriv)(unsigned int target, unsigned int pname, const int *params);
+    void (*glTexParameterfv)(unsigned int target, unsigned int pname, const float *params);
     void (*glGetTexLevelParameteriv)(unsigned int target, int level, unsigned int pname, int *params);
     void (*glTexImage2D)(unsigned int target, int level, int internalFormat, int width, int height, int border, unsigned int format, unsigned int type, const void *pixels);
     void (*glTexSubImage2D)(unsigned int target, int level, int xoffset, int yoffset, int width, int height, unsigned format, unsigned type, const void *pixels);
diff --git a/src/capture/kms.c b/src/capture/kms.c
index e98fab7..d834090 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -108,18 +108,22 @@ static int max_int(int a, int b) {
 }
 
 static void gsr_capture_kms_create_input_texture_ids(gsr_capture_kms *self) {
+    const float border_color[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
+
     self->params.egl->glGenTextures(1, &self->input_texture_id);
     self->params.egl->glBindTexture(GL_TEXTURE_2D, self->input_texture_id);
-    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_color);
     self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     self->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
 
     self->params.egl->glGenTextures(1, &self->external_input_texture_id);
     self->params.egl->glBindTexture(GL_TEXTURE_EXTERNAL_OES, self->external_input_texture_id);
-    self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameterfv(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_BORDER_COLOR, border_color);
     self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     self->params.egl->glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0);
@@ -129,8 +133,9 @@ static void gsr_capture_kms_create_input_texture_ids(gsr_capture_kms *self) {
 
     self->params.egl->glGenTextures(1, &self->cursor_texture_id);
     self->params.egl->glBindTexture(cursor_texture_id_target, self->cursor_texture_id);
-    self->params.egl->glTexParameteri(cursor_texture_id_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    self->params.egl->glTexParameteri(cursor_texture_id_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    self->params.egl->glTexParameteri(cursor_texture_id_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameteri(cursor_texture_id_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameterfv(cursor_texture_id_target, GL_TEXTURE_BORDER_COLOR, border_color);
     self->params.egl->glTexParameteri(cursor_texture_id_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     self->params.egl->glTexParameteri(cursor_texture_id_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     self->params.egl->glBindTexture(cursor_texture_id_target, 0);
diff --git a/src/capture/portal.c b/src/capture/portal.c
index 56072d8..6b25829 100644
--- a/src/capture/portal.c
+++ b/src/capture/portal.c
@@ -64,26 +64,31 @@ static void gsr_capture_portal_stop(gsr_capture_portal *self) {
 }
 
 static void gsr_capture_portal_create_input_textures(gsr_capture_portal *self) {
+    const float border_color[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
+
     self->params.egl->glGenTextures(1, &self->texture_map.texture_id);
     self->params.egl->glBindTexture(GL_TEXTURE_2D, self->texture_map.texture_id);
-    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_color);
     self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     self->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
 
     self->params.egl->glGenTextures(1, &self->texture_map.external_texture_id);
     self->params.egl->glBindTexture(GL_TEXTURE_EXTERNAL_OES, self->texture_map.external_texture_id);
-    self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameterfv(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_BORDER_COLOR, border_color);
     self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     self->params.egl->glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     self->params.egl->glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0);
 
     self->params.egl->glGenTextures(1, &self->texture_map.cursor_texture_id);
     self->params.egl->glBindTexture(GL_TEXTURE_2D, self->texture_map.cursor_texture_id);
-    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+    self->params.egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_color);
     self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     self->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
diff --git a/src/capture/xcomposite.c b/src/capture/xcomposite.c
index 16bc988..86f7388 100644
--- a/src/capture/xcomposite.c
+++ b/src/capture/xcomposite.c
@@ -274,15 +274,13 @@ static int gsr_capture_xcomposite_capture(gsr_capture *cap, gsr_capture_metadata
             target_pos.y + (self->cursor.position.y - self->cursor.hotspot.y) * scale.y
         };
 
-        self->params.egl->glEnable(GL_SCISSOR_TEST);
-        self->params.egl->glScissor(target_pos.x, target_pos.y, output_size.x, output_size.y);
+        if(cursor_pos.x < target_pos.x || cursor_pos.x + self->cursor.size.x > target_pos.x + output_size.x || cursor_pos.y < target_pos.y || cursor_pos.y + self->cursor.size.y > target_pos.y + output_size.y)
+            self->clear_background = true;
 
         gsr_color_conversion_draw(color_conversion, self->cursor.texture_id,
             cursor_pos, (vec2i){self->cursor.size.x * scale.x, self->cursor.size.y * scale.y},
             (vec2i){0, 0}, self->cursor.size,
             GSR_ROT_0, false, GSR_SOURCE_COLOR_RGB);
-
-        self->params.egl->glDisable(GL_SCISSOR_TEST);
     }
 
     //self->params.egl->glFlush();
diff --git a/src/color_conversion.c b/src/color_conversion.c
index 458d4c7..1662902 100644
--- a/src/color_conversion.c
+++ b/src/color_conversion.c
@@ -438,8 +438,8 @@ void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_
                 self->params.egl->glUniform2i(uniform->target_position, destination_pos.x, destination_pos.y);
                 self->params.egl->glUniform2f(uniform->scale, scale.x, scale.y);
                 self->params.egl->glBindImageTexture(0, self->params.destination_textures[0], 0, GL_FALSE, 0, GL_WRITE_ONLY, use_16bit_colors ? GL_R16 : GL_R8);
-                const double num_groups_x = ceil((double)texture_size.x/(double)self->max_local_size_dim);
-                const double num_groups_y = ceil((double)texture_size.y/(double)self->max_local_size_dim);
+                const double num_groups_x = ceil((double)destination_size.x/(double)self->max_local_size_dim);
+                const double num_groups_y = ceil((double)destination_size.y/(double)self->max_local_size_dim);
                 self->params.egl->glDispatchCompute(max_int(1, num_groups_x), max_int(1, num_groups_y), 1);
             }
 
@@ -456,8 +456,8 @@ void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_
                 self->params.egl->glUniform2i(uniform->target_position, destination_pos.x, destination_pos.y);
                 self->params.egl->glUniform2f(uniform->scale, scale.x, scale.y);
                 self->params.egl->glBindImageTexture(0, self->params.destination_textures[1], 0, GL_FALSE, 0, GL_WRITE_ONLY, use_16bit_colors ? GL_RG16 : GL_RG8);
-                const double num_groups_x = ceil((double)texture_size.x*0.5/(double)self->max_local_size_dim);
-                const double num_groups_y = ceil((double)texture_size.y*0.5/(double)self->max_local_size_dim);
+                const double num_groups_x = ceil((double)destination_size.x*0.5/(double)self->max_local_size_dim);
+                const double num_groups_y = ceil((double)destination_size.y*0.5/(double)self->max_local_size_dim);
                 self->params.egl->glDispatchCompute(max_int(1, num_groups_x), max_int(1, num_groups_y), 1);
             }
             break;
@@ -476,8 +476,8 @@ void gsr_color_conversion_draw(gsr_color_conversion *self, unsigned int texture_
             self->params.egl->glUniform2i(uniform->target_position, destination_pos.x, destination_pos.y);
             self->params.egl->glUniform2f(uniform->scale, scale.x, scale.y);
             self->params.egl->glBindImageTexture(0, self->params.destination_textures[0], 0, GL_FALSE, 0, GL_WRITE_ONLY, GL_RGBA8);
-            const double num_groups_x = ceil((double)texture_size.x/(double)self->max_local_size_dim);
-            const double num_groups_y = ceil((double)texture_size.y/(double)self->max_local_size_dim);
+            const double num_groups_x = ceil((double)destination_size.x/(double)self->max_local_size_dim);
+            const double num_groups_y = ceil((double)destination_size.y/(double)self->max_local_size_dim);
             self->params.egl->glDispatchCompute(max_int(1, num_groups_x), max_int(1, num_groups_y), 1);
             break;
         }
diff --git a/src/cursor.c b/src/cursor.c
index 56b9694..40532f8 100644
--- a/src/cursor.c
+++ b/src/cursor.c
@@ -56,8 +56,10 @@ static bool gsr_cursor_set_from_x11_cursor_image(gsr_cursor *self, XFixesCursorI
     self->egl->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, self->size.x, self->size.y, 0, GL_RGBA, GL_UNSIGNED_BYTE, cursor_data);
     free(cursor_data);
 
-    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    const float border_color[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
+    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+    self->egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_color);
     self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 
diff --git a/src/egl.c b/src/egl.c
index 946399c..446d105 100644
--- a/src/egl.c
+++ b/src/egl.c
@@ -288,6 +288,7 @@ static bool gsr_egl_load_gl(gsr_egl *self, void *library) {
         { (void**)&self->glBindImageTexture, "glBindImageTexture" },
         { (void**)&self->glTexParameteri, "glTexParameteri" },
         { (void**)&self->glTexParameteriv, "glTexParameteriv" },
+        { (void**)&self->glTexParameterfv, "glTexParameterfv" },
         { (void**)&self->glGetTexLevelParameteriv, "glGetTexLevelParameteriv" },
         { (void**)&self->glTexImage2D, "glTexImage2D" },
         { (void**)&self->glTexSubImage2D, "glTexSubImage2D" },
diff --git a/src/encoder/video/vaapi.c b/src/encoder/video/vaapi.c
index 1d5dae0..0607a79 100644
--- a/src/encoder/video/vaapi.c
+++ b/src/encoder/video/vaapi.c
@@ -120,9 +120,11 @@ static bool gsr_video_encoder_vaapi_setup_textures(gsr_video_encoder_vaapi *self
                 return false;
             }
 
+            const float border_color[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
             self->params.egl->glBindTexture(GL_TEXTURE_2D, self->target_textures[i]);
-            self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-            self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+            self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+            self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+            self->params.egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_color);
             self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
             self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
 
diff --git a/src/pipewire_video.c b/src/pipewire_video.c
index 3f7b2df..bbc3f5d 100644
--- a/src/pipewire_video.c
+++ b/src/pipewire_video.c
@@ -736,11 +736,13 @@ static void gsr_pipewire_video_update_cursor_texture(gsr_pipewire_video *self, g
     if(!self->cursor.data)
         return;
 
+    const float border_color[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
     self->egl->glBindTexture(GL_TEXTURE_2D, texture_map.cursor_texture_id);
     // TODO: glTextureSubImage2D if same size
     self->egl->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, self->cursor.width, self->cursor.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, self->cursor.data);
-    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+    self->egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_color);
     self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     self->egl->glBindTexture(GL_TEXTURE_2D, 0);
diff --git a/src/utils.c b/src/utils.c
index 943fb2d..24f5a6d 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -684,8 +684,10 @@ unsigned int gl_create_texture(gsr_egl *egl, int width, int height, int internal
     egl->glBindTexture(GL_TEXTURE_2D, texture_id);
     egl->glTexImage2D(GL_TEXTURE_2D, 0, internal_format, width, height, 0, format, GL_UNSIGNED_BYTE, NULL);
 
-    egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    const float border_color[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
+    egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+    egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+    egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_color);
     egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, filter);
     egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, filter);
 
diff --git a/src/window_texture.c b/src/window_texture.c
index 8eef4c9..4846bdc 100644
--- a/src/window_texture.c
+++ b/src/window_texture.c
@@ -85,8 +85,10 @@ int window_texture_on_resize(WindowTexture *self) {
         texture_id = self->texture_id;
     }
 
-    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    const float border_color[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
+    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+    self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+    self->egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_color);
     self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     self->egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);