Fix cursor rotated background in compute shader on wayland

5 files changed, 31 insertions, 10 deletions

diff --git a/src/color_conversion.c b/src/color_conversion.c
index a856b50..3cc9006 100644
--- a/src/color_conversion.c
+++ b/src/color_conversion.c
@@ -100,10 +100,11 @@ static int load_compute_shader_y(gsr_shader *shader, gsr_egl *egl, gsr_color_uni
         "    ivec2 size = ivec2(vec2(textureSize(img_input, 0)) * scale + 0.5);\n"
         "    ivec2 output_size = textureSize(img_background, 0);\n"
         "    vec2 rotated_texel_coord = vec2(texel_coord - source_position - size/2) * rotation_matrix + vec2(size/2) + 0.5;\n"
+        "    vec2 output_texel_coord = vec2(texel_coord - source_position + target_position) + 0.5;\n"
         "    vec2 tex_coord = vec2(rotated_texel_coord)/vec2(size);\n"
         "    vec4 source_color = texture(img_input, tex_coord);\n"
         "    vec4 source_color_yuv = RGBtoYUV * vec4(source_color.rgb, 1.0);\n"
-        "    vec4 output_color_yuv = texture(img_background, (rotated_texel_coord + vec2(target_position))/vec2(output_size));\n"
+        "    vec4 output_color_yuv = texture(img_background, output_texel_coord/vec2(output_size));\n"
         "    float y_color = mix(output_color_yuv.r, source_color_yuv.r, source_color.a);\n"
         "    imageStore(img_output, texel_coord + target_position, vec4(y_color, 1.0, 1.0, 1.0));\n"
         "}\n", max_local_size_dim, max_local_size_dim, external_texture ? "samplerExternalOES" : "sampler2D", color_transform_matrix);
@@ -141,10 +142,11 @@ static int load_compute_shader_uv(gsr_shader *shader, gsr_egl *egl, gsr_color_un
         "    ivec2 size = ivec2(vec2(textureSize(img_input, 0)) * scale + 0.5);\n"
         "    ivec2 output_size = textureSize(img_background, 0);\n"
         "    vec2 rotated_texel_coord = vec2(texel_coord - source_position/2 - size/4) * rotation_matrix + vec2(size/4) + 0.5;\n"
+        "    vec2 output_texel_coord = vec2(texel_coord - source_position/2 + target_position/2) + 0.5;\n"
         "    vec2 tex_coord = vec2(rotated_texel_coord)/vec2(size);\n"
         "    vec4 source_color = texture(img_input, tex_coord * 2.0);\n"
         "    vec4 source_color_yuv = RGBtoYUV * vec4(source_color.rgb, 1.0);\n"
-        "    vec4 output_color_yuv = texture(img_background, (rotated_texel_coord + vec2(target_position/2))/vec2(output_size));\n"
+        "    vec4 output_color_yuv = texture(img_background, output_texel_coord/vec2(output_size));\n"
         "    vec2 uv_color = mix(output_color_yuv.rg, source_color_yuv.gb, source_color.a);\n"
         "    imageStore(img_output, texel_coord + target_position/2, vec4(uv_color, 1.0, 1.0));\n"
         "}\n", max_local_size_dim, max_local_size_dim, external_texture ? "samplerExternalOES" : "sampler2D", color_transform_matrix);
@@ -179,9 +181,10 @@ static int load_compute_shader_rgb(gsr_shader *shader, gsr_egl *egl, gsr_color_u
         "    ivec2 size = ivec2(vec2(textureSize(img_input, 0)) * scale + 0.5);\n"
         "    ivec2 output_size = textureSize(img_background, 0);\n"
         "    vec2 rotated_texel_coord = vec2(texel_coord - source_position - size/2) * rotation_matrix + vec2(size/2) + 0.5;\n"
+        "    vec2 output_texel_coord = vec2(texel_coord - source_position + target_position) + 0.5;\n"
         "    vec2 tex_coord = vec2(rotated_texel_coord)/vec2(size);\n"
         "    vec4 source_color = texture(img_input, tex_coord);\n"
-        "    vec4 output_color = texture(img_background, (rotated_texel_coord + vec2(target_position))/vec2(output_size));\n"
+        "    vec4 output_color = texture(img_background, output_texel_coord/vec2(output_size));\n"
         "    vec3 color = mix(output_color.rgb, source_color.rgb, source_color.a);\n"
         "    imageStore(img_output, texel_coord + target_position, vec4(color, 1.0));\n"
         "}\n", max_local_size_dim, max_local_size_dim, external_texture ? "samplerExternalOES" : "sampler2D");
diff --git a/src/encoder/video/vaapi.c b/src/encoder/video/vaapi.c
index 0607a79..c7ccd26 100644
--- a/src/encoder/video/vaapi.c
+++ b/src/encoder/video/vaapi.c
@@ -92,6 +92,10 @@ static bool gsr_video_encoder_vaapi_setup_textures(gsr_video_encoder_vaapi *self
     if(self->prime.fourcc == VA_FOURCC_NV12 || self->prime.fourcc == VA_FOURCC_P010) {
         const uint32_t *formats = self->prime.fourcc == VA_FOURCC_NV12 ? formats_nv12 : formats_p010;
         const int div[2] = {1, 2}; // divide UV texture size by 2 because chroma is half size
+        const float border_colors[2][4] = {
+            {0.0f, 0.0f, 0.0f, 1.0f},
+            {0.5f, 0.5f, 0.0f, 1.0f}
+        };
 
         self->params.egl->glGenTextures(2, self->target_textures);
         for(int i = 0; i < 2; ++i) {
@@ -120,11 +124,10 @@ static bool gsr_video_encoder_vaapi_setup_textures(gsr_video_encoder_vaapi *self
                 return false;
             }
 
-            const float border_color[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
             self->params.egl->glBindTexture(GL_TEXTURE_2D, self->target_textures[i]);
             self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
             self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
-            self->params.egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_color);
+            self->params.egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_colors[i]);
             self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
             self->params.egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
 
diff --git a/src/encoder/video/vulkan.c b/src/encoder/video/vulkan.c
index 062967b..7210870 100644
--- a/src/encoder/video/vulkan.c
+++ b/src/encoder/video/vulkan.c
@@ -94,7 +94,7 @@ static bool gsr_video_encoder_vulkan_setup_textures(gsr_video_encoder_vulkan *se
     //AVVkFrame *target_surface_id = (AVVkFrame*)frame->data[0];
     self->vv = video_codec_context_get_vulkan_data(video_codec_context);
 
-    const unsigned int internal_formats_nv12[2] = { GL_RGBA8, GL_RGBA8 };
+    const unsigned int internal_formats_nv12[2] = { GL_RGBA8, GL_RGBA8 }; // TODO: GL_R8, GL_R16
     const unsigned int internal_formats_p010[2] = { GL_R16, GL_RG16 };
     const unsigned int formats[2] = { GL_RED, GL_RG };
     const int div[2] = {1, 2}; // divide UV texture size by 2 because chroma is half size
diff --git a/src/main.cpp b/src/main.cpp
index 4d231af..4e92730 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -82,11 +82,14 @@ typedef struct {
 static void monitor_output_callback_print(const gsr_monitor *monitor, void *userdata) {
     const MonitorOutputCallbackUserdata *options = (MonitorOutputCallbackUserdata*)userdata;
     vec2i monitor_position = monitor->pos;
+    vec2i monitor_size = monitor->size;
     if(gsr_window_get_display_server(options->window) == GSR_DISPLAY_SERVER_WAYLAND) {
         gsr_monitor_rotation monitor_rotation = GSR_MONITOR_ROT_0;
         drm_monitor_get_display_server_data(options->window, monitor, &monitor_rotation, &monitor_position);
+        if(monitor_rotation == GSR_MONITOR_ROT_90 || monitor_rotation == GSR_MONITOR_ROT_270)
+            std::swap(monitor_size.x, monitor_size.y);
     }
-    fprintf(stderr, "  \"%.*s\"    (%dx%d+%d+%d)\n", monitor->name_len, monitor->name, monitor->size.x, monitor->size.y, monitor_position.x, monitor_position.y);
+    fprintf(stderr, "  \"%.*s\"    (%dx%d+%d+%d)\n", monitor->name_len, monitor->name, monitor_size.x, monitor_size.y, monitor_position.x, monitor_position.y);
 }
 
 typedef struct {
@@ -110,9 +113,14 @@ typedef struct {
 static void get_monitor_by_position_callback(const gsr_monitor *monitor, void *userdata) {
     MonitorByPositionCallback *data = (MonitorByPositionCallback*)userdata;
 
-    gsr_monitor_rotation monitor_rotation = GSR_MONITOR_ROT_0;
     vec2i monitor_position = monitor->pos;
-    drm_monitor_get_display_server_data(data->window, monitor, &monitor_rotation, &monitor_position);
+    vec2i monitor_size = monitor->size;
+    if(gsr_window_get_display_server(data->window) == GSR_DISPLAY_SERVER_WAYLAND) {
+        gsr_monitor_rotation monitor_rotation = GSR_MONITOR_ROT_0;
+        drm_monitor_get_display_server_data(data->window, monitor, &monitor_rotation, &monitor_position);
+        if(monitor_rotation == GSR_MONITOR_ROT_90 || monitor_rotation == GSR_MONITOR_ROT_270)
+            std::swap(monitor_size.x, monitor_size.y);
+    }
 
     if(!data->output_name && data->position.x >= monitor_position.x && data->position.x <= monitor_position.x + monitor->size.x
         && data->position.y >= monitor_position.y && data->position.y <= monitor_position.y + monitor->size.y)
diff --git a/src/utils.c b/src/utils.c
index d3b7e59..4aab138 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -671,12 +671,19 @@ vec2i scale_keep_aspect_ratio(vec2i from, vec2i to) {
 }
 
 unsigned int gl_create_texture(gsr_egl *egl, int width, int height, int internal_format, unsigned int format, int filter) {
+    float border_color[4] = { 0.0f, 0.0f, 0.0f, 1.0f };
+    if(format == GL_RG) { // UV
+        border_color[0] = 0.5f;
+        border_color[1] = 0.5f;
+        border_color[2] = 0.0f;
+        border_color[3] = 1.0f;
+    }
+
     unsigned int texture_id = 0;
     egl->glGenTextures(1, &texture_id);
     egl->glBindTexture(GL_TEXTURE_2D, texture_id);
     egl->glTexImage2D(GL_TEXTURE_2D, 0, internal_format, width, height, 0, format, GL_UNSIGNED_BYTE, NULL);
 
-    const float border_color[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
     egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
     egl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
     egl->glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, border_color);