Fix green screen sometimes when capturing amd/intel window

6 files changed, 27 insertions, 22 deletions

diff --git a/include/utils.h b/include/utils.h
index 68d4e81..32977a1 100644
--- a/include/utils.h
+++ b/include/utils.h
@@ -50,4 +50,6 @@ bool gl_get_gpu_info(Display *dpy, gsr_gpu_info *info, bool wayland);
 /* |output| should be at least 128 bytes in size */
 bool gsr_get_valid_card_path(char *output);
 
+int even_number_ceil(int value);
+
 #endif /* GSR_UTILS_H */
diff --git a/src/capture/kms_cuda.c b/src/capture/kms_cuda.c
index b171421..fc7da5b 100644
--- a/src/capture/kms_cuda.c
+++ b/src/capture/kms_cuda.c
@@ -152,8 +152,8 @@ static int gsr_capture_kms_cuda_start(gsr_capture *cap, AVCodecContext *video_co
     cap_kms->capture_pos = monitor.pos;
     cap_kms->capture_size = monitor.size;
 
-    video_codec_context->width = max_int(2, cap_kms->capture_size.x & ~1);
-    video_codec_context->height = max_int(2, cap_kms->capture_size.y & ~1);
+    video_codec_context->width = max_int(2, even_number_ceil(cap_kms->capture_size.x));
+    video_codec_context->height = max_int(2, even_number_ceil(cap_kms->capture_size.y));
 
     if(!gsr_egl_load(&cap_kms->egl, NULL, true)) {
         fprintf(stderr, "gsr error: gsr_capture_kms_cuda_start: failed to load opengl\n");
diff --git a/src/capture/kms_vaapi.c b/src/capture/kms_vaapi.c
index 2c836b3..69b9d94 100644
--- a/src/capture/kms_vaapi.c
+++ b/src/capture/kms_vaapi.c
@@ -257,8 +257,8 @@ static int gsr_capture_kms_vaapi_start(gsr_capture *cap, AVCodecContext *video_c
     /* Disable vsync */
     cap_kms->egl.eglSwapInterval(cap_kms->egl.egl_display, 0);
 
-    video_codec_context->width = max_int(2, cap_kms->capture_size.x & ~1);
-    video_codec_context->height = max_int(2, cap_kms->capture_size.y & ~1);
+    video_codec_context->width = max_int(2, even_number_ceil(cap_kms->capture_size.x));
+    video_codec_context->height = max_int(2, even_number_ceil(cap_kms->capture_size.y));
 
     if(!drm_create_codec_context(cap_kms, video_codec_context)) {
         gsr_capture_kms_vaapi_stop(cap, video_codec_context);
diff --git a/src/capture/xcomposite_cuda.c b/src/capture/xcomposite_cuda.c
index 7cbc561..fd967a3 100644
--- a/src/capture/xcomposite_cuda.c
+++ b/src/capture/xcomposite_cuda.c
@@ -208,15 +208,15 @@ static int gsr_capture_xcomposite_cuda_start(gsr_capture *cap, AVCodecContext *v
     cap_xcomp->egl.glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &cap_xcomp->texture_size.y);
     cap_xcomp->egl.glBindTexture(GL_TEXTURE_2D, 0);
 
-    cap_xcomp->texture_size.x = max_int(2, cap_xcomp->texture_size.x & ~1);
-    cap_xcomp->texture_size.y = max_int(2, cap_xcomp->texture_size.y & ~1);
+    cap_xcomp->texture_size.x = max_int(2, even_number_ceil(cap_xcomp->texture_size.x));
+    cap_xcomp->texture_size.y = max_int(2, even_number_ceil(cap_xcomp->texture_size.y));
 
     video_codec_context->width = cap_xcomp->texture_size.x;
     video_codec_context->height = cap_xcomp->texture_size.y;
 
     if(cap_xcomp->params.region_size.x > 0 && cap_xcomp->params.region_size.y > 0) {
-        video_codec_context->width = max_int(2, cap_xcomp->params.region_size.x & ~1);
-        video_codec_context->height = max_int(2, cap_xcomp->params.region_size.y & ~1);
+        video_codec_context->width = max_int(2, even_number_ceil(cap_xcomp->params.region_size.x));
+        video_codec_context->height = max_int(2, even_number_ceil(cap_xcomp->params.region_size.y));
     }
 
     cap_xcomp->target_texture_id = gl_create_texture(cap_xcomp, video_codec_context->width, video_codec_context->height);
@@ -361,8 +361,8 @@ static void gsr_capture_xcomposite_cuda_tick(gsr_capture *cap, AVCodecContext *v
             cap_xcomp->egl.glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &cap_xcomp->texture_size.y);
             cap_xcomp->egl.glBindTexture(GL_TEXTURE_2D, 0);
 
-            cap_xcomp->texture_size.x = min_int(video_codec_context->width, max_int(2, cap_xcomp->texture_size.x & ~1));
-            cap_xcomp->texture_size.y = min_int(video_codec_context->height, max_int(2, cap_xcomp->texture_size.y & ~1));
+            cap_xcomp->texture_size.x = min_int(video_codec_context->width, max_int(2, even_number_ceil(cap_xcomp->texture_size.x)));
+            cap_xcomp->texture_size.y = min_int(video_codec_context->height, max_int(2, even_number_ceil(cap_xcomp->texture_size.y)));
         }
     }
 
@@ -384,8 +384,8 @@ static void gsr_capture_xcomposite_cuda_tick(gsr_capture *cap, AVCodecContext *v
         cap_xcomp->egl.glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &cap_xcomp->texture_size.y);
         cap_xcomp->egl.glBindTexture(GL_TEXTURE_2D, 0);
 
-        cap_xcomp->texture_size.x = min_int(video_codec_context->width, max_int(2, cap_xcomp->texture_size.x & ~1));
-        cap_xcomp->texture_size.y = min_int(video_codec_context->height, max_int(2, cap_xcomp->texture_size.y & ~1));
+        cap_xcomp->texture_size.x = min_int(video_codec_context->width, max_int(2, even_number_ceil(cap_xcomp->texture_size.x)));
+        cap_xcomp->texture_size.y = min_int(video_codec_context->height, max_int(2, even_number_ceil(cap_xcomp->texture_size.y)));
 
         if(!cap_xcomp->created_hw_frame) {
             cap_xcomp->created_hw_frame = true;
diff --git a/src/capture/xcomposite_vaapi.c b/src/capture/xcomposite_vaapi.c
index 99c4a50..752be69 100644
--- a/src/capture/xcomposite_vaapi.c
+++ b/src/capture/xcomposite_vaapi.c
@@ -176,21 +176,19 @@ static int gsr_capture_xcomposite_vaapi_start(gsr_capture *cap, AVCodecContext *
     cap_xcomp->texture_size.y = 0;
 
     cap_xcomp->egl.glBindTexture(GL_TEXTURE_2D, window_texture_get_opengl_texture_id(&cap_xcomp->window_texture));
-    cap_xcomp->texture_size.x = 0;
-    cap_xcomp->texture_size.y = 0;
     cap_xcomp->egl.glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &cap_xcomp->texture_size.x);
     cap_xcomp->egl.glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &cap_xcomp->texture_size.y);
     cap_xcomp->egl.glBindTexture(GL_TEXTURE_2D, 0);
 
-    cap_xcomp->texture_size.x = max_int(2, cap_xcomp->texture_size.x & ~1);
-    cap_xcomp->texture_size.y = max_int(2, cap_xcomp->texture_size.y & ~1);
+    cap_xcomp->texture_size.x = max_int(2, even_number_ceil(cap_xcomp->texture_size.x));
+    cap_xcomp->texture_size.y = max_int(2, even_number_ceil(cap_xcomp->texture_size.y));
 
     video_codec_context->width = cap_xcomp->texture_size.x;
     video_codec_context->height = cap_xcomp->texture_size.y;
 
     if(cap_xcomp->params.region_size.x > 0 && cap_xcomp->params.region_size.y > 0) {
-        video_codec_context->width = max_int(2, cap_xcomp->params.region_size.x & ~1);
-        video_codec_context->height = max_int(2, cap_xcomp->params.region_size.y & ~1);
+        video_codec_context->width = max_int(2, even_number_ceil(cap_xcomp->params.region_size.x));
+        video_codec_context->height = max_int(2, even_number_ceil(cap_xcomp->params.region_size.y));
     }
 
     if(!drm_create_codec_context(cap_xcomp, video_codec_context)) {
@@ -282,8 +280,8 @@ static void gsr_capture_xcomposite_vaapi_tick(gsr_capture *cap, AVCodecContext *
             cap_xcomp->egl.glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &cap_xcomp->texture_size.y);
             cap_xcomp->egl.glBindTexture(GL_TEXTURE_2D, 0);
 
-            cap_xcomp->texture_size.x = min_int(video_codec_context->width, max_int(2, cap_xcomp->texture_size.x & ~1));
-            cap_xcomp->texture_size.y = min_int(video_codec_context->height, max_int(2, cap_xcomp->texture_size.y & ~1));
+            cap_xcomp->texture_size.x = min_int(video_codec_context->width, max_int(2, even_number_ceil(cap_xcomp->texture_size.x)));
+            cap_xcomp->texture_size.y = min_int(video_codec_context->height, max_int(2, even_number_ceil(cap_xcomp->texture_size.y)));
         }
     }
 
@@ -306,8 +304,8 @@ static void gsr_capture_xcomposite_vaapi_tick(gsr_capture *cap, AVCodecContext *
         cap_xcomp->egl.glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &cap_xcomp->texture_size.y);
         cap_xcomp->egl.glBindTexture(GL_TEXTURE_2D, 0);
 
-        cap_xcomp->texture_size.x = min_int(video_codec_context->width, max_int(2, cap_xcomp->texture_size.x & ~1));
-        cap_xcomp->texture_size.y = min_int(video_codec_context->height, max_int(2, cap_xcomp->texture_size.y & ~1));
+        cap_xcomp->texture_size.x = min_int(video_codec_context->width, max_int(2, even_number_ceil(cap_xcomp->texture_size.x)));
+        cap_xcomp->texture_size.y = min_int(video_codec_context->height, max_int(2, even_number_ceil(cap_xcomp->texture_size.y)));
 
         if(cap_xcomp->buffer_id) {
             vaDestroyBuffer(cap_xcomp->va_dpy, cap_xcomp->buffer_id);
@@ -565,6 +563,7 @@ static int gsr_capture_xcomposite_vaapi_capture(gsr_capture *cap, AVFrame *frame
 static void gsr_capture_xcomposite_vaapi_stop(gsr_capture *cap, AVCodecContext *video_codec_context) {
     gsr_capture_xcomposite_vaapi *cap_xcomp = cap->priv;
 
+    // TODO: buffer_id == 0 is valid.. may be same for some of these other values. Handle that! also in other files
     if(cap_xcomp->buffer_id) {
         vaDestroyBuffer(cap_xcomp->va_dpy, cap_xcomp->buffer_id);
         cap_xcomp->buffer_id = 0;
diff --git a/src/utils.c b/src/utils.c
index 722fe06..27cca42 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -271,3 +271,7 @@ bool gsr_get_valid_card_path(char *output) {
     }
     return false;
 }
+
+int even_number_ceil(int value) {
+    return value + (value & 1);
+}