Workaround amd driver bug: hevc ffmpeg/mesa misaligned resolution causing glitched video output on right/bottom side

4 files changed, 25 insertions, 26 deletions

diff --git a/include/utils.h b/include/utils.h
index 58ef099..c5d659a 100644
--- a/include/utils.h
+++ b/include/utils.h
@@ -41,6 +41,4 @@ bool gsr_get_valid_card_path(gsr_egl *egl, char *output, bool is_monitor_capture
 /* |render_path| should be at least 128 bytes in size */
 bool gsr_card_path_get_render_path(const char *card_path, char *render_path);
 
-int even_number_ceil(int value);
-
 #endif /* GSR_UTILS_H */
diff --git a/src/capture/kms.c b/src/capture/kms.c
index fed9e58..c2309d5 100644
--- a/src/capture/kms.c
+++ b/src/capture/kms.c
@@ -35,10 +35,6 @@ static void monitor_callback(const gsr_monitor *monitor, void *userdata) {
         fprintf(stderr, "gsr warning: reached max connector ids\n");
 }
 
-static int max_int(int a, int b) {
-    return a > b ? a : b;
-}
-
 int gsr_capture_kms_start(gsr_capture_kms *self, const char *display_to_capture, gsr_egl *egl, AVCodecContext *video_codec_context, AVFrame *frame) {
     memset(self, 0, sizeof(*self));
     self->base.video_codec_context = video_codec_context;
@@ -77,8 +73,14 @@ int gsr_capture_kms_start(gsr_capture_kms *self, const char *display_to_capture,
     /* Disable vsync */
     egl->eglSwapInterval(egl->egl_display, 0);
 
-    self->base.video_codec_context->width = max_int(2, even_number_ceil(self->capture_size.x));
-    self->base.video_codec_context->height = max_int(2, even_number_ceil(self->capture_size.y));
+    if(egl->gpu_info.vendor == GSR_GPU_VENDOR_AMD && video_codec_context->codec_id == AV_CODEC_ID_HEVC) {
+        // TODO: dont do this if using ffmpeg reports that this is not needed (AMD driver bug that was fixed recently)
+        self->base.video_codec_context->width = FFALIGN(self->capture_size.x, 64);
+        self->base.video_codec_context->height = FFALIGN(self->capture_size.y, 16);
+    } else {
+        self->base.video_codec_context->width = FFALIGN(self->capture_size.x, 2);
+        self->base.video_codec_context->height = FFALIGN(self->capture_size.y, 2);
+    }
 
     frame->width = self->base.video_codec_context->width;
     frame->height = self->base.video_codec_context->height;
diff --git a/src/capture/xcomposite.c b/src/capture/xcomposite.c
index 29b42d5..bb2b0e3 100644
--- a/src/capture/xcomposite.c
+++ b/src/capture/xcomposite.c
@@ -17,10 +17,6 @@ static int max_int(int a, int b) {
     return a > b ? a : b;
 }
 
-static int min_int(int a, int b) {
-    return a < b ? a : b;
-}
-
 void gsr_capture_xcomposite_init(gsr_capture_xcomposite *self, const gsr_capture_xcomposite_params *params) {
     memset(self, 0, sizeof(*self));
     self->params = *params;
@@ -102,15 +98,24 @@ int gsr_capture_xcomposite_start(gsr_capture_xcomposite *self, AVCodecContext *v
     self->params.egl->glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &self->texture_size.y);
     self->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
 
-    self->texture_size.x = max_int(2, even_number_ceil(self->texture_size.x));
-    self->texture_size.y = max_int(2, even_number_ceil(self->texture_size.y));
-
-    video_codec_context->width = self->texture_size.x;
-    video_codec_context->height = self->texture_size.y;
+    if(self->params.egl->gpu_info.vendor == GSR_GPU_VENDOR_AMD && video_codec_context->codec_id == AV_CODEC_ID_HEVC) {
+        // TODO: dont do this if using ffmpeg reports that this is not needed (AMD driver bug that was fixed recently)
+        video_codec_context->width = FFALIGN(self->texture_size.x, 64);
+        video_codec_context->height = FFALIGN(self->texture_size.y, 16);
+    } else {
+        video_codec_context->width = FFALIGN(self->texture_size.x, 2);
+        video_codec_context->height = FFALIGN(self->texture_size.y, 2);
+    }
 
     if(self->params.region_size.x > 0 && self->params.region_size.y > 0) {
-        video_codec_context->width = max_int(2, even_number_ceil(self->params.region_size.x));
-        video_codec_context->height = max_int(2, even_number_ceil(self->params.region_size.y));
+        if(self->params.egl->gpu_info.vendor == GSR_GPU_VENDOR_AMD && video_codec_context->codec_id == AV_CODEC_ID_HEVC) {
+            // TODO: dont do this if using ffmpeg reports that this is not needed (AMD driver bug that was fixed recently)
+            video_codec_context->width = FFALIGN(self->params.region_size.x, 64);
+            video_codec_context->height = FFALIGN(self->params.region_size.y, 16);
+        } else {
+            video_codec_context->width = FFALIGN(self->params.region_size.x, 2);
+            video_codec_context->height = FFALIGN(self->params.region_size.y, 2);
+        }
     }
 
     frame->width = video_codec_context->width;
@@ -128,6 +133,7 @@ void gsr_capture_xcomposite_stop(gsr_capture_xcomposite *self) {
 }
 
 void gsr_capture_xcomposite_tick(gsr_capture_xcomposite *self, AVCodecContext *video_codec_context) {
+    (void)video_codec_context;
     //self->params.egl->glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
     self->params.egl->glClear(0);
 
@@ -220,9 +226,6 @@ void gsr_capture_xcomposite_tick(gsr_capture_xcomposite *self, AVCodecContext *v
         self->params.egl->glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &self->texture_size.y);
         self->params.egl->glBindTexture(GL_TEXTURE_2D, 0);
 
-        self->texture_size.x = min_int(video_codec_context->width, max_int(2, even_number_ceil(self->texture_size.x)));
-        self->texture_size.y = min_int(video_codec_context->height, max_int(2, even_number_ceil(self->texture_size.y)));
-
         gsr_color_conversion_clear(&self->base.color_conversion);
     }
 }
diff --git a/src/utils.c b/src/utils.c
index b9995e8..8d726ef 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -453,7 +453,3 @@ bool gsr_card_path_get_render_path(const char *card_path, char *render_path) {
     close(fd);
     return false;
 }
-
-int even_number_ceil(int value) {
-    return value + (value & 1);
-}