1 files changed, 34 insertions, 193 deletions

diff --git a/src/encoder/video/vaapi.c b/src/encoder/video/vaapi.c
index 03218cb..d558785 100644
--- a/src/encoder/video/vaapi.c
+++ b/src/encoder/video/vaapi.c
@@ -4,9 +4,9 @@
 
 #include <libavcodec/avcodec.h>
 #include <libavutil/hwcontext_vaapi.h>
+#include <libavutil/intreadwrite.h>
 
 #include <va/va_drmcommon.h>
-#include <va/va_drm.h>
 
 #include <stdlib.h>
 #include <unistd.h>
@@ -102,6 +102,7 @@ static bool gsr_video_encoder_vaapi_setup_textures(gsr_video_encoder_vaapi *self
             uint32_t pitches[4];
             uint64_t modifiers[4];
             for(uint32_t j = 0; j < self->prime.layers[layer].num_planes; ++j) {
+                // TODO: Close these? in _stop, using self->prime
                 fds[j] = self->prime.objects[self->prime.layers[layer].object_index[j]].fd;
                 offsets[j] = self->prime.layers[layer].offset[j];
                 pitches[j] = self->prime.layers[layer].pitch[j];
@@ -147,194 +148,16 @@ static bool gsr_video_encoder_vaapi_setup_textures(gsr_video_encoder_vaapi *self
     }
 }
 
-static bool profile_is_h264(VAProfile profile) {
-    switch(profile) {
-        case 5: // VAProfileH264Baseline
-        case VAProfileH264Main:
-        case VAProfileH264High:
-        case VAProfileH264ConstrainedBaseline:
-            return true;
-        default:
-            return false;
-    }
-}
-
-static bool profile_is_hevc_8bit(VAProfile profile) {
-    switch(profile) {
-        case VAProfileHEVCMain:
-            return true;
-        default:
-            return false;
-    }
-}
-
-static bool profile_is_hevc_10bit(VAProfile profile) {
-    switch(profile) {
-        case VAProfileHEVCMain10:
-        //case VAProfileHEVCMain12:
-        //case VAProfileHEVCMain422_10:
-        //case VAProfileHEVCMain422_12:
-        //case VAProfileHEVCMain444:
-        //case VAProfileHEVCMain444_10:
-        //case VAProfileHEVCMain444_12:
-            return true;
-        default:
-            return false;
-    }
-}
-
-static bool profile_is_av1(VAProfile profile) {
-    switch(profile) {
-        case VAProfileAV1Profile0:
-        case VAProfileAV1Profile1:
-            return true;
-        default:
-            return false;
-    }
-}
-
-static bool profile_is_vp8(VAProfile profile) {
-    switch(profile) {
-        case VAProfileVP8Version0_3:
-            return true;
-        default:
-            return false;
-    }
-}
-
-static bool profile_is_vp9(VAProfile profile) {
-    switch(profile) {
-        case VAProfileVP9Profile0:
-        case VAProfileVP9Profile1:
-        case VAProfileVP9Profile2:
-        case VAProfileVP9Profile3:
-            return true;
-        default:
-            return false;
-    }
-}
-
-static bool profile_supports_video_encoding(VADisplay va_dpy, VAProfile profile) {
-    int num_entrypoints = vaMaxNumEntrypoints(va_dpy);
-    if(num_entrypoints <= 0)
-        return false;
-
-    VAEntrypoint *entrypoint_list = calloc(num_entrypoints, sizeof(VAEntrypoint));
-    if(!entrypoint_list)
-        return false;
-
-    bool supported = false;
-    if(vaQueryConfigEntrypoints(va_dpy, profile, entrypoint_list, &num_entrypoints) == VA_STATUS_SUCCESS) {
-        for(int i = 0; i < num_entrypoints; ++i) {
-            if(entrypoint_list[i] == VAEntrypointEncSlice) {
-                supported = true;
-                break;
-            }
-        }
-    }
-
-    free(entrypoint_list);
-    return supported;
-}
-
-static bool get_supported_video_codecs(VADisplay va_dpy, gsr_supported_video_codecs *video_codecs, bool cleanup) {
-    *video_codecs = (gsr_supported_video_codecs){0};
-    bool success = false;
-    VAProfile *profile_list = NULL;
-
-    vaSetInfoCallback(va_dpy, NULL, NULL);
-
-    int va_major = 0;
-    int va_minor = 0;
-    if(vaInitialize(va_dpy, &va_major, &va_minor) != VA_STATUS_SUCCESS) {
-        fprintf(stderr, "gsr error: gsr_video_encoder_vaapi_get_supported_codecs: vaInitialize failed\n");
-        goto fail;
-    }
-
-    int num_profiles = vaMaxNumProfiles(va_dpy);
-    if(num_profiles <= 0)
-        goto fail;
-
-    profile_list = calloc(num_profiles, sizeof(VAProfile));
-    if(!profile_list || vaQueryConfigProfiles(va_dpy, profile_list, &num_profiles) != VA_STATUS_SUCCESS)
-        goto fail;
-
-    for(int i = 0; i < num_profiles; ++i) {
-        if(profile_is_h264(profile_list[i])) {
-            if(profile_supports_video_encoding(va_dpy, profile_list[i]))
-                video_codecs->h264 = true;
-        } else if(profile_is_hevc_8bit(profile_list[i])) {
-            if(profile_supports_video_encoding(va_dpy, profile_list[i]))
-                video_codecs->hevc = true;
-        } else if(profile_is_hevc_10bit(profile_list[i])) {
-            if(profile_supports_video_encoding(va_dpy, profile_list[i])) {
-                video_codecs->hevc_hdr = true;
-                video_codecs->hevc_10bit = true;
-            }
-        } else if(profile_is_av1(profile_list[i])) {
-            if(profile_supports_video_encoding(va_dpy, profile_list[i])) {
-                video_codecs->av1 = true;
-                video_codecs->av1_hdr = true;
-                video_codecs->av1_10bit = true;
-            }
-        } else if(profile_is_vp8(profile_list[i])) {
-            if(profile_supports_video_encoding(va_dpy, profile_list[i]))
-                video_codecs->vp8 = true;
-        } else if(profile_is_vp9(profile_list[i])) {
-            if(profile_supports_video_encoding(va_dpy, profile_list[i]))
-                video_codecs->vp9 = true;
-        }
-    }
-
-    success = true;
-    fail:
-    if(profile_list)
-        free(profile_list);
-
-    if(cleanup)
-        vaTerminate(va_dpy);
-
-    return success;
-}
-
-static gsr_supported_video_codecs gsr_video_encoder_vaapi_get_supported_codecs(gsr_video_encoder *encoder, bool cleanup) {
-    gsr_video_encoder_vaapi *encoder_vaapi = encoder->priv;
-    gsr_supported_video_codecs supported_video_codecs = {0};
-
-    char render_path[128];
-    if(!gsr_card_path_get_render_path(encoder_vaapi->params.egl->card_path, render_path)) {
-        fprintf(stderr, "gsr error: gsr_video_encoder_vaapi_get_supported_codecs: failed to get /dev/dri/renderDXXX file from %s\n", encoder_vaapi->params.egl->card_path);
-        return supported_video_codecs;
-    }
-
-    const int drm_fd = open(render_path, O_RDWR);
-    if(drm_fd == -1) {
-        fprintf(stderr, "gsr error: gsr_video_encoder_vaapi_get_supported_codecs: failed to open device %s\n", render_path);
-        return supported_video_codecs;
-    }
-
-    VADisplay va_dpy = vaGetDisplayDRM(drm_fd);
-    if(va_dpy) {
-        if(!get_supported_video_codecs(va_dpy, &supported_video_codecs, cleanup))
-            fprintf(stderr, "gsr error: gsr_video_encoder_vaapi_get_supported_codecs: failed to query supported video codecs for device %s\n", render_path);
-    }
-
-    if(cleanup)
-        close(drm_fd);
-
-    return supported_video_codecs;
-}
-
 static void gsr_video_encoder_vaapi_stop(gsr_video_encoder_vaapi *self, AVCodecContext *video_codec_context);
 
 static bool gsr_video_encoder_vaapi_start(gsr_video_encoder *encoder, AVCodecContext *video_codec_context, AVFrame *frame) {
-    gsr_video_encoder_vaapi *encoder_vaapi = encoder->priv;
+    gsr_video_encoder_vaapi *self = encoder->priv;
 
-    if(encoder_vaapi->params.egl->gpu_info.vendor == GSR_GPU_VENDOR_AMD && video_codec_context->codec_id == AV_CODEC_ID_HEVC) {
+    if(self->params.egl->gpu_info.vendor == GSR_GPU_VENDOR_AMD && video_codec_context->codec_id == AV_CODEC_ID_HEVC) {
         // TODO: dont do this if using ffmpeg reports that this is not needed (AMD driver bug that was fixed recently)
         video_codec_context->width = FFALIGN(video_codec_context->width, 64);
         video_codec_context->height = FFALIGN(video_codec_context->height, 16);
-    } else if(encoder_vaapi->params.egl->gpu_info.vendor == GSR_GPU_VENDOR_AMD && video_codec_context->codec_id == AV_CODEC_ID_AV1) {
+    } else if(self->params.egl->gpu_info.vendor == GSR_GPU_VENDOR_AMD && video_codec_context->codec_id == AV_CODEC_ID_AV1) {
         // TODO: Dont do this for VCN 5 and forward which should fix this hardware bug
         video_codec_context->width = FFALIGN(video_codec_context->width, 64);
         // AMD driver has special case handling for 1080 height to set it to 1082 instead of 1088 (1080 aligned to 16).
@@ -346,20 +169,40 @@ static bool gsr_video_encoder_vaapi_start(gsr_video_encoder *encoder, AVCodecCon
         }
     }
 
-    if(video_codec_context->width != frame->width || video_codec_context->height != frame->height) {
+    const int crop_top = (video_codec_context->height - frame->height) / 2;
+    const int crop_left = (video_codec_context->width - frame->width) / 2;
+    if(crop_top != 0 || crop_left != 0) {
         fprintf(stderr, "gsr warning: gsr_video_encoder_vaapi_start: black bars have been added to the video because of a bug in AMD drivers/hardware. Record with h264 codec instead (-k h264) to get around this issue\n");
+#if 0
+        #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(61, 10, 100)
+        const int crop_bottom = crop_top;
+        const int crop_right = crop_left;
+        fprintf(stderr, "gsr info: cropping metadata has been added to the file to try and workaround this issue. Video players that support this will remove the black bars when the video is playing\n");
+        const int frame_cropping_data_size = 4 * sizeof(uint32_t);
+        uint8_t *frame_cropping = av_malloc(frame_cropping_data_size);
+        if(frame_cropping) {
+            AV_WL32(frame_cropping + 0, crop_top);
+            AV_WL32(frame_cropping + 4, crop_bottom);
+            AV_WL32(frame_cropping + 8, crop_left);
+            AV_WL32(frame_cropping + 12, crop_right);
+            const bool sidedata_added = av_packet_side_data_add(&video_stream->codecpar->coded_side_data, &video_stream->codecpar->nb_coded_side_data, AV_PKT_DATA_FRAME_CROPPING, frame_cropping, frame_cropping_data_size, 0) != NULL;
+            if(!sidedata_added)
+                av_free(frame_cropping);
+        }
+        #endif
+#endif
     }
 
     frame->width = video_codec_context->width;
     frame->height = video_codec_context->height;
 
-    if(!gsr_video_encoder_vaapi_setup_context(encoder_vaapi, video_codec_context)) {
-        gsr_video_encoder_vaapi_stop(encoder_vaapi, video_codec_context);
+    if(!gsr_video_encoder_vaapi_setup_context(self, video_codec_context)) {
+        gsr_video_encoder_vaapi_stop(self, video_codec_context);
         return false;
     }
 
-    if(!gsr_video_encoder_vaapi_setup_textures(encoder_vaapi, video_codec_context, frame)) {
-        gsr_video_encoder_vaapi_stop(encoder_vaapi, video_codec_context);
+    if(!gsr_video_encoder_vaapi_setup_textures(self, video_codec_context, frame)) {
+        gsr_video_encoder_vaapi_stop(self, video_codec_context);
         return false;
     }
 
@@ -385,11 +228,11 @@ void gsr_video_encoder_vaapi_stop(gsr_video_encoder_vaapi *self, AVCodecContext
 }
 
 static void gsr_video_encoder_vaapi_get_textures(gsr_video_encoder *encoder, unsigned int *textures, int *num_textures, gsr_destination_color *destination_color) {
-    gsr_video_encoder_vaapi *encoder_vaapi = encoder->priv;
-    textures[0] = encoder_vaapi->target_textures[0];
-    textures[1] = encoder_vaapi->target_textures[1];
+    gsr_video_encoder_vaapi *self = encoder->priv;
+    textures[0] = self->target_textures[0];
+    textures[1] = self->target_textures[1];
     *num_textures = 2;
-    *destination_color = encoder_vaapi->params.color_depth == GSR_COLOR_DEPTH_10_BITS ? GSR_DESTINATION_COLOR_P010 : GSR_DESTINATION_COLOR_NV12;
+    *destination_color = self->params.color_depth == GSR_COLOR_DEPTH_10_BITS ? GSR_DESTINATION_COLOR_P010 : GSR_DESTINATION_COLOR_NV12;
 }
 
 static void gsr_video_encoder_vaapi_destroy(gsr_video_encoder *encoder, AVCodecContext *video_codec_context) {
@@ -412,9 +255,7 @@ gsr_video_encoder* gsr_video_encoder_vaapi_create(const gsr_video_encoder_vaapi_
     encoder_vaapi->params = *params;
 
     *encoder = (gsr_video_encoder) {
-        .get_supported_codecs = gsr_video_encoder_vaapi_get_supported_codecs,
         .start = gsr_video_encoder_vaapi_start,
-        .copy_textures_to_frame = NULL,
         .get_textures = gsr_video_encoder_vaapi_get_textures,
         .destroy = gsr_video_encoder_vaapi_destroy,
         .priv = encoder_vaapi