From f9a41041ec3c2f88033f1ec0dd32ed0270b75c2f Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sun, 21 Jul 2024 19:12:06 +0200 Subject: Fix HDR capture (HDR metadata is now correct). Note that HDR capture is only available on Wayland and when recording a monitor without the desktop portal option --- README.md | 3 +-- TODO | 9 +++++---- include/capture/capture.h | 5 +++-- src/capture/capture.c | 4 ++-- src/capture/kms.c | 40 ++++++++++++++++++++++++++++++---------- src/capture/nvfbc.c | 3 ++- src/capture/portal.c | 3 ++- src/capture/xcomposite.c | 3 ++- src/dbus.c | 4 +--- src/main.cpp | 5 ++--- 10 files changed, 50 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index e18e74a..07c3c09 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,7 @@ This software works on x11 and wayland. ### TEMPORARY ISSUES 1) screen-direct capture has been temporary disabled as it causes issues with stuttering. This might be a nvfbc bug. 2) Videos are in variable framerate format. Use MPV to play such videos, otherwise you might experience stuttering in the video if you are using a buggy video player. You can try saving the video into a .mkv file instead as some software may have better support for .mkv files (such as kdenlive). You can use the "-fm cfr" option to to use constant framerate mode. -3) HDR capture is supported (on wayland), but all GPU drivers have bugs that ignore HDR metadata so the HDR metadata will be missing in the video file. I will eventually patch the video file to workaround these GPU driver issues. -4) FLAC audio codec is disabled at the moment because of temporary issues. +3) FLAC audio codec is disabled at the moment because of temporary issues. ### AMD/Intel/Wayland root permission When recording a window under AMD/Intel no special user permission is required, however when recording a monitor (or when using wayland) the program needs root permission (to access KMS).\ This is safe in GPU Screen Recorder as the part that needs root access has been moved to its own small program that only does one thing.\ diff --git a/TODO b/TODO index 0b3d700..a878acd 100644 --- a/TODO +++ b/TODO @@ -121,8 +121,6 @@ Go back to using pure vaapi without opengl for video encoding? rotation (transpo Implement scaling and use lanczos resampling for better quality. Lanczos resampling can also be used for YUV chroma for better color quality on small text. -Try fixing HDR by passing HDR+10 data as well, and in the packet. Run "ffprobe -loglevel quiet -read_intervals "%+#2" -select_streams v:0 -show_entries side_data video.mp4" to test if the file has correct metadata. - Flac is disabled because the frame sizes are too large which causes big audio/video desync. Add 10-bit capture option. This is good because it reduces banding and quality in very dark areas while reducing the file size compared to doing the same thing with 8-bits. @@ -139,7 +137,6 @@ Remove is_damaged and clear_damage and return a value from capture function inst When adding support for steam deck, add option to send video to another computer. New gpu screen recorder gui should have the option to cut the video directly, maybe running an ffmpeg command or implementing that ourselves. Only support gpu screen recorder video files. -Add hdr metadata to encoder settings metadata. Check if is software renderer by using eglQueryDisplayAttribEXT(egl_display, EGL_DEVICE_EXT..) eglQueryDeviceStringEXT(egl_device, EGL_EXTENSIONS) and check for "EGL_MESA_device_software". @@ -153,4 +150,8 @@ Cleanup pipewire code and add more error checks. Detect if recording monitor on intel and plane is compressed. In that case give an error and tell the user to use -w portal instead. -Make dbus code and pipewire setup non blocking. \ No newline at end of file +Make dbus code and pipewire setup non blocking. + +Support portal (pipewire) hdr capture when pipewire adds support for it. + +HDR support on x11? \ No newline at end of file diff --git a/include/capture/capture.h b/include/capture/capture.h index 026a955..1e7b25f 100644 --- a/include/capture/capture.h +++ b/include/capture/capture.h @@ -5,6 +5,7 @@ #include typedef struct AVCodecContext AVCodecContext; +typedef struct AVStream AVStream; typedef struct AVFrame AVFrame; typedef struct gsr_capture gsr_capture; @@ -15,7 +16,7 @@ struct gsr_capture { bool (*is_damaged)(gsr_capture *cap); /* can be NULL */ void (*clear_damage)(gsr_capture *cap); /* can be NULL */ bool (*should_stop)(gsr_capture *cap, bool *err); /* can be NULL. If NULL, return false */ - int (*capture)(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *color_conversion); + int (*capture)(gsr_capture *cap, AVStream *video_stream, AVFrame *frame, gsr_color_conversion *color_conversion); void (*capture_end)(gsr_capture *cap, AVFrame *frame); /* can be NULL */ gsr_source_color (*get_source_color)(gsr_capture *cap); bool (*uses_external_image)(gsr_capture *cap); /* can be NULL. If NULL, return false */ @@ -28,7 +29,7 @@ struct gsr_capture { int gsr_capture_start(gsr_capture *cap, AVCodecContext *video_codec_context, AVFrame *frame); void gsr_capture_tick(gsr_capture *cap, AVCodecContext *video_codec_context); bool gsr_capture_should_stop(gsr_capture *cap, bool *err); -int gsr_capture_capture(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *color_conversion); +int gsr_capture_capture(gsr_capture *cap, AVStream *video_stream, AVFrame *frame, gsr_color_conversion *color_conversion); void gsr_capture_capture_end(gsr_capture *cap, AVFrame *frame); gsr_source_color gsr_capture_get_source_color(gsr_capture *cap); bool gsr_capture_uses_external_image(gsr_capture *cap); diff --git a/src/capture/capture.c b/src/capture/capture.c index 40507bf..283c0a1 100644 --- a/src/capture/capture.c +++ b/src/capture/capture.c @@ -24,9 +24,9 @@ bool gsr_capture_should_stop(gsr_capture *cap, bool *err) { return false; } -int gsr_capture_capture(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *color_conversion) { +int gsr_capture_capture(gsr_capture *cap, AVStream *video_stream, AVFrame *frame, gsr_color_conversion *color_conversion) { assert(cap->started); - return cap->capture(cap, frame, color_conversion); + return cap->capture(cap, video_stream, frame, color_conversion); } void gsr_capture_capture_end(gsr_capture *cap, AVFrame *frame) { diff --git a/src/capture/kms.c b/src/capture/kms.c index f522df3..c078b90 100644 --- a/src/capture/kms.c +++ b/src/capture/kms.c @@ -12,6 +12,7 @@ #include #include +#include #define HDMI_STATIC_METADATA_TYPE1 0 #define HDMI_EOTF_SMPTE_ST2084 2 @@ -35,6 +36,8 @@ typedef struct { AVMasteringDisplayMetadata *mastering_display_metadata; AVContentLightMetadata *light_metadata; + size_t light_metadata_size; + bool hdr_metadata_set; gsr_monitor_rotation monitor_rotation; @@ -229,12 +232,26 @@ static bool hdr_metadata_is_supported_format(const struct hdr_output_metadata *h hdr_metadata->hdmi_metadata_type1.eotf == HDMI_EOTF_SMPTE_ST2084; } -static void gsr_kms_set_hdr_metadata(gsr_capture_kms *self, AVFrame *frame, gsr_kms_response_fd *drm_fd) { - if(!self->mastering_display_metadata) - self->mastering_display_metadata = av_mastering_display_metadata_create_side_data(frame); +static void gsr_kms_set_hdr_metadata(gsr_capture_kms *self, AVStream *video_stream, gsr_kms_response_fd *drm_fd) { + if(self->hdr_metadata_set) + return; if(!self->light_metadata) - self->light_metadata = av_content_light_metadata_create_side_data(frame); + self->light_metadata = av_content_light_metadata_alloc(&self->light_metadata_size); + + if(!self->mastering_display_metadata) + self->mastering_display_metadata = av_mastering_display_metadata_alloc(); + + if(self->light_metadata) { + self->light_metadata->MaxCLL = drm_fd->hdr_metadata.hdmi_metadata_type1.max_cll; + self->light_metadata->MaxFALL = drm_fd->hdr_metadata.hdmi_metadata_type1.max_fall; + + #if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(60, 31, 102) + av_stream_add_side_data(video_stream, AV_PKT_DATA_CONTENT_LIGHT_LEVEL, self->light_metadata, self->light_metadata_size); + #else + av_packet_side_data_add(&video_stream->codecpar->coded_side_data, &video_stream->codecpar->nb_coded_side_data, AV_PKT_DATA_CONTENT_LIGHT_LEVEL, self->light_metadata, self->light_metadata_size, 0); + #endif + } if(self->mastering_display_metadata) { for(int i = 0; i < 3; ++i) { @@ -250,12 +267,15 @@ static void gsr_kms_set_hdr_metadata(gsr_capture_kms *self, AVFrame *frame, gsr_ self->mastering_display_metadata->has_primaries = self->mastering_display_metadata->display_primaries[0][0].num > 0; self->mastering_display_metadata->has_luminance = self->mastering_display_metadata->max_luminance.num > 0; - } - if(self->light_metadata) { - self->light_metadata->MaxCLL = drm_fd->hdr_metadata.hdmi_metadata_type1.max_cll; - self->light_metadata->MaxFALL = drm_fd->hdr_metadata.hdmi_metadata_type1.max_fall; + #if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(60, 31, 102) + av_stream_add_side_data(video_stream, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, self->mastering_display_metadata, sizeof(*self->mastering_display_metadata)); + #else + av_packet_side_data_add(&video_stream->codecpar->coded_side_data, &video_stream->codecpar->nb_coded_side_data, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, self->mastering_display_metadata, sizeof(*self->mastering_display_metadata), 0); + #endif } + + self->hdr_metadata_set = true; } static vec2i swap_vec2i(vec2i value) { @@ -283,7 +303,7 @@ static bool is_plane_compressed(uint64_t modifier) { return false; } -static int gsr_capture_kms_capture(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *color_conversion) { +static int gsr_capture_kms_capture(gsr_capture *cap, AVStream *video_stream, AVFrame *frame, gsr_color_conversion *color_conversion) { gsr_capture_kms *self = cap->priv; const bool screen_plane_use_modifiers = self->params.egl->gpu_info.vendor != GSR_GPU_VENDOR_AMD; const bool cursor_texture_id_is_external = self->params.egl->gpu_info.vendor == GSR_GPU_VENDOR_NVIDIA; @@ -334,7 +354,7 @@ static int gsr_capture_kms_capture(gsr_capture *cap, AVFrame *frame, gsr_color_c cursor_drm_fd = NULL; if(drm_fd->has_hdr_metadata && self->params.hdr && hdr_metadata_is_supported_format(&drm_fd->hdr_metadata)) - gsr_kms_set_hdr_metadata(self, frame, drm_fd); + gsr_kms_set_hdr_metadata(self, video_stream, drm_fd); // TODO: This causes a crash sometimes on steam deck, why? is it a driver bug? a vaapi pure version doesn't cause a crash. // Even ffmpeg kmsgrab causes this crash. The error is: diff --git a/src/capture/nvfbc.c b/src/capture/nvfbc.c index 97e0283..4bf6186 100644 --- a/src/capture/nvfbc.c +++ b/src/capture/nvfbc.c @@ -378,8 +378,9 @@ static int gsr_capture_nvfbc_start(gsr_capture *cap, AVCodecContext *video_codec return -1; } -static int gsr_capture_nvfbc_capture(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *color_conversion) { +static int gsr_capture_nvfbc_capture(gsr_capture *cap, AVStream *video_stream, AVFrame *frame, gsr_color_conversion *color_conversion) { gsr_capture_nvfbc *cap_nvfbc = cap->priv; + (void)video_stream; const double nvfbc_recreate_retry_time_seconds = 1.0; if(cap_nvfbc->nvfbc_needs_recreate) { diff --git a/src/capture/portal.c b/src/capture/portal.c index 6263b8b..9bac3cf 100644 --- a/src/capture/portal.c +++ b/src/capture/portal.c @@ -287,7 +287,8 @@ static int max_int(int a, int b) { return a > b ? a : b; } -static int gsr_capture_portal_capture(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *color_conversion) { +static int gsr_capture_portal_capture(gsr_capture *cap, AVStream *video_stream, AVFrame *frame, gsr_color_conversion *color_conversion) { + (void)video_stream; (void)frame; (void)color_conversion; gsr_capture_portal *self = cap->priv; diff --git a/src/capture/xcomposite.c b/src/capture/xcomposite.c index 83c4800..d9cb595 100644 --- a/src/capture/xcomposite.c +++ b/src/capture/xcomposite.c @@ -323,8 +323,9 @@ static bool gsr_capture_xcomposite_should_stop(gsr_capture *cap, bool *err) { return false; } -static int gsr_capture_xcomposite_capture(gsr_capture *cap, AVFrame *frame, gsr_color_conversion *color_conversion) { +static int gsr_capture_xcomposite_capture(gsr_capture *cap, AVStream *video_stream, AVFrame *frame, gsr_color_conversion *color_conversion) { gsr_capture_xcomposite *self = cap->priv; + (void)video_stream; (void)frame; //self->params.egl->glClearColor(0.0f, 0.0f, 0.0f, 1.0f); diff --git a/src/dbus.c b/src/dbus.c index b67632b..5757b8b 100644 --- a/src/dbus.c +++ b/src/dbus.c @@ -868,9 +868,7 @@ int gsr_dbus_screencast_start(gsr_dbus *self, const char *session_handle, uint32 bool gsr_dbus_screencast_open_pipewire_remote(gsr_dbus *self, const char *session_handle, int *pipewire_fd) { assert(session_handle); *pipewire_fd = -1; - - dict_entry args[1]; - return gsr_dbus_call_screencast_method(self, "OpenPipeWireRemote", session_handle, NULL, args, 0, pipewire_fd, NULL); + return gsr_dbus_call_screencast_method(self, "OpenPipeWireRemote", session_handle, NULL, NULL, 0, pipewire_fd, NULL); } const char* gsr_dbus_screencast_get_restore_token(gsr_dbus *self) { diff --git a/src/main.cpp b/src/main.cpp index 4b3cd0c..355789f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1047,8 +1047,7 @@ static void usage_full() { fprintf(stderr, "\n"); fprintf(stderr, " -k Video codec to use. Should be either 'auto', 'h264', 'hevc', 'av1', 'hevc_hdr', 'av1_hdr', 'vp8' or 'vp9'. Optional, set to 'auto' by default which defaults to 'h264'.\n"); fprintf(stderr, " Forcefully set to 'h264' if the file container type is 'flv'.\n"); - fprintf(stderr, " 'hevc_hdr' and 'av1_hdr' option is not available on X11.\n"); - fprintf(stderr, " Note: hdr metadata is not included in the video when recording with 'hevc_hdr'/'av1_hdr' because of bugs in AMD, Intel and NVIDIA drivers (amazin', they are all bugged).\n"); + fprintf(stderr, " 'hevc_hdr' and 'av1_hdr' option is not available on X11 nor when using the portal capture option.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -ac Audio codec to use. Should be either 'aac', 'opus' or 'flac'. Optional, set to 'opus' for .mp4/.mkv files, otherwise set to 'aac'.\n"); fprintf(stderr, " 'opus' and 'flac' is only supported by .mp4/.mkv files. 'opus' is recommended for best performance and smallest audio size.\n"); @@ -3099,7 +3098,7 @@ int main(int argc, char **argv) { const int num_frames = framerate_mode == FramerateMode::CONSTANT ? std::max((int64_t)0LL, expected_frames - video_pts_counter) : 1; if(num_frames > 0 && !paused) { - gsr_capture_capture(capture, video_frame, &color_conversion); + gsr_capture_capture(capture, video_stream, video_frame, &color_conversion); gsr_video_encoder_copy_textures_to_frame(video_encoder, video_frame); // TODO: Check if duplicate frame can be saved just by writing it with a different pts instead of sending it again -- cgit v1.2.3