Better audio/video sync, disable broken screen-direct again (nvidia driver bug)

2 files changed, 27 insertions, 21 deletions

diff --git a/README.md b/README.md
index d0d5ea5..fcd5898 100644
--- a/README.md
+++ b/README.md
@@ -12,6 +12,8 @@ If you are using a variable refresh rate monitor, then choose to record "screen-
 For screen capture to work with PRIME (laptops with a nvidia gpu), you must set the primary GPU to use your dedicated nvidia graphics card. You can do this by selecting "NVIDIA (Performance Mode) in nvidia settings:\
 ![](https://dec05eba.com/images/nvidia-settings-prime.png)\
 and then rebooting your laptop.
+### TEMPORARY ISSUE ###
+screen-direct capture has been temporary disabled as it causes issues with stuttering. This might be a nvfbc bug.
 
 # Performance
 When recording Legend of Zelda Breath of the Wild at 4k, fps drops from 30 to 7 when using OBS Studio + nvenc, however when using this screen recorder the fps remains at 30.\
diff --git a/src/main.cpp b/src/main.cpp
index f5ffa85..85e25d2 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -518,7 +518,10 @@ static void receive_frames(AVCodecContext *av_codec_context, int stream_index, A
             av_packet.stream_index = stream_index;
             av_packet.pts = av_packet.dts = frame->pts;
 
-			std::lock_guard<std::mutex> lock(write_output_mutex);
+            if(frame->flags & AV_FRAME_FLAG_DISCARD)
+                av_packet.flags |= AV_PKT_FLAG_DISCARD;
+
+            std::lock_guard<std::mutex> lock(write_output_mutex);
             if(replay_buffer_size_secs != -1) {
                 double time_now = clock_get_monotonic_seconds();
                 double replay_time_elapsed = time_now - replay_start_time;
@@ -1364,8 +1367,12 @@ int main(int argc, char **argv) {
 
         const char *capture_target = window_str;
         bool direct_capture = strcmp(window_str, "screen-direct") == 0;
-        if(direct_capture)
+        if(direct_capture) {
             capture_target = "screen";
+            // TODO: Temporary disable direct capture because push model causes stuttering when it's direct capturing. This might be a nvfbc bug. This does not happen when using a compositor.
+            direct_capture = false;
+            fprintf(stderr, "Warning: screen-direct has temporary been disabled as it causes stuttering. This is likely a NvFBC bug. Falling back to \"screen\".\n");
+        }
 
         if(!nv_fbc_library.create(capture_target, fps, &window_width, &window_height, region_x, region_y, region_width, region_height, direct_capture))
             return 1;
@@ -1748,15 +1755,21 @@ int main(int argc, char **argv) {
                     break;
                 }
 
-                const int64_t num_missing_frames = std::round((this_audio_frame_time - received_audio_time) / target_audio_hz / (int64_t)audio_track.frame->nb_samples);
+                int64_t num_missing_frames = std::round((this_audio_frame_time - received_audio_time) / target_audio_hz / (int64_t)audio_track.frame->nb_samples);
+                if(got_audio_data)
+                    num_missing_frames = std::max((int64_t)0, num_missing_frames - 1);
+
+                if(!audio_track.sound_device.handle)
+                    num_missing_frames = std::max((int64_t)1, num_missing_frames);
+
                 // Jesus is there a better way to do this? I JUST WANT TO KEEP VIDEO AND AUDIO SYNCED HOLY FUCK I WANT TO KILL MYSELF NOW.
                 // THIS PIECE OF SHIT WANTS EMPTY FRAMES OTHERWISE VIDEO PLAYS TOO FAST TO KEEP UP WITH AUDIO OR THE AUDIO PLAYS TOO EARLY.
                 // BUT WE CANT USE DELAYS TO GIVE DUMMY DATA BECAUSE PULSEAUDIO MIGHT GIVE AUDIO A BIG DELAYED!!!
-                if(num_missing_frames >= 5 || (num_missing_frames > 0 && got_audio_data)) {
+                if(num_missing_frames >= 5 || !audio_track.sound_device.handle) {
                     // TODO:
                     //audio_track.frame->data[0] = empty_audio;
                     received_audio_time = this_audio_frame_time;
-                    swr_convert(swr, &audio_track.frame->data[0], audio_track.frame->nb_samples, (const uint8_t**)&empty_audio, audio_track.sound_device.frames);
+                    swr_convert(swr, &audio_track.frame->data[0], audio_track.frame->nb_samples, (const uint8_t**)&empty_audio, audio_track.codec_context->frame_size);
                     // TODO: Check if duplicate frame can be saved just by writing it with a different pts instead of sending it again
                     for(int i = 0; i < num_missing_frames; ++i) {
                         audio_track.frame->pts = pts;
@@ -1770,26 +1783,12 @@ int main(int argc, char **argv) {
                     }
                 }
 
-                if(!audio_track.sound_device.handle) {
-                    // TODO:
-                    //audio_track.frame->data[0] = empty_audio;
-                    received_audio_time = this_audio_frame_time;
-                    swr_convert(swr, &audio_track.frame->data[0], audio_track.frame->nb_samples, (const uint8_t**)&empty_audio, audio_track.codec_context->frame_size);
-                    audio_track.frame->pts = pts;
-                    pts += audio_track.frame->nb_samples;
-                    ret = avcodec_send_frame(audio_track.codec_context, audio_track.frame);
-                    if(ret >= 0){
-                        receive_frames(audio_track.codec_context, audio_track.stream_index, audio_track.stream, audio_track.frame, av_format_context, record_start_time, frame_data_queue, replay_buffer_size_secs, frames_erased, *write_output_mutex);
-                    } else {
-                        fprintf(stderr, "Failed to encode audio!\n");
-                    }
-
+                if(!audio_track.sound_device.handle)
                     usleep(timeout_ms * 1000);
-                }
 
                 if(got_audio_data) {
                     // TODO: Instead of converting audio, get float audio from alsa. Or does alsa do conversion internally to get this format?
-                    swr_convert(swr, &audio_track.frame->data[0], audio_track.frame->nb_samples, (const uint8_t**)&sound_buffer, audio_track.sound_device.frames);
+                    swr_convert(swr, &audio_track.frame->data[0], audio_track.frame->nb_samples, (const uint8_t**)&sound_buffer, audio_track.codec_context->frame_size);
 
                     audio_track.frame->pts = pts;
                     pts += audio_track.frame->nb_samples;
@@ -2022,8 +2021,13 @@ int main(int argc, char **argv) {
             const int64_t expected_frames = std::round((this_video_frame_time - start_time_pts) / target_fps);
 
             const int num_frames = std::max(0L, expected_frames - video_pts_counter);
+
+            frame->flags &= ~AV_FRAME_FLAG_DISCARD;
             // TODO: Check if duplicate frame can be saved just by writing it with a different pts instead of sending it again
             for(int i = 0; i < num_frames; ++i) {
+                if(i > 0)
+                    frame->flags |= AV_FRAME_FLAG_DISCARD;
+
                 frame->pts = video_pts_counter + i;
                 if (avcodec_send_frame(video_codec_context, frame) >= 0) {
                     receive_frames(video_codec_context, VIDEO_STREAM_INDEX, video_stream, frame, av_format_context,