Temporary depend on yt-dlp for youtube since youtube broke (throttle sig)

5 files changed, 93 insertions, 138 deletions

diff --git a/src/Downloader.cpp b/src/Downloader.cpp
index 57a0349..75cef1c 100644
--- a/src/Downloader.cpp
+++ b/src/Downloader.cpp
@@ -2,6 +2,7 @@
 #include "../include/Storage.hpp"
 #include "../include/NetUtils.hpp"
 #include "../include/Notification.hpp"
+#include "../include/StringUtils.hpp"
 #include <unistd.h>
 #include <signal.h>
 #include <sys/wait.h>
@@ -147,7 +148,44 @@ namespace QuickMedia {
         return download_speed_text;
     }
 
-    YoutubeDlDownloader::YoutubeDlDownloader(const std::string &url, const std::string &output_filepath, bool no_video) : Downloader(url, output_filepath), no_video(no_video) {
+    static bool parse_ytdl_download_line(const char *line, size_t size, std::string &progress, std::string &content_size, std::string &download_speed) {
+        progress.clear();
+        content_size.clear();
+        download_speed.clear();
+
+        const std::string_view line_v(line, size);
+        size_t index = line_v.find("[download]", 0);
+        if(index == std::string::npos)
+            return false;
+
+        index += 10;
+        size_t end_index = line_v.find(" of ~ ", index);
+        if(end_index == std::string::npos)
+            return false;
+
+        progress = line_v.substr(index, end_index - index);
+        index = end_index + 6;
+
+        end_index = line_v.find(" at", index);
+        if(end_index == std::string::npos)
+            return false;
+
+        content_size = line_v.substr(index, end_index - index);
+        index = end_index + 3;
+
+        end_index = line_v.find(" ETA ", index);
+        if(end_index == std::string::npos)
+            return false;
+
+        download_speed = line_v.substr(index, end_index - index);
+
+        progress = strip(progress);
+        content_size = strip(content_size);
+        download_speed = strip(download_speed);
+        return true;
+    }
+
+    YoutubeDlDownloader::YoutubeDlDownloader(const char *yt_dl_name, const std::string &url, const std::string &output_filepath, bool no_video) : Downloader(url, output_filepath), no_video(no_video), yt_dl_name(yt_dl_name) {
         // youtube-dl requires a file extension for the file
         if(this->output_filepath.find('.') == std::string::npos)
             this->output_filepath += ".mkv";
@@ -161,7 +199,7 @@ namespace QuickMedia {
     bool YoutubeDlDownloader::start() {
         remove(output_filepath.c_str());
 
-        std::vector<const char*> args = { "youtube-dl", "--no-warnings", "--no-continue", "--output", output_filepath.c_str(), "--newline" };
+        std::vector<const char*> args = { yt_dl_name, "--no-warnings", "--no-continue", "--output", output_filepath.c_str(), "--newline" };
         if(no_video) {
             args.push_back("-f");
             args.push_back("bestaudio/best");
@@ -185,9 +223,9 @@ namespace QuickMedia {
         // TODO: Remove this async task and make the fd non blocking instead
         youtube_dl_output_reader = AsyncTask<bool>([this]{
             char line[128];
-            char progress_c[11];
-            char content_size_c[21];
-            char download_speed_c[21];
+            std::string progress_c;
+            std::string content_size_c;
+            std::string download_speed_c;
 
             while(true) {
                 if(fgets(line, sizeof(line), read_program_file)) {
@@ -197,10 +235,10 @@ namespace QuickMedia {
                         --len;
                     }
 
-                    if(sscanf(line, "[download] %10s of %20s at %20s", progress_c, content_size_c, download_speed_c) == 3) {
+                    if(parse_ytdl_download_line(line, len, progress_c, content_size_c, download_speed_c)) {
                         std::lock_guard<std::mutex> lock(progress_update_mutex);
 
-                        if(strcmp(progress_c, "Unknown") != 0 && strcmp(content_size_c, "Unknown") != 0) {
+                        if(strcmp(progress_c.c_str(), "Unknown") != 0 && strcmp(content_size_c.c_str(), "Unknown") != 0) {
                             std::string progress_str = progress_c;
                             progress_text = progress_str + " of " + content_size_c;
                             if(progress_str.back() == '%') {
@@ -212,7 +250,7 @@ namespace QuickMedia {
                             }
                         }
 
-                        if(strcmp(download_speed_c, "Unknown") == 0)
+                        if(strcmp(download_speed_c.c_str(), "Unknown") == 0)
                             download_speed_text = "Unknown/s";
                         else
                             download_speed_text = download_speed_c;
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index f91ef7e..b1bbcde 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -1080,10 +1080,17 @@ namespace QuickMedia {
             .related_media_thumbnail_handler({{"//div[data-role='video-relations']//img", "src", "xhcdn"}});
     }
 
-    static void check_youtube_dl_installed(const std::string &plugin_name) {
-        if(!is_program_executable_by_name("youtube-dl")) {
-            show_notification("QuickMedia", "youtube-dl needs to be installed to play " + plugin_name + " videos", Urgency::CRITICAL);
-            abort();
+    void Program::check_youtube_dl_installed(const std::string &plugin_name) {
+        if(yt_dl_name)
+            return;
+
+        if(is_program_executable_by_name("yt-dlp")) {
+            yt_dl_name = "yt-dlp";
+        } else if(is_program_executable_by_name("youtube-dl")) {
+            yt_dl_name = "youtube-dl";
+        } else {
+            show_notification("QuickMedia", "yt-dlp or youtube-dl needs to be installed to play " + plugin_name + " videos", Urgency::CRITICAL);
+            exit(10);
         }
     }
 
@@ -1322,6 +1329,8 @@ namespace QuickMedia {
             pipe_body->set_items(std::move(body_items));
             tabs.push_back(Tab{std::move(pipe_body), std::make_unique<PipePage>(this), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
         } else if(strcmp(plugin_name, "youtube") == 0) {
+            check_youtube_dl_installed(plugin_name);
+            use_youtube_dl = true;
             if(launch_url_type == LaunchUrlType::YOUTUBE_CHANNEL) {
                 YoutubeChannelPage::create_each_type(this, std::move(launch_url), "", "Channel", tabs);
             } else if(launch_url_type == LaunchUrlType::YOUTUBE_VIDEO) {
@@ -3031,6 +3040,8 @@ namespace QuickMedia {
         return url.find("pornhub.com") != std::string::npos
             || url.find("xhamster.com") != std::string::npos
             || url.find("spankbang.com") != std::string::npos
+            || url.find("youtube.com") != std::string::npos
+            || url.find("youtu.be") != std::string::npos
             // TODO: Remove when youtube-dl is no longer required to download soundcloud music
             || is_soundcloud(url);
     }
@@ -3215,9 +3226,11 @@ namespace QuickMedia {
         mgl::Clock update_window_focus_time; // HACK!
 
         std::string youtube_video_id_dummy;
-        const bool is_youtube = youtube_url_extract_id(video_page->get_url(), youtube_video_id_dummy);
+        //const bool is_youtube = youtube_url_extract_id(video_page->get_url(), youtube_video_id_dummy);
         const bool is_matrix = strcmp(plugin_name, "matrix") == 0;
         const bool is_youtube_plugin = strcmp(plugin_name, "youtube") == 0;
+        const bool is_youtube = false;
+        const bool is_youtube_rel = youtube_url_extract_id(video_page->get_url(), youtube_video_id_dummy);
 
         bool added_recommendations = false;
         mgl::Clock time_watched_timer;
@@ -3739,7 +3752,7 @@ namespace QuickMedia {
                 current_page = previous_page;
                 go_to_previous_page = true;
                 break;
-            } else if(update_err == VideoPlayer::Error::EXITED && video_player->exit_status == 0 && (!is_matrix || is_youtube)) {
+            } else if(update_err == VideoPlayer::Error::EXITED && video_player->exit_status == 0 && (!is_matrix || is_youtube_rel)) {
                 std::string new_video_url;
 
                 if(!video_page->should_autoplay()) {
@@ -7960,7 +7973,8 @@ namespace QuickMedia {
         const bool download_use_youtube_dl = url_should_download_with_youtube_dl(url);
         std::string filename;
         std::string video_id;
-        const bool url_is_youtube = youtube_url_extract_id(url, video_id);
+        //const bool url_is_youtube = youtube_url_extract_id(url, video_id);
+        const bool url_is_youtube = false;
         std::unique_ptr<YoutubeVideoPage> youtube_video_page;
 
         std::string video_url;
@@ -7970,14 +7984,18 @@ namespace QuickMedia {
 
         TaskResult task_result = TaskResult::TRUE;
         if(download_use_youtube_dl) {
-            if(!is_program_executable_by_name("youtube-dl")) {
-                show_notification("QuickMedia", "youtube-dl needs to be installed to download the video/music", Urgency::CRITICAL);
-                abort();
+            if(is_program_executable_by_name("yt-dlp")) {
+                yt_dl_name = "yt-dlp";
+            } else if(is_program_executable_by_name("youtube-dl")) {
+                yt_dl_name = "youtube-dl";
+            } else {
+                show_notification("QuickMedia", "yt-dlp or youtube-dl needs to be installed to download the video/music", Urgency::CRITICAL);
+                exit(10);
             }
 
             task_result = run_task_with_loading_screen([this, url, &filename]{
                 std::string json_str;
-                std::vector<const char*> args = { "youtube-dl", "--skip-download", "--print-json", "--no-warnings" };
+                std::vector<const char*> args = { yt_dl_name, "--skip-download", "--print-json", "--no-warnings" };
                 if(no_video) {
                     args.push_back("-f");
                     args.push_back("bestaudio/best");
@@ -8012,78 +8030,6 @@ namespace QuickMedia {
 
                 return !filename.empty();
             });
-        } else if(url_is_youtube) {
-            youtube_video_page = std::make_unique<YoutubeVideoPage>(this, url);
-            bool cancelled = false;
-            bool load_successful = false;
-            const int video_max_height = video_get_max_height();
-
-            std::string err_str;
-            for(int i = 0; i < 3; ++i) {
-                task_result = run_task_with_loading_screen([&]{
-                    VideoInfo video_info;
-                    SubmitArgs submit_args;
-                    if(youtube_video_page->load(submit_args, video_info, err_str) != PluginResult::OK)
-                        return false;
-
-                    filename = video_info.title;
-                    std::string ext;
-                    bool has_embedded_audio = true;
-                    video_url = no_video ? "" : youtube_video_page->get_video_url(video_max_height, has_embedded_audio, ext);
-                    audio_url.clear();
-                    
-                    if(!has_embedded_audio || no_video)
-                        audio_url = youtube_video_page->get_audio_url(ext);
-
-                    if(video_url.empty() && audio_url.empty())
-                        return false;
-
-                    if(!youtube_url_is_live_stream(video_url) && !youtube_url_is_live_stream(audio_url)) {
-                        video_content_length = 0;
-                        audio_content_length = 0;
-                        std::string new_video_url = video_url;
-                        std::string new_audio_url = audio_url;
-                        auto current_thread_id = std::this_thread::get_id();
-                        if(!youtube_custom_redirect(new_video_url, new_audio_url, video_content_length, audio_content_length, [current_thread_id]{ return !program_is_dead_in_thread(current_thread_id); })) {
-                            if(program_is_dead_in_current_thread())
-                                cancelled = true;
-                            return false;
-                        }
-
-                        video_url = std::move(new_video_url);
-                        audio_url = std::move(new_audio_url);
-                    }
-
-                    if(!video_url.empty() && !audio_url.empty())
-                        filename += ".mkv";
-                    else
-                        filename += ext;
-
-                    return true;
-                });
-
-                if(task_result == TaskResult::CANCEL || cancelled) {
-                    exit_code = 1;
-                    return;
-                } else if(task_result == TaskResult::FALSE) {
-                    continue;
-                }
-
-                load_successful = true;
-                break;
-            }
-
-            if(!load_successful) {
-                show_notification("QuickMedia", "Download failed" + (err_str.empty() ? "" : ", error: " + err_str), Urgency::CRITICAL);
-                exit_code = 1;
-                return;
-            }
-
-            if(youtube_url_is_live_stream(video_url) || youtube_url_is_live_stream(audio_url)) {
-                show_notification("QuickMedia", "Downloading youtube live streams is currently not supported", Urgency::CRITICAL);
-                exit_code = 1;
-                return;
-            }
         } else {
             if(download_filename.empty()) {
                 task_result = run_task_with_loading_screen([url, &filename]{
@@ -8173,17 +8119,7 @@ namespace QuickMedia {
 
         std::unique_ptr<Downloader> downloader;
         if(download_use_youtube_dl) {
-            downloader = std::make_unique<YoutubeDlDownloader>(url, output_filepath, no_video);
-        } else if(url_is_youtube) {
-            MediaMetadata video_metadata;
-            video_metadata.url = std::move(video_url);
-            video_metadata.content_length = video_content_length;
-
-            MediaMetadata audio_metadata;
-            audio_metadata.url = std::move(audio_url);
-            audio_metadata.content_length = audio_content_length;
-
-            downloader = std::make_unique<YoutubeDownloader>(video_metadata, audio_metadata, output_filepath);
+            downloader = std::make_unique<YoutubeDlDownloader>(yt_dl_name, url, output_filepath, no_video);
         } else {
             downloader = std::make_unique<CurlDownloader>(url, output_filepath);
         }
diff --git a/src/VideoPlayer.cpp b/src/VideoPlayer.cpp
index 8614e94..32eda39 100644
--- a/src/VideoPlayer.cpp
+++ b/src/VideoPlayer.cpp
@@ -169,8 +169,9 @@ namespace QuickMedia {
         if(get_file_type(video_player_filepath.c_str()) != FileType::REGULAR)
             video_player_filepath = "/usr/bin/quickmedia-video-player";
 
-        std::string config_dir = "--config-dir=" + startup_args.resource_root + "mpv";
-        std::string input_conf_file = "--input-conf=" + startup_args.resource_root + "mpv/input.conf";
+        const std::string config_dir = "--config-dir=" + startup_args.resource_root + "mpv";
+        const std::string input_conf_file = "--input-conf=" + startup_args.resource_root + "mpv/input.conf";
+        const std::string ytdl_hook_file = "--scripts=" + startup_args.resource_root + "mpv/scripts/ytdl_hook.lua";
 
         std::vector<const char*> args;
         // TODO: Resume playback if the last video played matches the first video played next time QuickMedia is launched
@@ -228,10 +229,7 @@ namespace QuickMedia {
         else
             ytdl_format = "--ytdl-format=bestvideo[height<=?" + std::to_string(startup_args.monitor_height) + "]+bestaudio/best";
 
-        if(!startup_args.use_youtube_dl)
-            args.push_back("--ytdl=no");
-        else
-            args.push_back(ytdl_format.c_str());
+        args.push_back("--ytdl=no");
 
         // TODO: Properly escape referer quotes
         std::string referer_arg = "--http-header-fields=Referer: " + startup_args.referer;
@@ -248,6 +246,9 @@ namespace QuickMedia {
             args.push_back("--load-scripts=yes");
             args.push_back("--osc=yes");
             args.push_back(input_conf_file.c_str());
+            
+            if(startup_args.use_youtube_dl)
+                args.push_back(ytdl_hook_file.c_str());
         } else {
             args.insert(args.end(), {
                 config_dir.c_str(),
diff --git a/src/plugins/MediaGeneric.cpp b/src/plugins/MediaGeneric.cpp
index cc869ef..b2d2538 100644
--- a/src/plugins/MediaGeneric.cpp
+++ b/src/plugins/MediaGeneric.cpp
@@ -220,7 +220,7 @@ namespace QuickMedia {
         // TODO: Use max_height, if possible
         (void)max_height;
         has_embedded_audio = true;
-        ext = "m3u8";
+        ext = ".m3u8";
         return video_url;
     }
 
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index 9b64054..33e4cea 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -1568,18 +1568,6 @@ namespace QuickMedia {
         return fetch_comments(this, video_url, continuation_token, result_items);
     }
 
-    static const char* youtube_channel_page_type_to_api_params(YoutubeChannelPage::Type type) {
-        switch(type) {
-            case YoutubeChannelPage::Type::VIDEOS:
-                return "EgZ2aWRlb3PyBgQKAjoA";
-            case YoutubeChannelPage::Type::SHORTS:
-                return "EgZzaG9ydHPyBgUKA5oBAA%3D%3D";
-            case YoutubeChannelPage::Type::LIVE:
-                return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D";
-        }
-        return "";
-    }
-
     static const char* youtube_channel_page_type_get_endpoint(YoutubeChannelPage::Type type) {
         switch(type) {
             case YoutubeChannelPage::Type::VIDEOS:
@@ -2359,7 +2347,15 @@ namespace QuickMedia {
         return nullptr;
     }
 
+    std::string YoutubeVideoPage::get_download_url(int max_height) {
+        return url;
+    }
+
     std::string YoutubeVideoPage::get_video_url(int max_height, bool &has_embedded_audio, std::string &ext) {
+        has_embedded_audio = true;
+        ext = ".mp4";
+        return url;
+#if 0
         if(!livestream_url.empty()) {
             has_embedded_audio = true;
             return livestream_url;
@@ -2392,23 +2388,7 @@ namespace QuickMedia {
             ext = ".webm";
 
         return chosen_video_format->base.url;
-    }
-
-    std::string YoutubeVideoPage::get_audio_url(std::string &ext) {
-        if(audio_formats.empty())
-            return "";
-
-        const YoutubeAudioFormat *chosen_audio_format = &audio_formats.front();
-        fprintf(stderr, "Choosing youtube audio format: bitrate: %d, mime type: %s\n", chosen_audio_format->base.bitrate, chosen_audio_format->base.mime_type.c_str());
-
-        if(chosen_audio_format->base.mime_type.find("mp4") != std::string::npos)
-            ext = ".m4a";
-        else if(chosen_audio_format->base.mime_type.find("webm") != std::string::npos)
-            ext = ".opus"; // TODO: Detect if vorbis (.ogg) or opus (.opus)
-        else if(chosen_audio_format->base.mime_type.find("opus") != std::string::npos)
-            ext = ".opus";
-
-        return chosen_audio_format->base.url;
+#endif
     }
 
     // Returns -1 if timestamp is in an invalid format