From e671784144174c4fceaa6df3737ba9b4de4a6c63 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 17 Jul 2021 09:43:20 +0200 Subject: Youtube: remove dependency on youtube-dl for downloads (also fixes downloads of age restricted videos) --- src/QuickMedia.cpp | 443 ++++++++++++++++------------------------------------- 1 file changed, 134 insertions(+), 309 deletions(-) (limited to 'src/QuickMedia.cpp') diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 64833bd..fb6b425 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -29,6 +29,7 @@ #include "../include/Utils.hpp" #include "../include/Tabs.hpp" #include "../include/Theme.hpp" +#include "../include/Downloader.hpp" #include "../plugins/youtube/YoutubeMediaProxy.hpp" #include "../include/gui/Button.hpp" #include "../external/hash-library/sha256.h" @@ -387,7 +388,6 @@ namespace QuickMedia { Window parent_window = None; std::vector tabs; const char *url = nullptr; - bool download_use_youtube_dl = false; std::string program_path = dirname(argv[0]); for(int i = 1; i < argc; ++i) { @@ -425,8 +425,6 @@ namespace QuickMedia { } } else if(strcmp(argv[i], "--low-cpu-mode") == 0) { low_cpu_mode = true; - } else if(strcmp(argv[i], "--youtube-dl") == 0) { - download_use_youtube_dl = true; } else if(strcmp(argv[i], "-u") == 0) { if(i < argc - 1) { url = argv[i + 1]; @@ -533,7 +531,7 @@ namespace QuickMedia { usage(); return -1; } - download_page(url, download_use_youtube_dl); + download_page(url); return exit_code; } @@ -2404,9 +2402,17 @@ namespace QuickMedia { return false; } - void Program::video_page_download_video(const std::string &url, bool use_youtube_dl, sf::WindowHandle video_player_window) { - if(!use_youtube_dl) { - download_async_gui(url, file_manager_start_dir.string(), use_youtube_dl, no_video); + static bool url_should_download_with_youtube_dl(const std::string &url) { + return url.find("pornhub.com") != std::string::npos || url.find("xhamster.com") != std::string::npos || url.find("spankbang.com") != std::string::npos || url.find("xvideos.com") != std::string::npos; + } + + void Program::video_page_download_video(const std::string &url, sf::WindowHandle video_player_window) { + bool separate_audio_option = url_should_download_with_youtube_dl(url);; + std::string video_id; + separate_audio_option |= youtube_url_extract_id(url, video_id); + + if(!separate_audio_option) { + download_async_gui(url, file_manager_start_dir.string(), no_video); return; } @@ -2438,7 +2444,7 @@ namespace QuickMedia { if(!selected) return; - download_async_gui(url, file_manager_start_dir.string(), true, audio_only); + download_async_gui(url, file_manager_start_dir.string(), audio_only); } bool Program::video_download_if_non_streamable(std::string &video_url, std::string &audio_url, bool &is_audio_only, bool &has_embedded_audio, PageType previous_page) { @@ -2542,8 +2548,8 @@ namespace QuickMedia { std::unique_ptr youtube_video_media_proxy; std::unique_ptr youtube_audio_media_proxy; AsyncTask youtube_downloader_task; - int youtube_video_content_length = 0; - int youtube_audio_content_length = 0; + int64_t youtube_video_content_length = 0; + int64_t youtube_audio_content_length = 0; std::string channel_url; AsyncTask video_tasks; @@ -2585,11 +2591,12 @@ namespace QuickMedia { if(video_page->load(new_title, channel_url, media_chapters) != PluginResult::OK) return false; + std::string ext; if(!no_video) - video_url = video_page->get_video_url(largest_monitor_height, has_embedded_audio); + video_url = video_page->get_video_url(largest_monitor_height, has_embedded_audio, ext); if(video_url.empty() || no_video) { - video_url = video_page->get_audio_url(); + video_url = video_page->get_audio_url(ext); if(video_url.empty()) { video_url = video_page->get_url(); has_embedded_audio = true; @@ -2598,7 +2605,7 @@ namespace QuickMedia { has_embedded_audio = false; } } else if(!has_embedded_audio) { - audio_url = video_page->get_audio_url(); + audio_url = video_page->get_audio_url(ext); } if(!is_youtube && download_if_streaming_fails) { @@ -2677,7 +2684,7 @@ namespace QuickMedia { struct MediaProxyMetadata { std::unique_ptr *media_proxy; std::string *url; - int content_length; + int64_t content_length; }; MediaProxyMetadata media_proxies[2] = { @@ -2876,7 +2883,7 @@ namespace QuickMedia { } else if(pressed_keysym == XK_f && pressing_ctrl) { window_set_fullscreen(disp, window.getSystemHandle(), WindowFullscreenState::TOGGLE); } else if(pressed_keysym == XK_s && pressing_ctrl) { - video_page_download_video(video_page->get_url(), !is_matrix || is_youtube, video_player_window); + video_page_download_video(video_page->get_url(), video_player_window); } else if(pressed_keysym == XK_F5) { in_seeking = false; double resume_start_time = 0.0; @@ -3954,7 +3961,7 @@ namespace QuickMedia { } else if(event.key.code == sf::Keyboard::S && event.key.control) { BodyItem *selected_item = thread_body->get_selected(); if(selected_item && !selected_item->url.empty()) - download_async_gui(selected_item->url, file_manager_start_dir.string(), false, false); + download_async_gui(selected_item->url, file_manager_start_dir.string(), false); } BodyItem *selected_item = thread_body->get_selected(); @@ -4071,7 +4078,7 @@ namespace QuickMedia { redraw = true; frame_skip_text_entry = true; } else if(event.key.code == sf::Keyboard::S && event.key.control) { - download_async_gui(attached_image_url, file_manager_start_dir.string(), false, false); + download_async_gui(attached_image_url, file_manager_start_dir.string(), false); } } } @@ -5437,7 +5444,7 @@ namespace QuickMedia { avatar_applied = false; return true; } else if(message_type == MessageType::FILE) { - download_async_gui(selected->url, file_manager_start_dir.string(), false, no_video); + download_async_gui(selected->url, file_manager_start_dir.string(), no_video); return true; } @@ -5478,7 +5485,7 @@ namespace QuickMedia { if(selected_item_message) { MessageType message_type = selected_item_message->type; if(!selected->url.empty() && message_type >= MessageType::IMAGE && message_type <= MessageType::FILE) { - download_async_gui(selected->url, file_manager_start_dir.string(), false, no_video); + download_async_gui(selected->url, file_manager_start_dir.string(), no_video); return true; } } @@ -6456,290 +6463,6 @@ namespace QuickMedia { matrix->stop_sync(); } - enum class DownloadUpdateStatus { - DOWNLOADING, - FINISHED, - ERROR - }; - - class Downloader { - public: - Downloader(const std::string &url, const std::string &output_filepath) : url(url), output_filepath(output_filepath) {} - virtual ~Downloader() = default; - - virtual bool start() = 0; - virtual void stop(bool download_completed) = 0; - virtual DownloadUpdateStatus update() = 0; - - virtual float get_progress() = 0; - virtual std::string get_progress_text() = 0; - virtual std::string get_download_speed_text() = 0; - protected: - std::string url; - std::string output_filepath; - }; - - class CurlDownloader : public Downloader { - public: - CurlDownloader(const std::string &url, const std::string &output_filepath) : Downloader(url, output_filepath) { - output_filepath_tmp = output_filepath; - output_filepath_tmp.append(".tmp"); - read_program.pid = -1; - read_program.read_fd = -1; - progress_text = "0 bytes/Unknown"; - download_speed_text = "Unknown/s"; - } - - bool start() override { - remove(output_filepath_tmp.data.c_str()); - - const char *args[] = { "curl", - "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", - "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", - "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), - "-D", "/dev/stdout", - "--", url.c_str(), nullptr }; - - if(exec_program_pipe(args, &read_program) != 0) - return false; - - // TODO: Remove this async task and make the fd non blocking instead - header_reader = AsyncTask([this]{ - char tmp_buf[1024]; - while(true) { - ssize_t bytes_available = read(read_program.read_fd, tmp_buf, sizeof(tmp_buf)); - if(bytes_available == -1) { - return false; - } else if(bytes_available > 0 && content_length == (size_t)-1) { - header.append(tmp_buf, bytes_available); - if(header.find("\r\n\r\n") != std::string::npos) { - std::string content_length_str = header_extract_value(header, "content-length"); - if(!content_length_str.empty()) { - errno = 0; - char *endptr; - const long content_length_tmp = strtol(content_length_str.c_str(), &endptr, 10); - if(endptr != content_length_str.c_str() && errno == 0) { - std::lock_guard lock(content_length_mutex); - content_length = content_length_tmp; - } - } - } - } - } - return true; - }); - - return true; - } - - void stop(bool download_completed) override { - if(read_program.read_fd != -1) - close(read_program.read_fd); - if(read_program.pid != -1) - kill(read_program.pid, SIGTERM); - if(!download_completed) - remove(output_filepath_tmp.data.c_str()); - //header_reader.cancel(); - } - - DownloadUpdateStatus update() override { - int status = 0; - if(wait_program_non_blocking(read_program.pid, &status)) { - read_program.pid = -1; - if(status == 0 && rename_atomic(output_filepath_tmp.data.c_str(), output_filepath.c_str()) == 0) { - return DownloadUpdateStatus::FINISHED; - } else { - return DownloadUpdateStatus::ERROR; - } - } - - if(header_reader.ready()) { - if(!header_reader.get()) - return DownloadUpdateStatus::ERROR; - } - - std::lock_guard lock(content_length_mutex); - size_t output_file_size = 0; - file_get_size(output_filepath_tmp, &output_file_size); - size_t downloaded_size = std::min(output_file_size, content_length); - - if(content_length == (size_t)-1) { - progress_text = std::to_string(output_file_size / 1024) + "/Unknown"; - } else { - size_t percentage = 0; - if(output_file_size > 0) - percentage = (double)downloaded_size / (double)content_length * 100.0; - progress = (double)percentage / 100.0; - progress_text = file_size_to_human_readable_string(downloaded_size) + "/" + file_size_to_human_readable_string(content_length) + " (" + std::to_string(percentage) + "%)"; - } - - // TODO: Take into consideration time overflow? - size_t downloaded_diff = std::max(0lu, downloaded_size - downloaded_since_last_check); - download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s"; - downloaded_since_last_check = downloaded_size; - - return DownloadUpdateStatus::DOWNLOADING; - } - - float get_progress() override { - return progress; - } - - std::string get_progress_text() override { - return progress_text; - } - - std::string get_download_speed_text() override { - return download_speed_text; - } - private: - Path output_filepath_tmp; - ReadProgram read_program; - AsyncTask header_reader; - std::string header; - size_t content_length = (size_t)-1; - size_t downloaded_since_last_check = 0; - float progress = 0.0f; - std::mutex content_length_mutex; - std::string progress_text; - std::string download_speed_text; - }; - - class YoutubeDlDownloader : public Downloader { - public: - YoutubeDlDownloader(const std::string &url, const std::string &output_filepath, bool no_video) : Downloader(url, output_filepath), no_video(no_video) { - // youtube-dl requires a file extension for the file - if(this->output_filepath.find('.') == std::string::npos) - this->output_filepath += ".mkv"; - - read_program.pid = -1; - read_program.read_fd = -1; - progress_text = "0.0% of Unknown"; - download_speed_text = "Unknown/s"; - } - - bool start() override { - remove(output_filepath.c_str()); - - std::vector args = { "youtube-dl", "--no-warnings", "--no-continue", "--output", output_filepath.c_str(), "--newline" }; - if(no_video) { - args.push_back("-f"); - args.push_back("bestaudio/best"); - args.push_back("-x"); - } else { - args.push_back("-f"); - args.push_back("bestvideo+bestaudio/best"); - } - args.insert(args.end(), { "--", url.c_str(), nullptr }); - - if(exec_program_pipe(args.data(), &read_program) != 0) - return false; - - read_program_file = fdopen(read_program.read_fd, "rb"); - if(!read_program_file) { - wait_program(read_program.pid); - return false; - } - - // TODO: Remove this async task and make the fd non blocking instead - youtube_dl_output_reader = AsyncTask([this]{ - char line[128]; - char progress_c[10]; - char content_size_c[20]; - char download_speed_c[20]; - - while(true) { - if(fgets(line, sizeof(line), read_program_file)) { - int len = strlen(line); - if(len > 0 && line[len - 1] == '\n') { - line[len - 1] = '\0'; - --len; - } - - if(sscanf(line, "[download] %10s of %20s at %20s", progress_c, content_size_c, download_speed_c) == 3) { - std::lock_guard lock(progress_update_mutex); - - if(strcmp(progress_c, "Unknown") != 0 && strcmp(content_size_c, "Unknown") != 0) { - std::string progress_str = progress_c; - progress_text = progress_str + " of " + content_size_c; - if(progress_str.back() == '%') { - errno = 0; - char *endptr; - const double progress_tmp = strtod(progress_str.c_str(), &endptr); - if(endptr != progress_str.c_str() && errno == 0) - progress = progress_tmp / 100.0; - } - } - - if(strcmp(download_speed_c, "Unknown") == 0) - download_speed_text = "Unknown/s"; - else - download_speed_text = download_speed_c; - } - } else { - return false; - } - } - - return true; - }); - - return true; - } - - void stop(bool) override { - if(read_program_file) - fclose(read_program_file); - if(read_program.pid != -1) - kill(read_program.pid, SIGTERM); - // TODO: Remove the temporary files created by youtube-dl (if !download_completed) - //header_reader.cancel(); - } - - DownloadUpdateStatus update() override { - int status = 0; - if(wait_program_non_blocking(read_program.pid, &status)) { - read_program.pid = -1; - if(status == 0) { - return DownloadUpdateStatus::FINISHED; - } else { - return DownloadUpdateStatus::ERROR; - } - } - - if(youtube_dl_output_reader.ready()) { - if(!youtube_dl_output_reader.get()) - return DownloadUpdateStatus::ERROR; - } - - return DownloadUpdateStatus::DOWNLOADING; - } - - float get_progress() override { - std::lock_guard lock(progress_update_mutex); - return progress; - } - - std::string get_progress_text() override { - std::lock_guard lock(progress_update_mutex); - return progress_text; - } - - std::string get_download_speed_text() override { - std::lock_guard lock(progress_update_mutex); - return download_speed_text; - } - private: - ReadProgram read_program; - FILE *read_program_file = nullptr; - AsyncTask youtube_dl_output_reader; - std::mutex progress_update_mutex; - float progress = 0.0f; - std::string progress_text; - std::string download_speed_text; - bool no_video; - }; - static int accumulate_string(char *data, int size, void *userdata) { std::string *str = (std::string*)userdata; if(str->size() + size > 1024 * 1024 * 100) // 100mb sane limit, TODO: make configurable @@ -6748,12 +6471,27 @@ namespace QuickMedia { return 0; } - void Program::download_page(const char *url, bool download_use_youtube_dl) { + void Program::download_page(const std::string &url) { window.setTitle("QuickMedia - Select where you want to save " + std::string(url)); + const bool download_use_youtube_dl = url_should_download_with_youtube_dl(url); std::string filename; + std::string video_id; + const bool url_is_youtube = youtube_url_extract_id(url, video_id); + std::unique_ptr youtube_video_page; + + std::string video_url; + std::string audio_url; + int64_t video_content_length = 0; + int64_t audio_content_length = 0; + TaskResult task_result; if(download_use_youtube_dl) { + if(!is_program_executable_by_name("youtube-dl")) { + show_notification("QuickMedia", "youtube-dl needs to be installed to download the video/music", Urgency::CRITICAL); + abort(); + } + task_result = run_task_with_loading_screen([this, url, &filename]{ std::string json_str; std::vector args = { "youtube-dl", "--skip-download", "--print-json", "--no-warnings" }; @@ -6765,7 +6503,7 @@ namespace QuickMedia { args.push_back("-f"); args.push_back("bestvideo+bestaudio/best"); } - args.insert(args.end(), { "--", url, nullptr }); + args.insert(args.end(), { "--", url.c_str(), nullptr }); if(exec_program(args.data(), accumulate_string, &json_str) != 0) return false; @@ -6791,6 +6529,77 @@ namespace QuickMedia { return !filename.empty(); }); + } else if(url_is_youtube) { + youtube_video_page = std::make_unique(this, url); + bool cancelled = false; + bool load_successful = false; + const int largest_monitor_height = get_largest_monitor_height(disp); + + for(int i = 0; i < 3; ++i) { + task_result = run_task_with_loading_screen([this, &youtube_video_page, &filename, &video_url, &audio_url, &video_content_length, &audio_content_length, largest_monitor_height, &cancelled]{ + std::string channel_url; + std::vector chapters; + filename.clear(); + if(youtube_video_page->load(filename, channel_url, chapters) != PluginResult::OK) + return false; + + std::string ext; + bool has_embedded_audio = true; + video_url = no_video ? "" : youtube_video_page->get_video_url(largest_monitor_height, has_embedded_audio, ext); + audio_url.clear(); + + if(!has_embedded_audio || no_video) + audio_url = youtube_video_page->get_audio_url(ext); + + if(video_url.empty() && audio_url.empty()) + return false; + + if(!youtube_url_is_live_stream(video_url) && !youtube_url_is_live_stream(audio_url)) { + video_content_length = 0; + audio_content_length = 0; + std::string new_video_url = video_url; + std::string new_audio_url = audio_url; + auto current_thread_id = std::this_thread::get_id(); + if(!youtube_custom_redirect(new_video_url, new_audio_url, video_content_length, audio_content_length, [current_thread_id]{ return !program_is_dead_in_thread(current_thread_id); })) { + if(program_is_dead_in_current_thread()) + cancelled = true; + return false; + } + + video_url = std::move(new_video_url); + audio_url = std::move(new_audio_url); + } + + if(!video_url.empty() && !audio_url.empty()) + filename += ".mkv"; + else + filename += ext; + + return true; + }); + + if(task_result == TaskResult::CANCEL || cancelled) { + exit_code = 1; + return; + } else if(task_result == TaskResult::FALSE) { + continue; + } + + load_successful = true; + break; + } + + if(!load_successful) { + show_notification("QuickMedia", "Download failed", Urgency::CRITICAL); + exit_code = 1; + return; + } + + if(youtube_url_is_live_stream(video_url) || youtube_url_is_live_stream(audio_url)) { + show_notification("QuickMedia", "Downloading youtube live streams is currently not supported", Urgency::CRITICAL); + exit_code = 1; + return; + } } else { task_result = run_task_with_loading_screen([url, &filename]{ return url_get_remote_name(url, filename, true) == DownloadResult::OK; @@ -6800,6 +6609,10 @@ namespace QuickMedia { if(task_result == TaskResult::CANCEL) { exit_code = 1; return; + } else if(task_result == TaskResult::FALSE) { + show_notification("QuickMedia", "Download failed", Urgency::CRITICAL); + exit_code = 1; + return; } std::string output_filepath = file_save_page(filename); @@ -6862,10 +6675,21 @@ namespace QuickMedia { sf::Event event; std::unique_ptr downloader; - if(download_use_youtube_dl) + if(download_use_youtube_dl) { downloader = std::make_unique(url, output_filepath, no_video); - else + } else if(url_is_youtube) { + MediaMetadata video_metadata; + video_metadata.url = std::move(video_url); + video_metadata.content_length = video_content_length; + + MediaMetadata audio_metadata; + audio_metadata.url = std::move(audio_url); + audio_metadata.content_length = audio_content_length; + + downloader = std::make_unique(video_metadata, audio_metadata, output_filepath); + } else { downloader = std::make_unique(url, output_filepath); + } if(!downloader->start()) { show_notification("QuickMedia", std::string("Failed to download ") + url + " to " + output_filepath, Urgency::CRITICAL); @@ -6908,6 +6732,7 @@ namespace QuickMedia { download_completed = true; goto cleanup; case DownloadUpdateStatus::ERROR: + fprintf(stderr, "Download error on update\n"); goto cleanup; } @@ -6962,8 +6787,8 @@ namespace QuickMedia { } cleanup: - downloader->stop(download_completed); - if(download_completed) { + const bool stop_successful = downloader->stop(download_completed); + if(download_completed && stop_successful) { show_notification("QuickMedia", std::string("Download finished! Downloaded ") + Path(filename).filename() + " to " + output_filepath); exit_code = 0; } else { -- cgit v1.2.3