From e671784144174c4fceaa6df3737ba9b4de4a6c63 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 17 Jul 2021 09:43:20 +0200 Subject: Youtube: remove dependency on youtube-dl for downloads (also fixes downloads of age restricted videos) --- src/AsyncImageLoader.cpp | 6 +- src/DownloadUtils.cpp | 3 +- src/Downloader.cpp | 475 ++++++++++++++++++++++++++++++ src/QuickMedia.cpp | 443 +++++++++------------------- src/plugins/Youtube.cpp | 31 +- src/plugins/youtube/YoutubeMediaProxy.cpp | 28 +- 6 files changed, 645 insertions(+), 341 deletions(-) create mode 100644 src/Downloader.cpp (limited to 'src') diff --git a/src/AsyncImageLoader.cpp b/src/AsyncImageLoader.cpp index 3e00459..ccf9f42 100644 --- a/src/AsyncImageLoader.cpp +++ b/src/AsyncImageLoader.cpp @@ -21,11 +21,7 @@ namespace QuickMedia { static bool webp_to_png(const Path &thumbnail_path, const Path &destination_path) { const char *args[] = { "ffmpeg", "-y", "-v", "quiet", "-i", thumbnail_path.data.c_str(), "--", destination_path.data.c_str(), nullptr}; - int res = exec_program(args, nullptr, nullptr); - if(res != 0) - return false; - - return true; + return exec_program(args, nullptr, nullptr) == 0; } bool create_thumbnail(const Path &thumbnail_path, const Path &thumbnail_path_resized, sf::Vector2i resize_target_size, ContentType content_type) { diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp index dff0ecb..17612c6 100644 --- a/src/DownloadUtils.cpp +++ b/src/DownloadUtils.cpp @@ -283,7 +283,7 @@ namespace QuickMedia { return DownloadResult::OK; } - bool download_async_gui(const std::string &url, const std::string &file_manager_start_dir, bool use_youtube_dl, bool no_video) { + bool download_async_gui(const std::string &url, const std::string &file_manager_start_dir, bool no_video) { char quickmedia_path[PATH_MAX]; ssize_t bytes_written = readlink("/proc/self/exe", quickmedia_path, sizeof(quickmedia_path)); if(bytes_written == -1 || bytes_written == sizeof(quickmedia_path)) @@ -292,7 +292,6 @@ namespace QuickMedia { quickmedia_path[bytes_written] = '\0'; std::vector args = { quickmedia_path, "download", "-u", url.c_str(), "--dir", file_manager_start_dir.c_str() }; - if(use_youtube_dl) args.push_back("--youtube-dl"); if(no_video) args.push_back("--no-video"); args.push_back(nullptr); return exec_program_async(args.data(), nullptr) == 0; diff --git a/src/Downloader.cpp b/src/Downloader.cpp new file mode 100644 index 0000000..ffb8675 --- /dev/null +++ b/src/Downloader.cpp @@ -0,0 +1,475 @@ +#include "../include/Downloader.hpp" +#include "../include/Storage.hpp" +#include "../include/NetUtils.hpp" +#include "../include/Notification.hpp" +#include +#include +#include + +namespace QuickMedia { + // TODO: Test with video that has hlsManifestUrl + static bool youtube_url_is_live_stream(const std::string &url) { + return url.find("yt_live_broadcast") != std::string::npos; + } + + CurlDownloader::CurlDownloader(const std::string &url, const std::string &output_filepath, int64_t content_length) : Downloader(url, output_filepath), content_length(content_length) { + output_filepath_tmp = output_filepath; + output_filepath_tmp.append(".tmp"); + read_program.pid = -1; + read_program.read_fd = -1; + progress_text = "0 bytes/Unknown"; + download_speed_text = "Unknown/s"; + } + + bool CurlDownloader::start() { + remove(output_filepath_tmp.data.c_str()); + + if(content_length == -1) { + const char *args[] = { "curl", + "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", + "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), + "-D", "/dev/stdout", + "--", url.c_str(), nullptr }; + + if(exec_program_pipe(args, &read_program) != 0) + return false; + } else { + const char *args[] = { "curl", + "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", + "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), + "--", url.c_str(), nullptr }; + + if(exec_program_pipe(args, &read_program) != 0) + return false; + } + + if(content_length != -1) + return true; + + // TODO: Remove this async task and make the fd non blocking instead + header_reader = AsyncTask([this]{ + char tmp_buf[1024]; + while(true) { + ssize_t bytes_available = read(read_program.read_fd, tmp_buf, sizeof(tmp_buf)); + if(bytes_available == -1) { + return false; + } else if(bytes_available > 0 && content_length == -1) { + header.append(tmp_buf, bytes_available); + if(header.find("\r\n\r\n") != std::string::npos) { + std::string content_length_str = header_extract_value(header, "content-length"); + if(!content_length_str.empty()) { + errno = 0; + char *endptr; + const long content_length_tmp = strtol(content_length_str.c_str(), &endptr, 10); + if(endptr != content_length_str.c_str() && errno == 0) { + std::lock_guard lock(content_length_mutex); + content_length = content_length_tmp; + } + } + } + } + } + return true; + }); + + return true; + } + + bool CurlDownloader::stop(bool download_completed) { + if(read_program.read_fd != -1) + close(read_program.read_fd); + if(read_program.pid != -1) + kill(read_program.pid, SIGTERM); + if(!download_completed) + remove(output_filepath_tmp.data.c_str()); + //header_reader.cancel(); + finished = false; + return true; + } + + DownloadUpdateStatus CurlDownloader::update() { + if(finished) + return DownloadUpdateStatus::FINISHED; + + if(read_program.pid == -1) + return DownloadUpdateStatus::ERROR; + + int status = 0; + if(wait_program_non_blocking(read_program.pid, &status)) { + read_program.pid = -1; + if(status == 0 && rename_atomic(output_filepath_tmp.data.c_str(), output_filepath.c_str()) == 0) { + finished = true; + progress = 1.0f; + return DownloadUpdateStatus::FINISHED; + } else { + return DownloadUpdateStatus::ERROR; + } + } + + if(header_reader.ready()) { + if(!header_reader.get()) + return DownloadUpdateStatus::ERROR; + } + + std::lock_guard lock(content_length_mutex); + size_t output_file_size = 0; + file_get_size(output_filepath_tmp, &output_file_size); + size_t downloaded_size = std::min((int64_t)output_file_size, content_length); + + if(content_length == -1) { + progress_text = std::to_string(output_file_size / 1024) + "/Unknown"; + } else { + size_t percentage = 0; + if(output_file_size > 0) + percentage = (double)downloaded_size / (double)content_length * 100.0; + progress = (double)percentage / 100.0; + progress_text = file_size_to_human_readable_string(downloaded_size) + "/" + file_size_to_human_readable_string(content_length) + " (" + std::to_string(percentage) + "%)"; + } + + // TODO: Take into consideration time overflow? + size_t downloaded_diff = std::max(0lu, downloaded_size - downloaded_since_last_check); + download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s"; + downloaded_since_last_check = downloaded_size; + + return DownloadUpdateStatus::DOWNLOADING; + } + + float CurlDownloader::get_progress() { + return progress; + } + + std::string CurlDownloader::get_progress_text() { + return progress_text; + } + + std::string CurlDownloader::get_download_speed_text() { + return download_speed_text; + } + + YoutubeDlDownloader::YoutubeDlDownloader(const std::string &url, const std::string &output_filepath, bool no_video) : Downloader(url, output_filepath), no_video(no_video) { + // youtube-dl requires a file extension for the file + if(this->output_filepath.find('.') == std::string::npos) + this->output_filepath += ".mkv"; + + read_program.pid = -1; + read_program.read_fd = -1; + progress_text = "0.0% of Unknown"; + download_speed_text = "Unknown/s"; + } + + bool YoutubeDlDownloader::start() { + remove(output_filepath.c_str()); + + std::vector args = { "youtube-dl", "--no-warnings", "--no-continue", "--output", output_filepath.c_str(), "--newline" }; + if(no_video) { + args.push_back("-f"); + args.push_back("bestaudio/best"); + args.push_back("-x"); + } else { + args.push_back("-f"); + args.push_back("bestvideo+bestaudio/best"); + } + args.insert(args.end(), { "--", url.c_str(), nullptr }); + + if(exec_program_pipe(args.data(), &read_program) != 0) + return false; + + read_program_file = fdopen(read_program.read_fd, "rb"); + if(!read_program_file) { + if(read_program.pid != -1) + wait_program(read_program.pid); + return false; + } + + // TODO: Remove this async task and make the fd non blocking instead + youtube_dl_output_reader = AsyncTask([this]{ + char line[128]; + char progress_c[10]; + char content_size_c[20]; + char download_speed_c[20]; + + while(true) { + if(fgets(line, sizeof(line), read_program_file)) { + int len = strlen(line); + if(len > 0 && line[len - 1] == '\n') { + line[len - 1] = '\0'; + --len; + } + + if(sscanf(line, "[download] %10s of %20s at %20s", progress_c, content_size_c, download_speed_c) == 3) { + std::lock_guard lock(progress_update_mutex); + + if(strcmp(progress_c, "Unknown") != 0 && strcmp(content_size_c, "Unknown") != 0) { + std::string progress_str = progress_c; + progress_text = progress_str + " of " + content_size_c; + if(progress_str.back() == '%') { + errno = 0; + char *endptr; + const double progress_tmp = strtod(progress_str.c_str(), &endptr); + if(endptr != progress_str.c_str() && errno == 0) + progress = progress_tmp / 100.0; + } + } + + if(strcmp(download_speed_c, "Unknown") == 0) + download_speed_text = "Unknown/s"; + else + download_speed_text = download_speed_c; + } + } else { + return false; + } + } + + return true; + }); + + return true; + } + + bool YoutubeDlDownloader::stop(bool) { + if(read_program_file) + fclose(read_program_file); + if(read_program.pid != -1) + kill(read_program.pid, SIGTERM); + // TODO: Remove the temporary files created by youtube-dl (if !download_completed) + //header_reader.cancel(); + finished = false; + return true; + } + + DownloadUpdateStatus YoutubeDlDownloader::update() { + if(finished) + return DownloadUpdateStatus::FINISHED; + + if(read_program.pid == -1) + return DownloadUpdateStatus::ERROR; + + int status = 0; + if(wait_program_non_blocking(read_program.pid, &status)) { + read_program.pid = -1; + if(status == 0) { + finished = true; + return DownloadUpdateStatus::FINISHED; + } else { + return DownloadUpdateStatus::ERROR; + } + } + + if(youtube_dl_output_reader.ready()) { + if(!youtube_dl_output_reader.get()) + return DownloadUpdateStatus::ERROR; + } + + return DownloadUpdateStatus::DOWNLOADING; + } + + float YoutubeDlDownloader::get_progress() { + std::lock_guard lock(progress_update_mutex); + return progress; + } + + std::string YoutubeDlDownloader::get_progress_text() { + std::lock_guard lock(progress_update_mutex); + return progress_text; + } + + std::string YoutubeDlDownloader::get_download_speed_text() { + std::lock_guard lock(progress_update_mutex); + return download_speed_text; + } + + YoutubeDownloader::YoutubeDownloader(const MediaMetadata &video_metadata, const MediaMetadata &audio_metadata, const std::string &output_filepath) : + Downloader("", output_filepath), + video_metadata(video_metadata), + audio_metadata(audio_metadata) + { + for(int i = 0; i < 2; ++i) + downloaders[i] = nullptr; + } + + bool YoutubeDownloader::start() { + struct MediaProxyMetadata { + std::unique_ptr *media_proxy; + MediaMetadata *media_metadata; + std::unique_ptr *downloader; + std::string *output_filepath; + bool tmp_audio_file; + }; + + const bool has_video = !video_metadata.url.empty(); + + MediaProxyMetadata media_proxies[2] = { + { &youtube_video_media_proxy, &video_metadata, &downloaders[0], &video_output_filepath, false }, + { &youtube_audio_media_proxy, &audio_metadata, &downloaders[1], &audio_output_filepath, has_video } + }; + + int num_proxied_media = 0; + for(int i = 0; i < 2; ++i) { + media_proxies[i].output_filepath->clear(); + if(media_proxies[i].media_metadata->url.empty()) + continue; + + if(youtube_url_is_live_stream(media_proxies[i].media_metadata->url)) + *media_proxies[i].media_proxy = std::make_unique(); + else + *media_proxies[i].media_proxy = std::make_unique(); + + if(!(*media_proxies[i].media_proxy)->start(media_proxies[i].media_metadata->url, media_proxies[i].media_metadata->content_length)) { + fprintf(stderr, "Failed to load start youtube media proxy\n"); + return false; + } + + std::string media_proxy_addr; + if(!(*media_proxies[i].media_proxy)->get_address(media_proxy_addr)) { + fprintf(stderr, "Failed to load start youtube media proxy\n"); + return false; + } + + if(media_proxies[i].tmp_audio_file) + *media_proxies[i].output_filepath = output_filepath + ".audio"; + else + *media_proxies[i].output_filepath = output_filepath; + + remove(media_proxies[i].output_filepath->c_str()); + fprintf(stderr, "Downloading %s to %s\n", media_proxy_addr.c_str(), media_proxies[i].output_filepath->c_str()); + *media_proxies[i].downloader = std::make_unique(media_proxy_addr, *media_proxies[i].output_filepath, media_proxies[i].media_metadata->content_length); + *media_proxies[i].output_filepath = (*media_proxies[i].downloader)->get_output_filepath(); + if(!(*media_proxies[i].downloader)->start()) + return false; + ++num_proxied_media; + } + + if(num_proxied_media == 0) { + return false; + } + + downloader_task = AsyncTask([this]() { + sf::Clock timer; + const double sleep_time_millisec = 1; + while(!program_is_dead_in_current_thread()) { + if(youtube_video_media_proxy) + youtube_video_media_proxy->update(); + + if(youtube_audio_media_proxy) + youtube_audio_media_proxy->update(); + + const int sleep_time = sleep_time_millisec - timer.restart().asMilliseconds(); + if(sleep_time > 0) + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time)); + } + }); + + return true; + } + + static bool ffmpeg_merge_audio_and_video(const std::string &video_filepath, const std::string &audio_filepath, const std::string &output_filepath) { + const char *args[] = { "ffmpeg", "-y", "-v", "quiet", "-i", video_filepath.c_str(), "-i", audio_filepath.c_str(), "-c", "copy", "--", output_filepath.c_str(), nullptr }; + return exec_program(args, nullptr, nullptr) == 0; + } + + bool YoutubeDownloader::stop(bool download_completed) { + for(int i = 0; i < 2; ++i) { + if(downloaders[i]) { + downloaders[i]->stop(download_completed); + downloaders[i].reset(); + } + } + + if(youtube_video_media_proxy) + youtube_video_media_proxy->stop(); + + if(youtube_audio_media_proxy) + youtube_audio_media_proxy->stop(); + + if(downloader_task.valid()) + downloader_task.cancel(); + + youtube_video_media_proxy.reset(); + youtube_audio_media_proxy.reset(); + + if(!download_completed) + return false; + + bool success = true; + if(!video_metadata.url.empty() && !audio_metadata.url.empty()) { + std::string tmp_file = video_output_filepath + ".tmp.mkv"; + if(ffmpeg_merge_audio_and_video(video_output_filepath, audio_output_filepath, tmp_file)) { + success = rename_atomic(tmp_file.c_str(), output_filepath.c_str()) == 0; + if(success) { + remove(audio_output_filepath.c_str()); + } else { + remove(tmp_file.c_str()); + } + } else { + success = false; + } + } + return success; + } + + DownloadUpdateStatus YoutubeDownloader::update() { + int num_downloading = 0; + int num_finished = 0; + int num_failed = 0; + int num_downloaders = 0; + + for(int i = 0; i < 2; ++i) { + if(downloaders[i]) { + ++num_downloaders; + DownloadUpdateStatus update_status = downloaders[i]->update(); + switch(update_status) { + case DownloadUpdateStatus::DOWNLOADING: + ++num_downloading; + break; + case DownloadUpdateStatus::FINISHED: + ++num_finished; + break; + case DownloadUpdateStatus::ERROR: + ++num_failed; + break; + } + } + } + + if(num_failed > 0) + return DownloadUpdateStatus::ERROR; + else if(num_finished == num_downloaders) + return DownloadUpdateStatus::FINISHED; + else + return DownloadUpdateStatus::DOWNLOADING; + } + + float YoutubeDownloader::get_progress() { + return get_min_progress_downloader()->get_progress(); + } + + std::string YoutubeDownloader::get_progress_text() { + return get_min_progress_downloader()->get_progress_text(); + } + + std::string YoutubeDownloader::get_download_speed_text() { + return get_min_progress_downloader()->get_download_speed_text(); + } + + CurlDownloader* YoutubeDownloader::get_min_progress_downloader() { + int min_index = -1; + float progress = 999999.0f; + for(int i = 0; i < 2; ++i) { + if(downloaders[i]) { + float item_progress = downloaders[i]->get_progress(); + if(item_progress < progress) { + min_index = i; + progress = item_progress; + } + } + } + //assert(min_index != -1); + // Shouldn't happen + if(min_index == -1) + min_index = 0; + return downloaders[min_index].get(); + } +} \ No newline at end of file diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 64833bd..fb6b425 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -29,6 +29,7 @@ #include "../include/Utils.hpp" #include "../include/Tabs.hpp" #include "../include/Theme.hpp" +#include "../include/Downloader.hpp" #include "../plugins/youtube/YoutubeMediaProxy.hpp" #include "../include/gui/Button.hpp" #include "../external/hash-library/sha256.h" @@ -387,7 +388,6 @@ namespace QuickMedia { Window parent_window = None; std::vector tabs; const char *url = nullptr; - bool download_use_youtube_dl = false; std::string program_path = dirname(argv[0]); for(int i = 1; i < argc; ++i) { @@ -425,8 +425,6 @@ namespace QuickMedia { } } else if(strcmp(argv[i], "--low-cpu-mode") == 0) { low_cpu_mode = true; - } else if(strcmp(argv[i], "--youtube-dl") == 0) { - download_use_youtube_dl = true; } else if(strcmp(argv[i], "-u") == 0) { if(i < argc - 1) { url = argv[i + 1]; @@ -533,7 +531,7 @@ namespace QuickMedia { usage(); return -1; } - download_page(url, download_use_youtube_dl); + download_page(url); return exit_code; } @@ -2404,9 +2402,17 @@ namespace QuickMedia { return false; } - void Program::video_page_download_video(const std::string &url, bool use_youtube_dl, sf::WindowHandle video_player_window) { - if(!use_youtube_dl) { - download_async_gui(url, file_manager_start_dir.string(), use_youtube_dl, no_video); + static bool url_should_download_with_youtube_dl(const std::string &url) { + return url.find("pornhub.com") != std::string::npos || url.find("xhamster.com") != std::string::npos || url.find("spankbang.com") != std::string::npos || url.find("xvideos.com") != std::string::npos; + } + + void Program::video_page_download_video(const std::string &url, sf::WindowHandle video_player_window) { + bool separate_audio_option = url_should_download_with_youtube_dl(url);; + std::string video_id; + separate_audio_option |= youtube_url_extract_id(url, video_id); + + if(!separate_audio_option) { + download_async_gui(url, file_manager_start_dir.string(), no_video); return; } @@ -2438,7 +2444,7 @@ namespace QuickMedia { if(!selected) return; - download_async_gui(url, file_manager_start_dir.string(), true, audio_only); + download_async_gui(url, file_manager_start_dir.string(), audio_only); } bool Program::video_download_if_non_streamable(std::string &video_url, std::string &audio_url, bool &is_audio_only, bool &has_embedded_audio, PageType previous_page) { @@ -2542,8 +2548,8 @@ namespace QuickMedia { std::unique_ptr youtube_video_media_proxy; std::unique_ptr youtube_audio_media_proxy; AsyncTask youtube_downloader_task; - int youtube_video_content_length = 0; - int youtube_audio_content_length = 0; + int64_t youtube_video_content_length = 0; + int64_t youtube_audio_content_length = 0; std::string channel_url; AsyncTask video_tasks; @@ -2585,11 +2591,12 @@ namespace QuickMedia { if(video_page->load(new_title, channel_url, media_chapters) != PluginResult::OK) return false; + std::string ext; if(!no_video) - video_url = video_page->get_video_url(largest_monitor_height, has_embedded_audio); + video_url = video_page->get_video_url(largest_monitor_height, has_embedded_audio, ext); if(video_url.empty() || no_video) { - video_url = video_page->get_audio_url(); + video_url = video_page->get_audio_url(ext); if(video_url.empty()) { video_url = video_page->get_url(); has_embedded_audio = true; @@ -2598,7 +2605,7 @@ namespace QuickMedia { has_embedded_audio = false; } } else if(!has_embedded_audio) { - audio_url = video_page->get_audio_url(); + audio_url = video_page->get_audio_url(ext); } if(!is_youtube && download_if_streaming_fails) { @@ -2677,7 +2684,7 @@ namespace QuickMedia { struct MediaProxyMetadata { std::unique_ptr *media_proxy; std::string *url; - int content_length; + int64_t content_length; }; MediaProxyMetadata media_proxies[2] = { @@ -2876,7 +2883,7 @@ namespace QuickMedia { } else if(pressed_keysym == XK_f && pressing_ctrl) { window_set_fullscreen(disp, window.getSystemHandle(), WindowFullscreenState::TOGGLE); } else if(pressed_keysym == XK_s && pressing_ctrl) { - video_page_download_video(video_page->get_url(), !is_matrix || is_youtube, video_player_window); + video_page_download_video(video_page->get_url(), video_player_window); } else if(pressed_keysym == XK_F5) { in_seeking = false; double resume_start_time = 0.0; @@ -3954,7 +3961,7 @@ namespace QuickMedia { } else if(event.key.code == sf::Keyboard::S && event.key.control) { BodyItem *selected_item = thread_body->get_selected(); if(selected_item && !selected_item->url.empty()) - download_async_gui(selected_item->url, file_manager_start_dir.string(), false, false); + download_async_gui(selected_item->url, file_manager_start_dir.string(), false); } BodyItem *selected_item = thread_body->get_selected(); @@ -4071,7 +4078,7 @@ namespace QuickMedia { redraw = true; frame_skip_text_entry = true; } else if(event.key.code == sf::Keyboard::S && event.key.control) { - download_async_gui(attached_image_url, file_manager_start_dir.string(), false, false); + download_async_gui(attached_image_url, file_manager_start_dir.string(), false); } } } @@ -5437,7 +5444,7 @@ namespace QuickMedia { avatar_applied = false; return true; } else if(message_type == MessageType::FILE) { - download_async_gui(selected->url, file_manager_start_dir.string(), false, no_video); + download_async_gui(selected->url, file_manager_start_dir.string(), no_video); return true; } @@ -5478,7 +5485,7 @@ namespace QuickMedia { if(selected_item_message) { MessageType message_type = selected_item_message->type; if(!selected->url.empty() && message_type >= MessageType::IMAGE && message_type <= MessageType::FILE) { - download_async_gui(selected->url, file_manager_start_dir.string(), false, no_video); + download_async_gui(selected->url, file_manager_start_dir.string(), no_video); return true; } } @@ -6456,290 +6463,6 @@ namespace QuickMedia { matrix->stop_sync(); } - enum class DownloadUpdateStatus { - DOWNLOADING, - FINISHED, - ERROR - }; - - class Downloader { - public: - Downloader(const std::string &url, const std::string &output_filepath) : url(url), output_filepath(output_filepath) {} - virtual ~Downloader() = default; - - virtual bool start() = 0; - virtual void stop(bool download_completed) = 0; - virtual DownloadUpdateStatus update() = 0; - - virtual float get_progress() = 0; - virtual std::string get_progress_text() = 0; - virtual std::string get_download_speed_text() = 0; - protected: - std::string url; - std::string output_filepath; - }; - - class CurlDownloader : public Downloader { - public: - CurlDownloader(const std::string &url, const std::string &output_filepath) : Downloader(url, output_filepath) { - output_filepath_tmp = output_filepath; - output_filepath_tmp.append(".tmp"); - read_program.pid = -1; - read_program.read_fd = -1; - progress_text = "0 bytes/Unknown"; - download_speed_text = "Unknown/s"; - } - - bool start() override { - remove(output_filepath_tmp.data.c_str()); - - const char *args[] = { "curl", - "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", - "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", - "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), - "-D", "/dev/stdout", - "--", url.c_str(), nullptr }; - - if(exec_program_pipe(args, &read_program) != 0) - return false; - - // TODO: Remove this async task and make the fd non blocking instead - header_reader = AsyncTask([this]{ - char tmp_buf[1024]; - while(true) { - ssize_t bytes_available = read(read_program.read_fd, tmp_buf, sizeof(tmp_buf)); - if(bytes_available == -1) { - return false; - } else if(bytes_available > 0 && content_length == (size_t)-1) { - header.append(tmp_buf, bytes_available); - if(header.find("\r\n\r\n") != std::string::npos) { - std::string content_length_str = header_extract_value(header, "content-length"); - if(!content_length_str.empty()) { - errno = 0; - char *endptr; - const long content_length_tmp = strtol(content_length_str.c_str(), &endptr, 10); - if(endptr != content_length_str.c_str() && errno == 0) { - std::lock_guard lock(content_length_mutex); - content_length = content_length_tmp; - } - } - } - } - } - return true; - }); - - return true; - } - - void stop(bool download_completed) override { - if(read_program.read_fd != -1) - close(read_program.read_fd); - if(read_program.pid != -1) - kill(read_program.pid, SIGTERM); - if(!download_completed) - remove(output_filepath_tmp.data.c_str()); - //header_reader.cancel(); - } - - DownloadUpdateStatus update() override { - int status = 0; - if(wait_program_non_blocking(read_program.pid, &status)) { - read_program.pid = -1; - if(status == 0 && rename_atomic(output_filepath_tmp.data.c_str(), output_filepath.c_str()) == 0) { - return DownloadUpdateStatus::FINISHED; - } else { - return DownloadUpdateStatus::ERROR; - } - } - - if(header_reader.ready()) { - if(!header_reader.get()) - return DownloadUpdateStatus::ERROR; - } - - std::lock_guard lock(content_length_mutex); - size_t output_file_size = 0; - file_get_size(output_filepath_tmp, &output_file_size); - size_t downloaded_size = std::min(output_file_size, content_length); - - if(content_length == (size_t)-1) { - progress_text = std::to_string(output_file_size / 1024) + "/Unknown"; - } else { - size_t percentage = 0; - if(output_file_size > 0) - percentage = (double)downloaded_size / (double)content_length * 100.0; - progress = (double)percentage / 100.0; - progress_text = file_size_to_human_readable_string(downloaded_size) + "/" + file_size_to_human_readable_string(content_length) + " (" + std::to_string(percentage) + "%)"; - } - - // TODO: Take into consideration time overflow? - size_t downloaded_diff = std::max(0lu, downloaded_size - downloaded_since_last_check); - download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s"; - downloaded_since_last_check = downloaded_size; - - return DownloadUpdateStatus::DOWNLOADING; - } - - float get_progress() override { - return progress; - } - - std::string get_progress_text() override { - return progress_text; - } - - std::string get_download_speed_text() override { - return download_speed_text; - } - private: - Path output_filepath_tmp; - ReadProgram read_program; - AsyncTask header_reader; - std::string header; - size_t content_length = (size_t)-1; - size_t downloaded_since_last_check = 0; - float progress = 0.0f; - std::mutex content_length_mutex; - std::string progress_text; - std::string download_speed_text; - }; - - class YoutubeDlDownloader : public Downloader { - public: - YoutubeDlDownloader(const std::string &url, const std::string &output_filepath, bool no_video) : Downloader(url, output_filepath), no_video(no_video) { - // youtube-dl requires a file extension for the file - if(this->output_filepath.find('.') == std::string::npos) - this->output_filepath += ".mkv"; - - read_program.pid = -1; - read_program.read_fd = -1; - progress_text = "0.0% of Unknown"; - download_speed_text = "Unknown/s"; - } - - bool start() override { - remove(output_filepath.c_str()); - - std::vector args = { "youtube-dl", "--no-warnings", "--no-continue", "--output", output_filepath.c_str(), "--newline" }; - if(no_video) { - args.push_back("-f"); - args.push_back("bestaudio/best"); - args.push_back("-x"); - } else { - args.push_back("-f"); - args.push_back("bestvideo+bestaudio/best"); - } - args.insert(args.end(), { "--", url.c_str(), nullptr }); - - if(exec_program_pipe(args.data(), &read_program) != 0) - return false; - - read_program_file = fdopen(read_program.read_fd, "rb"); - if(!read_program_file) { - wait_program(read_program.pid); - return false; - } - - // TODO: Remove this async task and make the fd non blocking instead - youtube_dl_output_reader = AsyncTask([this]{ - char line[128]; - char progress_c[10]; - char content_size_c[20]; - char download_speed_c[20]; - - while(true) { - if(fgets(line, sizeof(line), read_program_file)) { - int len = strlen(line); - if(len > 0 && line[len - 1] == '\n') { - line[len - 1] = '\0'; - --len; - } - - if(sscanf(line, "[download] %10s of %20s at %20s", progress_c, content_size_c, download_speed_c) == 3) { - std::lock_guard lock(progress_update_mutex); - - if(strcmp(progress_c, "Unknown") != 0 && strcmp(content_size_c, "Unknown") != 0) { - std::string progress_str = progress_c; - progress_text = progress_str + " of " + content_size_c; - if(progress_str.back() == '%') { - errno = 0; - char *endptr; - const double progress_tmp = strtod(progress_str.c_str(), &endptr); - if(endptr != progress_str.c_str() && errno == 0) - progress = progress_tmp / 100.0; - } - } - - if(strcmp(download_speed_c, "Unknown") == 0) - download_speed_text = "Unknown/s"; - else - download_speed_text = download_speed_c; - } - } else { - return false; - } - } - - return true; - }); - - return true; - } - - void stop(bool) override { - if(read_program_file) - fclose(read_program_file); - if(read_program.pid != -1) - kill(read_program.pid, SIGTERM); - // TODO: Remove the temporary files created by youtube-dl (if !download_completed) - //header_reader.cancel(); - } - - DownloadUpdateStatus update() override { - int status = 0; - if(wait_program_non_blocking(read_program.pid, &status)) { - read_program.pid = -1; - if(status == 0) { - return DownloadUpdateStatus::FINISHED; - } else { - return DownloadUpdateStatus::ERROR; - } - } - - if(youtube_dl_output_reader.ready()) { - if(!youtube_dl_output_reader.get()) - return DownloadUpdateStatus::ERROR; - } - - return DownloadUpdateStatus::DOWNLOADING; - } - - float get_progress() override { - std::lock_guard lock(progress_update_mutex); - return progress; - } - - std::string get_progress_text() override { - std::lock_guard lock(progress_update_mutex); - return progress_text; - } - - std::string get_download_speed_text() override { - std::lock_guard lock(progress_update_mutex); - return download_speed_text; - } - private: - ReadProgram read_program; - FILE *read_program_file = nullptr; - AsyncTask youtube_dl_output_reader; - std::mutex progress_update_mutex; - float progress = 0.0f; - std::string progress_text; - std::string download_speed_text; - bool no_video; - }; - static int accumulate_string(char *data, int size, void *userdata) { std::string *str = (std::string*)userdata; if(str->size() + size > 1024 * 1024 * 100) // 100mb sane limit, TODO: make configurable @@ -6748,12 +6471,27 @@ namespace QuickMedia { return 0; } - void Program::download_page(const char *url, bool download_use_youtube_dl) { + void Program::download_page(const std::string &url) { window.setTitle("QuickMedia - Select where you want to save " + std::string(url)); + const bool download_use_youtube_dl = url_should_download_with_youtube_dl(url); std::string filename; + std::string video_id; + const bool url_is_youtube = youtube_url_extract_id(url, video_id); + std::unique_ptr youtube_video_page; + + std::string video_url; + std::string audio_url; + int64_t video_content_length = 0; + int64_t audio_content_length = 0; + TaskResult task_result; if(download_use_youtube_dl) { + if(!is_program_executable_by_name("youtube-dl")) { + show_notification("QuickMedia", "youtube-dl needs to be installed to download the video/music", Urgency::CRITICAL); + abort(); + } + task_result = run_task_with_loading_screen([this, url, &filename]{ std::string json_str; std::vector args = { "youtube-dl", "--skip-download", "--print-json", "--no-warnings" }; @@ -6765,7 +6503,7 @@ namespace QuickMedia { args.push_back("-f"); args.push_back("bestvideo+bestaudio/best"); } - args.insert(args.end(), { "--", url, nullptr }); + args.insert(args.end(), { "--", url.c_str(), nullptr }); if(exec_program(args.data(), accumulate_string, &json_str) != 0) return false; @@ -6791,6 +6529,77 @@ namespace QuickMedia { return !filename.empty(); }); + } else if(url_is_youtube) { + youtube_video_page = std::make_unique(this, url); + bool cancelled = false; + bool load_successful = false; + const int largest_monitor_height = get_largest_monitor_height(disp); + + for(int i = 0; i < 3; ++i) { + task_result = run_task_with_loading_screen([this, &youtube_video_page, &filename, &video_url, &audio_url, &video_content_length, &audio_content_length, largest_monitor_height, &cancelled]{ + std::string channel_url; + std::vector chapters; + filename.clear(); + if(youtube_video_page->load(filename, channel_url, chapters) != PluginResult::OK) + return false; + + std::string ext; + bool has_embedded_audio = true; + video_url = no_video ? "" : youtube_video_page->get_video_url(largest_monitor_height, has_embedded_audio, ext); + audio_url.clear(); + + if(!has_embedded_audio || no_video) + audio_url = youtube_video_page->get_audio_url(ext); + + if(video_url.empty() && audio_url.empty()) + return false; + + if(!youtube_url_is_live_stream(video_url) && !youtube_url_is_live_stream(audio_url)) { + video_content_length = 0; + audio_content_length = 0; + std::string new_video_url = video_url; + std::string new_audio_url = audio_url; + auto current_thread_id = std::this_thread::get_id(); + if(!youtube_custom_redirect(new_video_url, new_audio_url, video_content_length, audio_content_length, [current_thread_id]{ return !program_is_dead_in_thread(current_thread_id); })) { + if(program_is_dead_in_current_thread()) + cancelled = true; + return false; + } + + video_url = std::move(new_video_url); + audio_url = std::move(new_audio_url); + } + + if(!video_url.empty() && !audio_url.empty()) + filename += ".mkv"; + else + filename += ext; + + return true; + }); + + if(task_result == TaskResult::CANCEL || cancelled) { + exit_code = 1; + return; + } else if(task_result == TaskResult::FALSE) { + continue; + } + + load_successful = true; + break; + } + + if(!load_successful) { + show_notification("QuickMedia", "Download failed", Urgency::CRITICAL); + exit_code = 1; + return; + } + + if(youtube_url_is_live_stream(video_url) || youtube_url_is_live_stream(audio_url)) { + show_notification("QuickMedia", "Downloading youtube live streams is currently not supported", Urgency::CRITICAL); + exit_code = 1; + return; + } } else { task_result = run_task_with_loading_screen([url, &filename]{ return url_get_remote_name(url, filename, true) == DownloadResult::OK; @@ -6800,6 +6609,10 @@ namespace QuickMedia { if(task_result == TaskResult::CANCEL) { exit_code = 1; return; + } else if(task_result == TaskResult::FALSE) { + show_notification("QuickMedia", "Download failed", Urgency::CRITICAL); + exit_code = 1; + return; } std::string output_filepath = file_save_page(filename); @@ -6862,10 +6675,21 @@ namespace QuickMedia { sf::Event event; std::unique_ptr downloader; - if(download_use_youtube_dl) + if(download_use_youtube_dl) { downloader = std::make_unique(url, output_filepath, no_video); - else + } else if(url_is_youtube) { + MediaMetadata video_metadata; + video_metadata.url = std::move(video_url); + video_metadata.content_length = video_content_length; + + MediaMetadata audio_metadata; + audio_metadata.url = std::move(audio_url); + audio_metadata.content_length = audio_content_length; + + downloader = std::make_unique(video_metadata, audio_metadata, output_filepath); + } else { downloader = std::make_unique(url, output_filepath); + } if(!downloader->start()) { show_notification("QuickMedia", std::string("Failed to download ") + url + " to " + output_filepath, Urgency::CRITICAL); @@ -6908,6 +6732,7 @@ namespace QuickMedia { download_completed = true; goto cleanup; case DownloadUpdateStatus::ERROR: + fprintf(stderr, "Download error on update\n"); goto cleanup; } @@ -6962,8 +6787,8 @@ namespace QuickMedia { } cleanup: - downloader->stop(download_completed); - if(download_completed) { + const bool stop_successful = downloader->stop(download_completed); + if(download_completed && stop_successful) { show_notification("QuickMedia", std::string("Download finished! Downloaded ") + Path(filename).filename() + " to " + output_filepath); exit_code = 0; } else { diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index eb95c05..2fd193f 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -266,9 +266,8 @@ R"END( // Sometimes youtube returns a redirect url (not in the header but in the body...). // TODO: Find why this happens and if there is a way bypass it. - static std::string get_playback_url_recursive(std::string playback_url, int &content_length) { + static std::string get_playback_url_recursive(std::string playback_url, int64_t &content_length) { std::vector additional_args = get_cookies(); - additional_args.push_back({ "-r", "0-4096" }); const int max_redirects = 5; for(int i = 0; i < max_redirects; ++i) { @@ -307,7 +306,7 @@ R"END( return playback_url; } - bool youtube_custom_redirect(std::string &video_url, std::string &audio_url, int &video_content_length, int &audio_content_length, std::function active_handler) { + bool youtube_custom_redirect(std::string &video_url, std::string &audio_url, int64_t &video_content_length, int64_t &audio_content_length, std::function active_handler) { // TODO: Do this without threads int num_total_tasks = 0; AsyncTask tasks[2]; @@ -2164,10 +2163,6 @@ R"END( return url.substr(index, end - index); } - static void print_chosen_format(const YoutubeVideoFormat &format) { - fprintf(stderr, "Choosing youtube video format: width: %d, height: %d, fps: %d, bitrate: %d, mime type: %s\n", format.width, format.height, format.fps, format.base.bitrate, format.base.mime_type.c_str()); - } - static const YoutubeVideoFormat* get_highest_resolution_mp4_non_av1(const std::vector &video_formats, int max_height) { for(const YoutubeVideoFormat &video_format : video_formats) { if(video_format.height <= max_height && video_format.base.mime_type.find("mp4") != std::string::npos && video_format.base.mime_type.find("av01") == std::string::npos) @@ -2184,14 +2179,14 @@ R"END( return nullptr; } - std::string YoutubeVideoPage::get_video_url(int max_height, bool &has_embedded_audio) { + std::string YoutubeVideoPage::get_video_url(int max_height, bool &has_embedded_audio, std::string &ext) { if(!livestream_url.empty()) { has_embedded_audio = true; return livestream_url; } if(video_formats.empty()) { - has_embedded_audio = true; + has_embedded_audio = false; return ""; } @@ -2208,17 +2203,31 @@ R"END( if(!chosen_video_format) chosen_video_format = &video_formats.back(); - print_chosen_format(*chosen_video_format); + fprintf(stderr, "Choosing youtube video format: width: %d, height: %d, fps: %d, bitrate: %d, mime type: %s\n", chosen_video_format->width, chosen_video_format->height, chosen_video_format->fps, chosen_video_format->base.bitrate, chosen_video_format->base.mime_type.c_str()); has_embedded_audio = chosen_video_format->has_embedded_audio; + + if(chosen_video_format->base.mime_type.find("mp4") != std::string::npos) + ext = ".mp4"; + else if(chosen_video_format->base.mime_type.find("webm") != std::string::npos) + ext = ".webm"; + return chosen_video_format->base.url; } - std::string YoutubeVideoPage::get_audio_url() { + std::string YoutubeVideoPage::get_audio_url(std::string &ext) { if(audio_formats.empty()) return ""; const YoutubeAudioFormat *chosen_audio_format = &audio_formats.front(); fprintf(stderr, "Choosing youtube audio format: bitrate: %d, mime type: %s\n", chosen_audio_format->base.bitrate, chosen_audio_format->base.mime_type.c_str()); + + if(chosen_audio_format->base.mime_type.find("mp4") != std::string::npos) + ext = ".m4a"; + else if(chosen_audio_format->base.mime_type.find("webm") != std::string::npos) + ext = ".webm"; + else if(chosen_audio_format->base.mime_type.find("opus") != std::string::npos) + ext = ".opus"; + return chosen_audio_format->base.url; } diff --git a/src/plugins/youtube/YoutubeMediaProxy.cpp b/src/plugins/youtube/YoutubeMediaProxy.cpp index cba4a1d..2296d1a 100644 --- a/src/plugins/youtube/YoutubeMediaProxy.cpp +++ b/src/plugins/youtube/YoutubeMediaProxy.cpp @@ -77,7 +77,7 @@ namespace QuickMedia { stop(); } - bool YoutubeStaticMediaProxy::start(const std::string &youtube_media_url, int content_length) { + bool YoutubeStaticMediaProxy::start(const std::string &youtube_media_url, int64_t content_length) { if(socket_fd != -1) return false; @@ -251,15 +251,15 @@ namespace QuickMedia { int new_start_range = header_extract_start_range(client_request_buffer); //fprintf(stderr, "got new range from client: %d\n", new_start_range); - if(new_start_range >= 0) { - if(downloader_read_program.pid != -1) { - kill(downloader_read_program.pid, SIGTERM); - wait_program(downloader_read_program.pid); - downloader_read_program.pid = -1; - } - clear_download_state(); - update_download_program_status(false, new_start_range, true); + //if(new_start_range >= 0) { + if(downloader_read_program.pid != -1) { + kill(downloader_read_program.pid, SIGTERM); + wait_program(downloader_read_program.pid); + downloader_read_program.pid = -1; } + clear_download_state(); + return update_download_program_status(false, new_start_range, true); + //} } else { if(client_request_buffer.size() > MAX_BUFFER_SIZE) { client_request_finished = true; @@ -278,13 +278,13 @@ namespace QuickMedia { download_header_written_offset = 0; } - YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::update_download_program_status(bool client_disconnected, int new_range_start, bool restart_download) { + YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::update_download_program_status(bool client_disconnected, int64_t new_range_start, bool restart_download) { int program_status = 0; if(downloader_read_program.pid != -1) { if(client_disconnected) { wait_program(downloader_read_program.pid); } else { - if(!wait_program_non_blocking(downloader_read_program.pid, &program_status)) + if(wait_program_non_blocking(downloader_read_program.pid, &program_status) == 0) return Error::OK; } downloader_read_program.pid = -1; @@ -353,7 +353,7 @@ namespace QuickMedia { return Error::OK; } - static void header_replace_content_length(std::string &header, size_t header_size, int new_content_length) { + static void header_replace_content_length(std::string &header, size_t header_size, int64_t new_content_length) { if(new_content_length < 0) new_content_length = 0; @@ -536,7 +536,7 @@ namespace QuickMedia { stop(); } - bool YoutubeLiveStreamMediaProxy::start(const std::string &youtube_media_url, int) { + bool YoutubeLiveStreamMediaProxy::start(const std::string &youtube_media_url, int64_t) { fd[0] = -1; fd[1] = -1; if(pipe(fd) == -1) { @@ -587,7 +587,7 @@ namespace QuickMedia { YoutubeMediaProxy::Error YoutubeLiveStreamMediaProxy::update_download_program_status() { int program_status = 0; - if(!wait_program_non_blocking(downloader_read_program.pid, &program_status)) + if(wait_program_non_blocking(downloader_read_program.pid, &program_status) == 0) return Error::OK; downloader_read_program.pid = -1; -- cgit v1.2.3