From e671784144174c4fceaa6df3737ba9b4de4a6c63 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 17 Jul 2021 09:43:20 +0200 Subject: Youtube: remove dependency on youtube-dl for downloads (also fixes downloads of age restricted videos) --- src/Downloader.cpp | 475 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 475 insertions(+) create mode 100644 src/Downloader.cpp (limited to 'src/Downloader.cpp') diff --git a/src/Downloader.cpp b/src/Downloader.cpp new file mode 100644 index 0000000..ffb8675 --- /dev/null +++ b/src/Downloader.cpp @@ -0,0 +1,475 @@ +#include "../include/Downloader.hpp" +#include "../include/Storage.hpp" +#include "../include/NetUtils.hpp" +#include "../include/Notification.hpp" +#include +#include +#include + +namespace QuickMedia { + // TODO: Test with video that has hlsManifestUrl + static bool youtube_url_is_live_stream(const std::string &url) { + return url.find("yt_live_broadcast") != std::string::npos; + } + + CurlDownloader::CurlDownloader(const std::string &url, const std::string &output_filepath, int64_t content_length) : Downloader(url, output_filepath), content_length(content_length) { + output_filepath_tmp = output_filepath; + output_filepath_tmp.append(".tmp"); + read_program.pid = -1; + read_program.read_fd = -1; + progress_text = "0 bytes/Unknown"; + download_speed_text = "Unknown/s"; + } + + bool CurlDownloader::start() { + remove(output_filepath_tmp.data.c_str()); + + if(content_length == -1) { + const char *args[] = { "curl", + "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", + "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), + "-D", "/dev/stdout", + "--", url.c_str(), nullptr }; + + if(exec_program_pipe(args, &read_program) != 0) + return false; + } else { + const char *args[] = { "curl", + "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", + "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), + "--", url.c_str(), nullptr }; + + if(exec_program_pipe(args, &read_program) != 0) + return false; + } + + if(content_length != -1) + return true; + + // TODO: Remove this async task and make the fd non blocking instead + header_reader = AsyncTask([this]{ + char tmp_buf[1024]; + while(true) { + ssize_t bytes_available = read(read_program.read_fd, tmp_buf, sizeof(tmp_buf)); + if(bytes_available == -1) { + return false; + } else if(bytes_available > 0 && content_length == -1) { + header.append(tmp_buf, bytes_available); + if(header.find("\r\n\r\n") != std::string::npos) { + std::string content_length_str = header_extract_value(header, "content-length"); + if(!content_length_str.empty()) { + errno = 0; + char *endptr; + const long content_length_tmp = strtol(content_length_str.c_str(), &endptr, 10); + if(endptr != content_length_str.c_str() && errno == 0) { + std::lock_guard lock(content_length_mutex); + content_length = content_length_tmp; + } + } + } + } + } + return true; + }); + + return true; + } + + bool CurlDownloader::stop(bool download_completed) { + if(read_program.read_fd != -1) + close(read_program.read_fd); + if(read_program.pid != -1) + kill(read_program.pid, SIGTERM); + if(!download_completed) + remove(output_filepath_tmp.data.c_str()); + //header_reader.cancel(); + finished = false; + return true; + } + + DownloadUpdateStatus CurlDownloader::update() { + if(finished) + return DownloadUpdateStatus::FINISHED; + + if(read_program.pid == -1) + return DownloadUpdateStatus::ERROR; + + int status = 0; + if(wait_program_non_blocking(read_program.pid, &status)) { + read_program.pid = -1; + if(status == 0 && rename_atomic(output_filepath_tmp.data.c_str(), output_filepath.c_str()) == 0) { + finished = true; + progress = 1.0f; + return DownloadUpdateStatus::FINISHED; + } else { + return DownloadUpdateStatus::ERROR; + } + } + + if(header_reader.ready()) { + if(!header_reader.get()) + return DownloadUpdateStatus::ERROR; + } + + std::lock_guard lock(content_length_mutex); + size_t output_file_size = 0; + file_get_size(output_filepath_tmp, &output_file_size); + size_t downloaded_size = std::min((int64_t)output_file_size, content_length); + + if(content_length == -1) { + progress_text = std::to_string(output_file_size / 1024) + "/Unknown"; + } else { + size_t percentage = 0; + if(output_file_size > 0) + percentage = (double)downloaded_size / (double)content_length * 100.0; + progress = (double)percentage / 100.0; + progress_text = file_size_to_human_readable_string(downloaded_size) + "/" + file_size_to_human_readable_string(content_length) + " (" + std::to_string(percentage) + "%)"; + } + + // TODO: Take into consideration time overflow? + size_t downloaded_diff = std::max(0lu, downloaded_size - downloaded_since_last_check); + download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s"; + downloaded_since_last_check = downloaded_size; + + return DownloadUpdateStatus::DOWNLOADING; + } + + float CurlDownloader::get_progress() { + return progress; + } + + std::string CurlDownloader::get_progress_text() { + return progress_text; + } + + std::string CurlDownloader::get_download_speed_text() { + return download_speed_text; + } + + YoutubeDlDownloader::YoutubeDlDownloader(const std::string &url, const std::string &output_filepath, bool no_video) : Downloader(url, output_filepath), no_video(no_video) { + // youtube-dl requires a file extension for the file + if(this->output_filepath.find('.') == std::string::npos) + this->output_filepath += ".mkv"; + + read_program.pid = -1; + read_program.read_fd = -1; + progress_text = "0.0% of Unknown"; + download_speed_text = "Unknown/s"; + } + + bool YoutubeDlDownloader::start() { + remove(output_filepath.c_str()); + + std::vector args = { "youtube-dl", "--no-warnings", "--no-continue", "--output", output_filepath.c_str(), "--newline" }; + if(no_video) { + args.push_back("-f"); + args.push_back("bestaudio/best"); + args.push_back("-x"); + } else { + args.push_back("-f"); + args.push_back("bestvideo+bestaudio/best"); + } + args.insert(args.end(), { "--", url.c_str(), nullptr }); + + if(exec_program_pipe(args.data(), &read_program) != 0) + return false; + + read_program_file = fdopen(read_program.read_fd, "rb"); + if(!read_program_file) { + if(read_program.pid != -1) + wait_program(read_program.pid); + return false; + } + + // TODO: Remove this async task and make the fd non blocking instead + youtube_dl_output_reader = AsyncTask([this]{ + char line[128]; + char progress_c[10]; + char content_size_c[20]; + char download_speed_c[20]; + + while(true) { + if(fgets(line, sizeof(line), read_program_file)) { + int len = strlen(line); + if(len > 0 && line[len - 1] == '\n') { + line[len - 1] = '\0'; + --len; + } + + if(sscanf(line, "[download] %10s of %20s at %20s", progress_c, content_size_c, download_speed_c) == 3) { + std::lock_guard lock(progress_update_mutex); + + if(strcmp(progress_c, "Unknown") != 0 && strcmp(content_size_c, "Unknown") != 0) { + std::string progress_str = progress_c; + progress_text = progress_str + " of " + content_size_c; + if(progress_str.back() == '%') { + errno = 0; + char *endptr; + const double progress_tmp = strtod(progress_str.c_str(), &endptr); + if(endptr != progress_str.c_str() && errno == 0) + progress = progress_tmp / 100.0; + } + } + + if(strcmp(download_speed_c, "Unknown") == 0) + download_speed_text = "Unknown/s"; + else + download_speed_text = download_speed_c; + } + } else { + return false; + } + } + + return true; + }); + + return true; + } + + bool YoutubeDlDownloader::stop(bool) { + if(read_program_file) + fclose(read_program_file); + if(read_program.pid != -1) + kill(read_program.pid, SIGTERM); + // TODO: Remove the temporary files created by youtube-dl (if !download_completed) + //header_reader.cancel(); + finished = false; + return true; + } + + DownloadUpdateStatus YoutubeDlDownloader::update() { + if(finished) + return DownloadUpdateStatus::FINISHED; + + if(read_program.pid == -1) + return DownloadUpdateStatus::ERROR; + + int status = 0; + if(wait_program_non_blocking(read_program.pid, &status)) { + read_program.pid = -1; + if(status == 0) { + finished = true; + return DownloadUpdateStatus::FINISHED; + } else { + return DownloadUpdateStatus::ERROR; + } + } + + if(youtube_dl_output_reader.ready()) { + if(!youtube_dl_output_reader.get()) + return DownloadUpdateStatus::ERROR; + } + + return DownloadUpdateStatus::DOWNLOADING; + } + + float YoutubeDlDownloader::get_progress() { + std::lock_guard lock(progress_update_mutex); + return progress; + } + + std::string YoutubeDlDownloader::get_progress_text() { + std::lock_guard lock(progress_update_mutex); + return progress_text; + } + + std::string YoutubeDlDownloader::get_download_speed_text() { + std::lock_guard lock(progress_update_mutex); + return download_speed_text; + } + + YoutubeDownloader::YoutubeDownloader(const MediaMetadata &video_metadata, const MediaMetadata &audio_metadata, const std::string &output_filepath) : + Downloader("", output_filepath), + video_metadata(video_metadata), + audio_metadata(audio_metadata) + { + for(int i = 0; i < 2; ++i) + downloaders[i] = nullptr; + } + + bool YoutubeDownloader::start() { + struct MediaProxyMetadata { + std::unique_ptr *media_proxy; + MediaMetadata *media_metadata; + std::unique_ptr *downloader; + std::string *output_filepath; + bool tmp_audio_file; + }; + + const bool has_video = !video_metadata.url.empty(); + + MediaProxyMetadata media_proxies[2] = { + { &youtube_video_media_proxy, &video_metadata, &downloaders[0], &video_output_filepath, false }, + { &youtube_audio_media_proxy, &audio_metadata, &downloaders[1], &audio_output_filepath, has_video } + }; + + int num_proxied_media = 0; + for(int i = 0; i < 2; ++i) { + media_proxies[i].output_filepath->clear(); + if(media_proxies[i].media_metadata->url.empty()) + continue; + + if(youtube_url_is_live_stream(media_proxies[i].media_metadata->url)) + *media_proxies[i].media_proxy = std::make_unique(); + else + *media_proxies[i].media_proxy = std::make_unique(); + + if(!(*media_proxies[i].media_proxy)->start(media_proxies[i].media_metadata->url, media_proxies[i].media_metadata->content_length)) { + fprintf(stderr, "Failed to load start youtube media proxy\n"); + return false; + } + + std::string media_proxy_addr; + if(!(*media_proxies[i].media_proxy)->get_address(media_proxy_addr)) { + fprintf(stderr, "Failed to load start youtube media proxy\n"); + return false; + } + + if(media_proxies[i].tmp_audio_file) + *media_proxies[i].output_filepath = output_filepath + ".audio"; + else + *media_proxies[i].output_filepath = output_filepath; + + remove(media_proxies[i].output_filepath->c_str()); + fprintf(stderr, "Downloading %s to %s\n", media_proxy_addr.c_str(), media_proxies[i].output_filepath->c_str()); + *media_proxies[i].downloader = std::make_unique(media_proxy_addr, *media_proxies[i].output_filepath, media_proxies[i].media_metadata->content_length); + *media_proxies[i].output_filepath = (*media_proxies[i].downloader)->get_output_filepath(); + if(!(*media_proxies[i].downloader)->start()) + return false; + ++num_proxied_media; + } + + if(num_proxied_media == 0) { + return false; + } + + downloader_task = AsyncTask([this]() { + sf::Clock timer; + const double sleep_time_millisec = 1; + while(!program_is_dead_in_current_thread()) { + if(youtube_video_media_proxy) + youtube_video_media_proxy->update(); + + if(youtube_audio_media_proxy) + youtube_audio_media_proxy->update(); + + const int sleep_time = sleep_time_millisec - timer.restart().asMilliseconds(); + if(sleep_time > 0) + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time)); + } + }); + + return true; + } + + static bool ffmpeg_merge_audio_and_video(const std::string &video_filepath, const std::string &audio_filepath, const std::string &output_filepath) { + const char *args[] = { "ffmpeg", "-y", "-v", "quiet", "-i", video_filepath.c_str(), "-i", audio_filepath.c_str(), "-c", "copy", "--", output_filepath.c_str(), nullptr }; + return exec_program(args, nullptr, nullptr) == 0; + } + + bool YoutubeDownloader::stop(bool download_completed) { + for(int i = 0; i < 2; ++i) { + if(downloaders[i]) { + downloaders[i]->stop(download_completed); + downloaders[i].reset(); + } + } + + if(youtube_video_media_proxy) + youtube_video_media_proxy->stop(); + + if(youtube_audio_media_proxy) + youtube_audio_media_proxy->stop(); + + if(downloader_task.valid()) + downloader_task.cancel(); + + youtube_video_media_proxy.reset(); + youtube_audio_media_proxy.reset(); + + if(!download_completed) + return false; + + bool success = true; + if(!video_metadata.url.empty() && !audio_metadata.url.empty()) { + std::string tmp_file = video_output_filepath + ".tmp.mkv"; + if(ffmpeg_merge_audio_and_video(video_output_filepath, audio_output_filepath, tmp_file)) { + success = rename_atomic(tmp_file.c_str(), output_filepath.c_str()) == 0; + if(success) { + remove(audio_output_filepath.c_str()); + } else { + remove(tmp_file.c_str()); + } + } else { + success = false; + } + } + return success; + } + + DownloadUpdateStatus YoutubeDownloader::update() { + int num_downloading = 0; + int num_finished = 0; + int num_failed = 0; + int num_downloaders = 0; + + for(int i = 0; i < 2; ++i) { + if(downloaders[i]) { + ++num_downloaders; + DownloadUpdateStatus update_status = downloaders[i]->update(); + switch(update_status) { + case DownloadUpdateStatus::DOWNLOADING: + ++num_downloading; + break; + case DownloadUpdateStatus::FINISHED: + ++num_finished; + break; + case DownloadUpdateStatus::ERROR: + ++num_failed; + break; + } + } + } + + if(num_failed > 0) + return DownloadUpdateStatus::ERROR; + else if(num_finished == num_downloaders) + return DownloadUpdateStatus::FINISHED; + else + return DownloadUpdateStatus::DOWNLOADING; + } + + float YoutubeDownloader::get_progress() { + return get_min_progress_downloader()->get_progress(); + } + + std::string YoutubeDownloader::get_progress_text() { + return get_min_progress_downloader()->get_progress_text(); + } + + std::string YoutubeDownloader::get_download_speed_text() { + return get_min_progress_downloader()->get_download_speed_text(); + } + + CurlDownloader* YoutubeDownloader::get_min_progress_downloader() { + int min_index = -1; + float progress = 999999.0f; + for(int i = 0; i < 2; ++i) { + if(downloaders[i]) { + float item_progress = downloaders[i]->get_progress(); + if(item_progress < progress) { + min_index = i; + progress = item_progress; + } + } + } + //assert(min_index != -1); + // Shouldn't happen + if(min_index == -1) + min_index = 0; + return downloaders[min_index].get(); + } +} \ No newline at end of file -- cgit v1.2.3