#include "../include/Downloader.hpp" #include "../include/Storage.hpp" #include "../include/NetUtils.hpp" #include "../include/Notification.hpp" #include #include #include namespace QuickMedia { static bool youtube_url_is_live_stream(const std::string &url) { return url.find("yt_live_broadcast") != std::string::npos || url.find("manifest/") != std::string::npos; } CurlDownloader::CurlDownloader(const std::string &url, const std::string &output_filepath, int64_t content_length) : Downloader(url, output_filepath), content_length(content_length) { output_filepath_tmp = output_filepath; output_filepath_tmp.append(".tmp"); read_program.pid = -1; read_program.read_fd = -1; progress_text = "0 bytes/Unknown"; download_speed_text = "Unknown/s"; } bool CurlDownloader::start() { remove(output_filepath_tmp.data.c_str()); if(content_length == -1) { const char *args[] = { "curl", "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", "--no-buffer", "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), "-D", "/dev/stdout", "--", url.c_str(), nullptr }; if(exec_program_pipe(args, &read_program) != 0) return false; } else { const char *args[] = { "curl", "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", "--no-buffer", "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), "--", url.c_str(), nullptr }; if(exec_program_pipe(args, &read_program) != 0) return false; } if(content_length != -1) return true; // TODO: Remove this async task and make the fd non blocking instead header_reader = AsyncTask([this]{ char tmp_buf[1024]; while(true) { ssize_t bytes_available = read(read_program.read_fd, tmp_buf, sizeof(tmp_buf)); if(bytes_available == -1) { return false; } else if(bytes_available > 0 && content_length == -1) { header.append(tmp_buf, bytes_available); if(header.find("\r\n\r\n") != std::string::npos) { std::string content_length_str = header_extract_value(header, "content-length"); if(!content_length_str.empty()) { errno = 0; char *endptr; const int64_t content_length_tmp = strtoll(content_length_str.c_str(), &endptr, 10); if(endptr != content_length_str.c_str() && errno == 0) { std::lock_guard lock(content_length_mutex); content_length = content_length_tmp; } } } } } return true; }); return true; } bool CurlDownloader::stop(bool download_completed) { if(read_program.read_fd != -1) close(read_program.read_fd); if(read_program.pid != -1) kill(read_program.pid, SIGTERM); if(!download_completed) remove(output_filepath_tmp.data.c_str()); //header_reader.cancel(); finished = false; return true; } DownloadUpdateStatus CurlDownloader::update() { if(finished) return DownloadUpdateStatus::FINISHED; if(read_program.pid == -1) return DownloadUpdateStatus::ERROR; int status = 0; if(wait_program_non_blocking(read_program.pid, &status)) { read_program.pid = -1; if(status == 0 && rename_atomic(output_filepath_tmp.data.c_str(), output_filepath.c_str()) == 0) { finished = true; progress = 1.0f; return DownloadUpdateStatus::FINISHED; } else { return DownloadUpdateStatus::ERROR; } } if(header_reader.ready()) { if(!header_reader.get()) return DownloadUpdateStatus::ERROR; } std::lock_guard lock(content_length_mutex); int64_t output_file_size = 0; file_get_size(output_filepath_tmp, &output_file_size); int64_t downloaded_size = std::min((int64_t)output_file_size, content_length); if(content_length == -1) { progress_text = std::to_string(output_file_size / 1024) + "/Unknown"; } else { size_t percentage = 0; if(output_file_size > 0) percentage = (double)downloaded_size / (double)content_length * 100.0; progress = (double)percentage / 100.0; progress_text = file_size_to_human_readable_string(downloaded_size) + "/" + file_size_to_human_readable_string(content_length) + " (" + std::to_string(percentage) + "%)"; } // TODO: Take into consideration time overflow? int64_t downloaded_diff = std::max((int64_t)0, downloaded_size - downloaded_since_last_check); download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s"; downloaded_since_last_check = downloaded_size; return DownloadUpdateStatus::DOWNLOADING; } float CurlDownloader::get_progress() { return progress; } std::string CurlDownloader::get_progress_text() { return progress_text; } std::string CurlDownloader::get_download_speed_text() { return download_speed_text; } YoutubeDlDownloader::YoutubeDlDownloader(const std::string &url, const std::string &output_filepath, bool no_video) : Downloader(url, output_filepath), no_video(no_video) { // youtube-dl requires a file extension for the file if(this->output_filepath.find('.') == std::string::npos) this->output_filepath += ".mkv"; read_program.pid = -1; read_program.read_fd = -1; progress_text = "0.0% of Unknown"; download_speed_text = "Unknown/s"; } bool YoutubeDlDownloader::start() { remove(output_filepath.c_str()); std::vector args = { "youtube-dl", "--no-warnings", "--no-continue", "--output", output_filepath.c_str(), "--newline" }; if(no_video) { args.push_back("-f"); args.push_back("bestaudio/best"); args.push_back("-x"); } else { args.push_back("-f"); args.push_back("bestvideo+bestaudio/best"); } args.insert(args.end(), { "--", url.c_str(), nullptr }); if(exec_program_pipe(args.data(), &read_program) != 0) return false; read_program_file = fdopen(read_program.read_fd, "rb"); if(!read_program_file) { if(read_program.pid != -1) wait_program(read_program.pid); return false; } // TODO: Remove this async task and make the fd non blocking instead youtube_dl_output_reader = AsyncTask([this]{ char line[128]; char progress_c[11]; char content_size_c[21]; char download_speed_c[21]; while(true) { if(fgets(line, sizeof(line), read_program_file)) { int len = strlen(line); if(len > 0 && line[len - 1] == '\n') { line[len - 1] = '\0'; --len; } if(sscanf(line, "[download] %10s of %20s at %20s", progress_c, content_size_c, download_speed_c) == 3) { std::lock_guard lock(progress_update_mutex); if(strcmp(progress_c, "Unknown") != 0 && strcmp(content_size_c, "Unknown") != 0) { std::string progress_str = progress_c; progress_text = progress_str + " of " + content_size_c; if(progress_str.back() == '%') { errno = 0; char *endptr; const double progress_tmp = strtod(progress_str.c_str(), &endptr); if(endptr != progress_str.c_str() && errno == 0) progress = progress_tmp / 100.0; } } if(strcmp(download_speed_c, "Unknown") == 0) download_speed_text = "Unknown/s"; else download_speed_text = download_speed_c; } } else { return false; } } return true; }); return true; } bool YoutubeDlDownloader::stop(bool) { if(read_program_file) fclose(read_program_file); if(read_program.pid != -1) kill(read_program.pid, SIGTERM); // TODO: Remove the temporary files created by youtube-dl (if !download_completed) //header_reader.cancel(); finished = false; return true; } DownloadUpdateStatus YoutubeDlDownloader::update() { if(finished) return DownloadUpdateStatus::FINISHED; if(read_program.pid == -1) return DownloadUpdateStatus::ERROR; int status = 0; if(wait_program_non_blocking(read_program.pid, &status)) { read_program.pid = -1; if(status == 0) { finished = true; return DownloadUpdateStatus::FINISHED; } else { return DownloadUpdateStatus::ERROR; } } if(youtube_dl_output_reader.ready()) { if(!youtube_dl_output_reader.get()) return DownloadUpdateStatus::ERROR; } return DownloadUpdateStatus::DOWNLOADING; } float YoutubeDlDownloader::get_progress() { std::lock_guard lock(progress_update_mutex); return progress; } std::string YoutubeDlDownloader::get_progress_text() { std::lock_guard lock(progress_update_mutex); return progress_text; } std::string YoutubeDlDownloader::get_download_speed_text() { std::lock_guard lock(progress_update_mutex); return download_speed_text; } static int64_t seek_fn(YoutubeReadProgram *program, int64_t offset) { if(program->read_program.pid != -1) { kill(program->read_program.pid, SIGTERM); int status; waitpid(program->read_program.pid, &status, 0); program->read_program.pid = -1; } if(program->read_program.read_fd != -1) { close(program->read_program.read_fd); program->read_program.read_fd = -1; } program->offset = offset; char range[64]; snprintf(range, sizeof(range), "%lld-%lld", (long long)program->offset, (long long)program->offset + 5242870LL); const char *args[] = { "curl", "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--no-buffer", "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", "-g", "-s", "-L", "-f", "-r", range, "--", program->url, nullptr }; int res = exec_program_pipe(args, &program->read_program); if(res != 0) return -1; return offset; } static int64_t read_fn(YoutubeReadProgram *program, char *buf, uint64_t nbytes) { ssize_t bytes_read = read(program->read_program.read_fd, buf, nbytes); if(bytes_read > 0) { program->offset += bytes_read; program->bytes_downloaded += bytes_read; } else if(bytes_read == 0) { // End of current range, progress to next range bytes_read = seek_fn(program, program->offset); if(bytes_read >= 0) { bytes_read = read(program->read_program.read_fd, buf, nbytes); if(bytes_read > 0) { program->offset += bytes_read; program->bytes_downloaded += bytes_read; } } } else if(bytes_read < 0) { if(errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) return 0; } return bytes_read; } static void close_fn(YoutubeReadProgram *program) { if(program->read_program.pid != -1) { kill(program->read_program.pid, SIGTERM); int status; waitpid(program->read_program.pid, &status, 0); program->read_program.pid = -1; } if(program->read_program.read_fd != -1) { close(program->read_program.read_fd); program->read_program.read_fd = -1; } if(program->url) { free(program->url); program->url = nullptr; } } static YoutubeReadProgram* open_fn(const char *uri) { YoutubeReadProgram *program = new YoutubeReadProgram(); program->read_program.read_fd = -1; program->read_program.pid = -1; program->url = strdup(uri); program->offset = 0; if(seek_fn(program, program->offset) < 0) { delete program; return nullptr; } return program; } YoutubeDownloader::YoutubeDownloader(const MediaMetadata &video_metadata, const MediaMetadata &audio_metadata, const std::string &output_filepath) : Downloader("", output_filepath), video_metadata(video_metadata), audio_metadata(audio_metadata) { for(int i = 0; i < 2; ++i) program[i] = nullptr; } YoutubeDownloader::~YoutubeDownloader() { for(int i = 0; i < 2; ++i) { if(program[i]) { close_fn(program[i]); delete program[i]; program[i] = nullptr; } } } bool YoutubeDownloader::start() { struct MediaMetadataInfo { MediaMetadata *media_metadata; std::string *output_filepath; bool tmp_audio_file; }; const bool has_video = !video_metadata.url.empty(); MediaMetadataInfo media_info[2] = { { &video_metadata, &video_output_filepath, false }, { &audio_metadata, &audio_output_filepath, has_video } }; int num_streams = 0; for(int i = 0; i < 2; ++i) { media_info[i].output_filepath->clear(); if(media_info[i].media_metadata->url.empty() || youtube_url_is_live_stream(media_info[i].media_metadata->url)) continue; if(media_info[i].tmp_audio_file) *media_info[i].output_filepath = output_filepath + ".audio"; else *media_info[i].output_filepath = output_filepath; program[i] = open_fn(media_info[i].media_metadata->url.c_str()); if(!program[i]) { fprintf(stderr, "Error: open failed for %s\n", media_info[i].media_metadata->url.c_str()); return false; } program[i]->content_length = media_info[i].media_metadata->content_length; program[i]->output_filepath = *media_info[i].output_filepath; program[i]->output_filepath_tmp = program[i]->output_filepath + ".tmp"; program[i]->progress_text = "0 bytes/Unknown"; program[i]->download_speed_text = "Unknown/s"; remove(program[i]->output_filepath.c_str()); remove(program[i]->output_filepath_tmp.c_str()); downloader_tasks[i] = AsyncTask([this, i]() { FILE *f = fopen(program[i]->output_filepath_tmp.c_str(), "wb"); if(!f) return false; char buffer[8192]; while(!program_is_dead_in_current_thread()) { if(program[i]->offset >= program[i]->content_length) break; int64_t bytes_read = read_fn(program[i], buffer, sizeof(buffer)); if(bytes_read > 0) fwrite(buffer, 1, bytes_read, f); } fclose(f); close_fn(program[i]); return true; }); ++num_streams; } if(num_streams == 0) return false; return true; } static bool ffmpeg_merge_audio_and_video(const std::string &video_filepath, const std::string &audio_filepath, const std::string &output_filepath) { const char *args[] = { "ffmpeg", "-y", "-v", "quiet", "-i", video_filepath.c_str(), "-i", audio_filepath.c_str(), "-c", "copy", "--", output_filepath.c_str(), nullptr }; return exec_program(args, nullptr, nullptr) == 0; } bool YoutubeDownloader::stop(bool download_completed) { bool success = true; for(int i = 0; i < 2; ++i) { if(program[i]) { downloader_tasks[i].cancel(); if(downloader_tasks[i].ready() && !downloader_tasks[i].get()) success = false; if(rename_atomic(program[i]->output_filepath_tmp.c_str(), program[i]->output_filepath.c_str()) != 0) success = false; close_fn(program[i]); } } if(!download_completed || !success) return false; success = true; if(!video_metadata.url.empty() && !audio_metadata.url.empty()) { std::string tmp_file = video_output_filepath + ".tmp.mkv"; if(ffmpeg_merge_audio_and_video(video_output_filepath, audio_output_filepath, tmp_file)) { success = rename_atomic(tmp_file.c_str(), output_filepath.c_str()) == 0; if(success) { remove(audio_output_filepath.c_str()); } else { remove(tmp_file.c_str()); } } else { success = false; } } return success; } DownloadUpdateStatus YoutubeDownloader::update() { int num_downloading = 0; int num_finished = 0; int num_failed = 0; int num_downloaders = 0; for(int i = 0; i < 2; ++i) { if(program[i]) { ++num_downloaders; DownloadUpdateStatus update_status = update(i); switch(update_status) { case DownloadUpdateStatus::DOWNLOADING: ++num_downloading; break; case DownloadUpdateStatus::FINISHED: ++num_finished; break; case DownloadUpdateStatus::ERROR: ++num_failed; break; } } } if(num_failed > 0) return DownloadUpdateStatus::ERROR; else if(num_finished == num_downloaders) return DownloadUpdateStatus::FINISHED; else return DownloadUpdateStatus::DOWNLOADING; } float YoutubeDownloader::get_progress() { YoutubeReadProgram *min_program = get_min_progress_downloader(); return min_program->progress; } std::string YoutubeDownloader::get_progress_text() { YoutubeReadProgram *min_program = get_min_progress_downloader(); return min_program->progress_text; } std::string YoutubeDownloader::get_download_speed_text() { YoutubeReadProgram *min_program = get_min_progress_downloader(); return min_program->download_speed_text; } YoutubeReadProgram* YoutubeDownloader::get_min_progress_downloader() { int min_index = -1; float progress = 999999.0f; for(int i = 0; i < 2; ++i) { if(program[i]) { if(program[i]->progress < progress) { min_index = i; progress = program[i]->progress; } } } if(min_index == -1) min_index = 0; return program[min_index]; } DownloadUpdateStatus YoutubeDownloader::update(size_t program_index) { bool finished = false; YoutubeReadProgram *youtube_program = program[program_index]; auto &downloader_task = downloader_tasks[program_index]; if(youtube_program->finished) return DownloadUpdateStatus::FINISHED; if(downloader_task.ready()) { if(downloader_task.get()) { finished = true; youtube_program->finished = true; } else { return DownloadUpdateStatus::ERROR; } } int64_t output_file_size = 0; file_get_size(youtube_program->output_filepath_tmp, &output_file_size); int64_t downloaded_size = std::min((int64_t)output_file_size, youtube_program->content_length); int64_t percentage = 0; if(output_file_size > 0) percentage = (double)downloaded_size / (double)youtube_program->content_length * 100.0; youtube_program->progress = (double)percentage / 100.0; youtube_program->progress_text = file_size_to_human_readable_string(downloaded_size) + "/" + file_size_to_human_readable_string(youtube_program->content_length) + " (" + std::to_string(percentage) + "%)"; // TODO: Take into consideration time overflow? int64_t downloaded_diff = std::max((int64_t)0, downloaded_size - youtube_program->downloaded_since_last_check); youtube_program->download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s"; youtube_program->downloaded_since_last_check = downloaded_size; return finished ? DownloadUpdateStatus::FINISHED : DownloadUpdateStatus::DOWNLOADING; } }