diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | TODO | 3 | ||||
| -rw-r--r-- | include/DownloadUtils.hpp | 2 | ||||
| -rw-r--r-- | include/Downloader.hpp | 104 | ||||
| -rw-r--r-- | include/QuickMedia.hpp | 4 | ||||
| -rw-r--r-- | plugins/Page.hpp | 5 | ||||
| -rw-r--r-- | plugins/Youtube.hpp | 6 | ||||
| -rw-r--r-- | plugins/youtube/YoutubeMediaProxy.hpp | 14 | ||||
| -rw-r--r-- | src/AsyncImageLoader.cpp | 6 | ||||
| -rw-r--r-- | src/DownloadUtils.cpp | 3 | ||||
| -rw-r--r-- | src/Downloader.cpp | 475 | ||||
| -rw-r--r-- | src/QuickMedia.cpp | 443 | ||||
| -rw-r--r-- | src/plugins/Youtube.cpp | 31 | ||||
| -rw-r--r-- | src/plugins/youtube/YoutubeMediaProxy.cpp | 28 |
14 files changed, 769 insertions, 359 deletions
@@ -33,12 +33,12 @@ If you are running arch linux then you can install QuickMedia from aur (https:// ### Optional `noto-fonts-cjk` needs to be installed to view chinese, japanese and korean characters.\ `mpv` needs to be installed to play videos.\ -`youtube-dl` needs to be installed to download youtube music/videos. (Note: `youtube-dl` is not required to watch (stream) youtube music/videos).\ +`youtube-dl` needs to be installed to play/download xxx videos.\ `libnotify` which provides `notify-send` needs to be installed to show notifications (on Linux and other systems that uses d-bus notification system).\ [automedia](https://git.dec05eba.com/AutoMedia/) needs to be installed when tracking manga with `Ctrl + T`.\ `waifu2x-ncnn-vulkan` needs to be installed when using the `--upscale-images` or `--upscale-images-always` option.\ `xdg-utils` which provides `xdg-open` needs to be installed when downloading torrents with `nyaa.si` plugin.\ -`ffmpeg (and ffprobe which is included in ffmpeg)` needs to be installed to upload videos with thumbnails on matrix. +`ffmpeg (and ffprobe which is included in ffmpeg)` needs to be installed to display webp thumbnails, to upload videos with thumbnails on matrix or to merge video and audio when downloading youtube videos. ## Controls ### General control Type text and then wait and QuickMedia will automatically search.\ @@ -155,7 +155,6 @@ Embedding elements in rich text: first byte should be an invalid utf8 character Do not set fps to monitor hz if no key is pressed and cursor is not moving (for example when the computer is sleeping). (Maybe just detect if any sfml event is received), but maybe that wouldn't work with accelerometer. Ctrl+arrow key to move to previous/next video. Add keybinding to view file-manager images in fullscreen to preview them. Also add keybinding to create/delete directories. -Completely remove youtube-dl dependency (or at least for downloading videos/music). Add loading of english subtitles for youtube. Update item height when it switches from not being merged with previous to being merged with previous. This happens when loading previous messages in matrix and the message is the top one. Reload youtube video url if the video is idle for too long. The video url is only valid for a specific amount of time (the valid duration is in the json). @@ -173,3 +172,5 @@ Use the new media proxy for downloading youtube videos as well. PgUp/PgDown shouldn't move body by the number of visible items. It should instead move by the height of the body. Add option to view dead link in 4chan with 4chan archive and navigate to crossboard links. Show latest message before sync is done for a room when the latest message is an edit. Right now it has to fetch previous messages until the first non-edit message. +Allow resuming downloads. +Support downloading live youtube videos.
\ No newline at end of file diff --git a/include/DownloadUtils.hpp b/include/DownloadUtils.hpp index dbac921..398f82c 100644 --- a/include/DownloadUtils.hpp +++ b/include/DownloadUtils.hpp @@ -32,6 +32,6 @@ namespace QuickMedia { // Note: if |cloudflare_bypass| is set to true then tls is limited to version 1.1 and the user agent is changed. DownloadResult download_to_file(const std::string &url, const std::string &destination_filepath, const std::vector<CommandArg> &additional_args, bool use_browser_useragent = false, bool cloudflare_bypass = false); // Returns false if there was an error trying to create the download process - bool download_async_gui(const std::string &url, const std::string &file_manager_start_dir, bool use_youtube_dl, bool no_video); + bool download_async_gui(const std::string &url, const std::string &file_manager_start_dir, bool no_video); DownloadResult download_to_json(const std::string &url, rapidjson::Document &result, const std::vector<CommandArg> &additional_args, bool use_browser_useragent = false, bool fail_on_error = true); }
\ No newline at end of file diff --git a/include/Downloader.hpp b/include/Downloader.hpp new file mode 100644 index 0000000..f09f9bf --- /dev/null +++ b/include/Downloader.hpp @@ -0,0 +1,104 @@ +#pragma once + +#include "Path.hpp" +#include "AsyncTask.hpp" +#include "../plugins/youtube/YoutubeMediaProxy.hpp" +#include <string> + +namespace QuickMedia { + enum class DownloadUpdateStatus { + DOWNLOADING, + FINISHED, + ERROR + }; + + class Downloader { + public: + Downloader(const std::string &url, const std::string &output_filepath) : url(url), output_filepath(output_filepath) {} + virtual ~Downloader() = default; + + virtual bool start() = 0; + virtual bool stop(bool download_completed) = 0; + virtual DownloadUpdateStatus update() = 0; + + virtual float get_progress() = 0; + virtual std::string get_progress_text() = 0; + virtual std::string get_download_speed_text() = 0; + + const std::string& get_output_filepath() { return output_filepath; } + protected: + std::string url; + std::string output_filepath; + }; + + class CurlDownloader : public Downloader { + public: + CurlDownloader(const std::string &url, const std::string &output_filepath, int64_t content_length = -1); + bool start() override; + bool stop(bool download_completed) override; + DownloadUpdateStatus update() override; + float get_progress() override; + std::string get_progress_text() override; + std::string get_download_speed_text() override; + private: + Path output_filepath_tmp; + ReadProgram read_program; + AsyncTask<bool> header_reader; + std::string header; + int64_t content_length = -1; + size_t downloaded_since_last_check = 0; + float progress = 0.0f; + std::mutex content_length_mutex; + std::string progress_text; + std::string download_speed_text; + bool finished = false; + }; + + class YoutubeDlDownloader : public Downloader { + public: + YoutubeDlDownloader(const std::string &url, const std::string &output_filepath, bool no_video); + bool start() override; + bool stop(bool download_completed) override; + DownloadUpdateStatus update() override; + float get_progress() override; + std::string get_progress_text() override; + std::string get_download_speed_text() override; + private: + ReadProgram read_program; + FILE *read_program_file = nullptr; + AsyncTask<bool> youtube_dl_output_reader; + std::mutex progress_update_mutex; + float progress = 0.0f; + std::string progress_text; + std::string download_speed_text; + bool no_video; + bool finished = false; + }; + + struct MediaMetadata { + std::string url; + int64_t content_length; + }; + + class YoutubeDownloader : public Downloader { + public: + YoutubeDownloader(const MediaMetadata &video_metadata, const MediaMetadata &audio_metadata, const std::string &output_filepath); + bool start() override; + bool stop(bool download_completed) override; + DownloadUpdateStatus update() override; + float get_progress() override; + std::string get_progress_text() override; + std::string get_download_speed_text() override; + private: + CurlDownloader* get_min_progress_downloader(); + private: + MediaMetadata video_metadata; + MediaMetadata audio_metadata; + std::string video_output_filepath; + std::string audio_output_filepath; + std::unique_ptr<CurlDownloader> downloaders[2]; + AsyncTask<void> downloader_task; + std::unique_ptr<YoutubeMediaProxy> youtube_video_media_proxy; + std::unique_ptr<YoutubeMediaProxy> youtube_audio_media_proxy; + }; +}
\ No newline at end of file diff --git a/include/QuickMedia.hpp b/include/QuickMedia.hpp index cc4a10f..765a346 100644 --- a/include/QuickMedia.hpp +++ b/include/QuickMedia.hpp @@ -113,7 +113,7 @@ namespace QuickMedia { using PageLoopSubmitHandler = std::function<void(const std::vector<Tab> &new_tabs)>; // Returns false if the page loop was escaped by user navigation (pressing escape) or if there was an error at startup bool page_loop(std::vector<Tab> &tabs, int start_tab_index = 0, PageLoopSubmitHandler after_submit_handler = nullptr); - void video_page_download_video(const std::string &url, bool use_youtube_dl, sf::WindowHandle video_player_window = None); + void video_page_download_video(const std::string &url, sf::WindowHandle video_player_window = None); bool video_download_if_non_streamable(std::string &video_url, std::string &audio_url, bool &is_audio_only, bool &has_embedded_audio, PageType previous_page); void video_content_page(Page *parent_page, VideoPage *video_page, std::string video_title, bool download_if_streaming_fails, Body *parent_body, BodyItems &next_play_items, int play_index, int *parent_body_page = nullptr, const std::string &parent_page_search = ""); // Returns -1 to go to previous chapter, 0 to stay on same chapter and 1 to go to next chapter @@ -123,7 +123,7 @@ namespace QuickMedia { void chat_login_page(); bool chat_page(MatrixChatPage *matrix_chat_page, RoomData *current_room); void after_matrix_login_page(); - void download_page(const char *url, bool download_use_youtube_dl); + void download_page(const std::string &url); // Returns the full path where the file should be saved, or an empty string if the operation was cancelled std::string file_save_page(const std::string &filename); diff --git a/plugins/Page.hpp b/plugins/Page.hpp index 3a382ec..35e778b 100644 --- a/plugins/Page.hpp +++ b/plugins/Page.hpp @@ -120,14 +120,15 @@ namespace QuickMedia { virtual std::string get_url_timestamp() { return ""; } // Falls back to |get_url| if this and |get_audio_url| returns empty strings. // Might do a network request. - virtual std::string get_video_url(int max_height, bool &has_embedded_audio) { + virtual std::string get_video_url(int max_height, bool &has_embedded_audio, std::string &ext) { (void)max_height; + (void)ext; has_embedded_audio = true; return ""; } // Only used if |get_video_url| sets |has_embedded_audio| to false. // Might do a network request. - virtual std::string get_audio_url() { return ""; } + virtual std::string get_audio_url(std::string &ext) { (void)ext; return ""; } virtual std::string url_get_playable_url(const std::string &url) { return url; } virtual bool video_should_be_skipped(const std::string &url) { (void)url; return false; } virtual PluginResult load(std::string &title, std::string &channel_url, std::vector<MediaChapter> &chapters) { (void)title; (void)channel_url; (void)chapters; return PluginResult::OK; } diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp index 2958c44..c89d68d 100644 --- a/plugins/Youtube.hpp +++ b/plugins/Youtube.hpp @@ -26,7 +26,7 @@ namespace QuickMedia { bool youtube_url_extract_id(const std::string &youtube_url, std::string &youtube_video_id); // |video_url| or |audio_url| will be empty if there is an error and false will be returned. // If false is returned from |active_handler|, then this function is cancelled. - bool youtube_custom_redirect(std::string &video_url, std::string &audio_url, int &video_content_length, int &audio_content_length, std::function<bool()> active_handler); + bool youtube_custom_redirect(std::string &video_url, std::string &audio_url, int64_t &video_content_length, int64_t &audio_content_length, std::function<bool()> active_handler); class YoutubeSearchPage : public LazyFetchPage { public: @@ -144,8 +144,8 @@ namespace QuickMedia { std::unique_ptr<Page> create_channels_page(Program *program, const std::string &channel_url) override; void set_url(std::string new_url) override; std::string get_url_timestamp() override { return timestamp; } - std::string get_video_url(int max_height, bool &has_embedded_audio) override; - std::string get_audio_url() override; + std::string get_video_url(int max_height, bool &has_embedded_audio, std::string &ext) override; + std::string get_audio_url(std::string &ext) override; PluginResult load(std::string &title, std::string &channel_url, std::vector<MediaChapter> &chapters) override; void mark_watched() override; private: diff --git a/plugins/youtube/YoutubeMediaProxy.hpp b/plugins/youtube/YoutubeMediaProxy.hpp index 2d6ea93..28aa6fe 100644 --- a/plugins/youtube/YoutubeMediaProxy.hpp +++ b/plugins/youtube/YoutubeMediaProxy.hpp @@ -15,7 +15,7 @@ namespace QuickMedia { virtual ~YoutubeMediaProxy() = default; - virtual bool start(const std::string &youtube_media_url, int content_length) = 0; + virtual bool start(const std::string &youtube_media_url, int64_t content_length) = 0; // This should be the last call. Do not call |start| after this. TODO: Allow restarting with |start| after |stop| is called. virtual void stop() = 0; virtual Error update() = 0; @@ -34,7 +34,7 @@ namespace QuickMedia { YoutubeStaticMediaProxy&operator=(YoutubeStaticMediaProxy&) = delete; ~YoutubeStaticMediaProxy(); - bool start(const std::string &youtube_media_url, int content_length) override; + bool start(const std::string &youtube_media_url, int64_t content_length) override; void stop() override; Error update() override; bool get_address(std::string &address) override; @@ -43,7 +43,7 @@ namespace QuickMedia { Error read_client_data(); void clear_download_state(); // If |new_range_start| is not -1 then the start range is set to that - Error update_download_program_status(bool client_disconnected, int new_range_start = -1, bool restart_download = false); + Error update_download_program_status(bool client_disconnected, int64_t new_range_start = -1, bool restart_download = false); Error continue_send(const char *buffer_start, size_t total_bytes_to_write, int &buffer_offset); Error handle_download(); // Returns the client fd or -1 @@ -53,12 +53,12 @@ namespace QuickMedia { int port = 0; int client_fd = -1; std::string youtube_media_url; - int content_length = 0; + int64_t content_length = 0; ReadProgram downloader_read_program; int download_read_buffer_offset = 0; ssize_t downloader_num_read_bytes = 0; - int download_range_start = 0; - int current_download_range = 0; + int64_t download_range_start = 0; + int64_t current_download_range = 0; std::string download_header; bool download_header_finished = false; bool download_header_sent = false; @@ -77,7 +77,7 @@ namespace QuickMedia { YoutubeLiveStreamMediaProxy&operator=(YoutubeLiveStreamMediaProxy&) = delete; ~YoutubeLiveStreamMediaProxy(); - bool start(const std::string &youtube_media_url, int content_length) override; + bool start(const std::string &youtube_media_url, int64_t content_length) override; void stop() override; Error update() override; bool get_address(std::string &address) override; diff --git a/src/AsyncImageLoader.cpp b/src/AsyncImageLoader.cpp index 3e00459..ccf9f42 100644 --- a/src/AsyncImageLoader.cpp +++ b/src/AsyncImageLoader.cpp @@ -21,11 +21,7 @@ namespace QuickMedia { static bool webp_to_png(const Path &thumbnail_path, const Path &destination_path) { const char *args[] = { "ffmpeg", "-y", "-v", "quiet", "-i", thumbnail_path.data.c_str(), "--", destination_path.data.c_str(), nullptr}; - int res = exec_program(args, nullptr, nullptr); - if(res != 0) - return false; - - return true; + return exec_program(args, nullptr, nullptr) == 0; } bool create_thumbnail(const Path &thumbnail_path, const Path &thumbnail_path_resized, sf::Vector2i resize_target_size, ContentType content_type) { diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp index dff0ecb..17612c6 100644 --- a/src/DownloadUtils.cpp +++ b/src/DownloadUtils.cpp @@ -283,7 +283,7 @@ namespace QuickMedia { return DownloadResult::OK; } - bool download_async_gui(const std::string &url, const std::string &file_manager_start_dir, bool use_youtube_dl, bool no_video) { + bool download_async_gui(const std::string &url, const std::string &file_manager_start_dir, bool no_video) { char quickmedia_path[PATH_MAX]; ssize_t bytes_written = readlink("/proc/self/exe", quickmedia_path, sizeof(quickmedia_path)); if(bytes_written == -1 || bytes_written == sizeof(quickmedia_path)) @@ -292,7 +292,6 @@ namespace QuickMedia { quickmedia_path[bytes_written] = '\0'; std::vector<const char*> args = { quickmedia_path, "download", "-u", url.c_str(), "--dir", file_manager_start_dir.c_str() }; - if(use_youtube_dl) args.push_back("--youtube-dl"); if(no_video) args.push_back("--no-video"); args.push_back(nullptr); return exec_program_async(args.data(), nullptr) == 0; diff --git a/src/Downloader.cpp b/src/Downloader.cpp new file mode 100644 index 0000000..ffb8675 --- /dev/null +++ b/src/Downloader.cpp @@ -0,0 +1,475 @@ +#include "../include/Downloader.hpp" +#include "../include/Storage.hpp" +#include "../include/NetUtils.hpp" +#include "../include/Notification.hpp" +#include <SFML/System/Clock.hpp> +#include <unistd.h> +#include <signal.h> + +namespace QuickMedia { + // TODO: Test with video that has hlsManifestUrl + static bool youtube_url_is_live_stream(const std::string &url) { + return url.find("yt_live_broadcast") != std::string::npos; + } + + CurlDownloader::CurlDownloader(const std::string &url, const std::string &output_filepath, int64_t content_length) : Downloader(url, output_filepath), content_length(content_length) { + output_filepath_tmp = output_filepath; + output_filepath_tmp.append(".tmp"); + read_program.pid = -1; + read_program.read_fd = -1; + progress_text = "0 bytes/Unknown"; + download_speed_text = "Unknown/s"; + } + + bool CurlDownloader::start() { + remove(output_filepath_tmp.data.c_str()); + + if(content_length == -1) { + const char *args[] = { "curl", + "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", + "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), + "-D", "/dev/stdout", + "--", url.c_str(), nullptr }; + + if(exec_program_pipe(args, &read_program) != 0) + return false; + } else { + const char *args[] = { "curl", + "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", + "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), + "--", url.c_str(), nullptr }; + + if(exec_program_pipe(args, &read_program) != 0) + return false; + } + + if(content_length != -1) + return true; + + // TODO: Remove this async task and make the fd non blocking instead + header_reader = AsyncTask<bool>([this]{ + char tmp_buf[1024]; + while(true) { + ssize_t bytes_available = read(read_program.read_fd, tmp_buf, sizeof(tmp_buf)); + if(bytes_available == -1) { + return false; + } else if(bytes_available > 0 && content_length == -1) { + header.append(tmp_buf, bytes_available); + if(header.find("\r\n\r\n") != std::string::npos) { + std::string content_length_str = header_extract_value(header, "content-length"); + if(!content_length_str.empty()) { + errno = 0; + char *endptr; + const long content_length_tmp = strtol(content_length_str.c_str(), &endptr, 10); + if(endptr != content_length_str.c_str() && errno == 0) { + std::lock_guard<std::mutex> lock(content_length_mutex); + content_length = content_length_tmp; + } + } + } + } + } + return true; + }); + + return true; + } + + bool CurlDownloader::stop(bool download_completed) { + if(read_program.read_fd != -1) + close(read_program.read_fd); + if(read_program.pid != -1) + kill(read_program.pid, SIGTERM); + if(!download_completed) + remove(output_filepath_tmp.data.c_str()); + //header_reader.cancel(); + finished = false; + return true; + } + + DownloadUpdateStatus CurlDownloader::update() { + if(finished) + return DownloadUpdateStatus::FINISHED; + + if(read_program.pid == -1) + return DownloadUpdateStatus::ERROR; + + int status = 0; + if(wait_program_non_blocking(read_program.pid, &status)) { + read_program.pid = -1; + if(status == 0 && rename_atomic(output_filepath_tmp.data.c_str(), output_filepath.c_str()) == 0) { + finished = true; + progress = 1.0f; + return DownloadUpdateStatus::FINISHED; + } else { + return DownloadUpdateStatus::ERROR; + } + } + + if(header_reader.ready()) { + if(!header_reader.get()) + return DownloadUpdateStatus::ERROR; + } + + std::lock_guard<std::mutex> lock(content_length_mutex); + size_t output_file_size = 0; + file_get_size(output_filepath_tmp, &output_file_size); + size_t downloaded_size = std::min((int64_t)output_file_size, content_length); + + if(content_length == -1) { + progress_text = std::to_string(output_file_size / 1024) + "/Unknown"; + } else { + size_t percentage = 0; + if(output_file_size > 0) + percentage = (double)downloaded_size / (double)content_length * 100.0; + progress = (double)percentage / 100.0; + progress_text = file_size_to_human_readable_string(downloaded_size) + "/" + file_size_to_human_readable_string(content_length) + " (" + std::to_string(percentage) + "%)"; + } + + // TODO: Take into consideration time overflow? + size_t downloaded_diff = std::max(0lu, downloaded_size - downloaded_since_last_check); + download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s"; + downloaded_since_last_check = downloaded_size; + + return DownloadUpdateStatus::DOWNLOADING; + } + + float CurlDownloader::get_progress() { + return progress; + } + + std::string CurlDownloader::get_progress_text() { + return progress_text; + } + + std::string CurlDownloader::get_download_speed_text() { + return download_speed_text; + } + + YoutubeDlDownloader::YoutubeDlDownloader(const std::string &url, const std::string &output_filepath, bool no_video) : Downloader(url, output_filepath), no_video(no_video) { + // youtube-dl requires a file extension for the file + if(this->output_filepath.find('.') == std::string::npos) + this->output_filepath += ".mkv"; + + read_program.pid = -1; + read_program.read_fd = -1; + progress_text = "0.0% of Unknown"; + download_speed_text = "Unknown/s"; + } + + bool YoutubeDlDownloader::start() { + remove(output_filepath.c_str()); + + std::vector<const char*> args = { "youtube-dl", "--no-warnings", "--no-continue", "--output", output_filepath.c_str(), "--newline" }; + if(no_video) { + args.push_back("-f"); + args.push_back("bestaudio/best"); + args.push_back("-x"); + } else { + args.push_back("-f"); + args.push_back("bestvideo+bestaudio/best"); + } + args.insert(args.end(), { "--", url.c_str(), nullptr }); + + if(exec_program_pipe(args.data(), &read_program) != 0) + return false; + + read_program_file = fdopen(read_program.read_fd, "rb"); + if(!read_program_file) { + if(read_program.pid != -1) + wait_program(read_program.pid); + return false; + } + + // TODO: Remove this async task and make the fd non blocking instead + youtube_dl_output_reader = AsyncTask<bool>([this]{ + char line[128]; + char progress_c[10]; + char content_size_c[20]; + char download_speed_c[20]; + + while(true) { + if(fgets(line, sizeof(line), read_program_file)) { + int len = strlen(line); + if(len > 0 && line[len - 1] == '\n') { + line[len - 1] = '\0'; + --len; + } + + if(sscanf(line, "[download] %10s of %20s at %20s", progress_c, content_size_c, download_speed_c) == 3) { + std::lock_guard<std::mutex> lock(progress_update_mutex); + + if(strcmp(progress_c, "Unknown") != 0 && strcmp(content_size_c, "Unknown") != 0) { + std::string progress_str = progress_c; + progress_text = progress_str + " of " + content_size_c; + if(progress_str.back() == '%') { + errno = 0; + char *endptr; + const double progress_tmp = strtod(progress_str.c_str(), &endptr); + if(endptr != progress_str.c_str() && errno == 0) + progress = progress_tmp / 100.0; + } + } + + if(strcmp(download_speed_c, "Unknown") == 0) + download_speed_text = "Unknown/s"; + else + download_speed_text = download_speed_c; + } + } else { + return false; + } + } + + return true; + }); + + return true; + } + + bool YoutubeDlDownloader::stop(bool) { + if(read_program_file) + fclose(read_program_file); + if(read_program.pid != -1) + kill(read_program.pid, SIGTERM); + // TODO: Remove the temporary files created by youtube-dl (if !download_completed) + //header_reader.cancel(); + finished = false; + return true; + } + + DownloadUpdateStatus YoutubeDlDownloader::update() { + if(finished) + return DownloadUpdateStatus::FINISHED; + + if(read_program.pid == -1) + return DownloadUpdateStatus::ERROR; + + int status = 0; + if(wait_program_non_blocking(read_program.pid, &status)) { + read_program.pid = -1; + if(status == 0) { + finished = true; + return DownloadUpdateStatus::FINISHED; + } else { + return DownloadUpdateStatus::ERROR; + } + } + + if(youtube_dl_output_reader.ready()) { + if(!youtube_dl_output_reader.get()) + return DownloadUpdateStatus::ERROR; + } + + return DownloadUpdateStatus::DOWNLOADING; + } + + float YoutubeDlDownloader::get_progress() { + std::lock_guard<std::mutex> lock(progress_update_mutex); + return progress; + } + + std::string YoutubeDlDownloader::get_progress_text() { + std::lock_guard<std::mutex> lock(progress_update_mutex); + return progress_text; + } + + std::string YoutubeDlDownloader::get_download_speed_text() { + std::lock_guard<std::mutex> lock(progress_update_mutex); + return download_speed_text; + } + + YoutubeDownloader::YoutubeDownloader(const MediaMetadata &video_metadata, const MediaMetadata &audio_metadata, const std::string &output_filepath) : + Downloader("", output_filepath), + video_metadata(video_metadata), + audio_metadata(audio_metadata) + { + for(int i = 0; i < 2; ++i) + downloaders[i] = nullptr; + } + + bool YoutubeDownloader::start() { + struct MediaProxyMetadata { + std::unique_ptr<YoutubeMediaProxy> *media_proxy; + MediaMetadata *media_metadata; + std::unique_ptr<CurlDownloader> *downloader; + std::string *output_filepath; + bool tmp_audio_file; + }; + + const bool has_video = !video_metadata.url.empty(); + + MediaProxyMetadata media_proxies[2] = { + { &youtube_video_media_proxy, &video_metadata, &downloaders[0], &video_output_filepath, false }, + { &youtube_audio_media_proxy, &audio_metadata, &downloaders[1], &audio_output_filepath, has_video } + }; + + int num_proxied_media = 0; + for(int i = 0; i < 2; ++i) { + media_proxies[i].output_filepath->clear(); + if(media_proxies[i].media_metadata->url.empty()) + continue; + + if(youtube_url_is_live_stream(media_proxies[i].media_metadata->url)) + *media_proxies[i].media_proxy = std::make_unique<YoutubeLiveStreamMediaProxy>(); + else + *media_proxies[i].media_proxy = std::make_unique<YoutubeStaticMediaProxy>(); + + if(!(*media_proxies[i].media_proxy)->start(media_proxies[i].media_metadata->url, media_proxies[i].media_metadata->content_length)) { + fprintf(stderr, "Failed to load start youtube media proxy\n"); + return false; + } + + std::string media_proxy_addr; + if(!(*media_proxies[i].media_proxy)->get_address(media_proxy_addr)) { + fprintf(stderr, "Failed to load start youtube media proxy\n"); + return false; + } + + if(media_proxies[i].tmp_audio_file) + *media_proxies[i].output_filepath = output_filepath + ".audio"; + else + *media_proxies[i].output_filepath = output_filepath; + + remove(media_proxies[i].output_filepath->c_str()); + fprintf(stderr, "Downloading %s to %s\n", media_proxy_addr.c_str(), media_proxies[i].output_filepath->c_str()); + *media_proxies[i].downloader = std::make_unique<CurlDownloader>(media_proxy_addr, *media_proxies[i].output_filepath, media_proxies[i].media_metadata->content_length); + *media_proxies[i].output_filepath = (*media_proxies[i].downloader)->get_output_filepath(); + if(!(*media_proxies[i].downloader)->start()) + return false; + ++num_proxied_media; + } + + if(num_proxied_media == 0) { + return false; + } + + downloader_task = AsyncTask<void>([this]() { + sf::Clock timer; + const double sleep_time_millisec = 1; + while(!program_is_dead_in_current_thread()) { + if(youtube_video_media_proxy) + youtube_video_media_proxy->update(); + + if(youtube_audio_media_proxy) + youtube_audio_media_proxy->update(); + + const int sleep_time = sleep_time_millisec - timer.restart().asMilliseconds(); + if(sleep_time > 0) + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time)); + } + }); + + return true; + } + + static bool ffmpeg_merge_audio_and_video(const std::string &video_filepath, const std::string &audio_filepath, const std::string &output_filepath) { + const char *args[] = { "ffmpeg", "-y", "-v", "quiet", "-i", video_filepath.c_str(), "-i", audio_filepath.c_str(), "-c", "copy", "--", output_filepath.c_str(), nullptr }; + return exec_program(args, nullptr, nullptr) == 0; + } + + bool YoutubeDownloader::stop(bool download_completed) { + for(int i = 0; i < 2; ++i) { + if(downloaders[i]) { + downloaders[i]->stop(download_completed); + downloaders[i].reset(); + } + } + + if(youtube_video_media_proxy) + youtube_video_media_proxy->stop(); + + if(youtube_audio_media_proxy) + youtube_audio_media_proxy->stop(); + + if(downloader_task.valid()) + downloader_task.cancel(); + + youtube_video_media_proxy.reset(); + youtube_audio_media_proxy.reset(); + + if(!download_completed) + return false; + + bool success = true; + if(!video_metadata.url.empty() && !audio_metadata.url.empty()) { + std::string tmp_file = video_output_filepath + ".tmp.mkv"; + if(ffmpeg_merge_audio_and_video(video_output_filepath, audio_output_filepath, tmp_file)) { + success = rename_atomic(tmp_file.c_str(), output_filepath.c_str()) == 0; + if(success) { + remove(audio_output_filepath.c_str()); + } else { + remove(tmp_file.c_str()); + } + } else { + success = false; + } + } + return success; + } + + DownloadUpdateStatus YoutubeDownloader::update() { + int num_downloading = 0; + int num_finished = 0; + int num_failed = 0; + int num_downloaders = 0; + + for(int i = 0; i < 2; ++i) { + if(downloaders[i]) { + ++num_downloaders; + DownloadUpdateStatus update_status = downloaders[i]->update(); + switch(update_status) { + case DownloadUpdateStatus::DOWNLOADING: + ++num_downloading; + break; + case DownloadUpdateStatus::FINISHED: + ++num_finished; + break; + case DownloadUpdateStatus::ERROR: + ++num_failed; + break; + } + } + } + + if(num_failed > 0) + return DownloadUpdateStatus::ERROR; + else if(num_finished == num_downloaders) + return DownloadUpdateStatus::FINISHED; + else + return DownloadUpdateStatus::DOWNLOADING; + } + + float YoutubeDownloader::get_progress() { + return get_min_progress_downloader()->get_progress(); + } + + std::string YoutubeDownloader::get_progress_text() { + return get_min_progress_downloader()->get_progress_text(); + } + + std::string YoutubeDownloader::get_download_speed_text() { + return get_min_progress_downloader()->get_download_speed_text(); + } + + CurlDownloader* YoutubeDownloader::get_min_progress_downloader() { + int min_index = -1; + float progress = 999999.0f; + for(int i = 0; i < 2; ++i) { + if(downloaders[i]) { + float item_progress = downloaders[i]->get_progress(); + if(item_progress < progress) { + min_index = i; + progress = item_progress; + } + } + } + //assert(min_index != -1); + // Shouldn't happen + if(min_index == -1) + min_index = 0; + return downloaders[min_index].get(); + } +}
\ No newline at end of file diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 64833bd..fb6b425 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -29,6 +29,7 @@ #include "../include/Utils.hpp" #include "../include/Tabs.hpp" #include "../include/Theme.hpp" +#include "../include/Downloader.hpp" #include "../plugins/youtube/YoutubeMediaProxy.hpp" #include "../include/gui/Button.hpp" #include "../external/hash-library/sha256.h" @@ -387,7 +388,6 @@ namespace QuickMedia { Window parent_window = None; std::vector<Tab> tabs; const char *url = nullptr; - bool download_use_youtube_dl = false; std::string program_path = dirname(argv[0]); for(int i = 1; i < argc; ++i) { @@ -425,8 +425,6 @@ namespace QuickMedia { } } else if(strcmp(argv[i], "--low-cpu-mode") == 0) { low_cpu_mode = true; - } else if(strcmp(argv[i], "--youtube-dl") == 0) { - download_use_youtube_dl = true; } else if(strcmp(argv[i], "-u") == 0) { if(i < argc - 1) { url = argv[i + 1]; @@ -533,7 +531,7 @@ namespace QuickMedia { usage(); return -1; } - download_page(url, download_use_youtube_dl); + download_page(url); return exit_code; } @@ -2404,9 +2402,17 @@ namespace QuickMedia { return false; } - void Program::video_page_download_video(const std::string &url, bool use_youtube_dl, sf::WindowHandle video_player_window) { - if(!use_youtube_dl) { - download_async_gui(url, file_manager_start_dir.string(), use_youtube_dl, no_video); + static bool url_should_download_with_youtube_dl(const std::string &url) { + return url.find("pornhub.com") != std::string::npos || url.find("xhamster.com") != std::string::npos || url.find("spankbang.com") != std::string::npos || url.find("xvideos.com") != std::string::npos; + } + + void Program::video_page_download_video(const std::string &url, sf::WindowHandle video_player_window) { + bool separate_audio_option = url_should_download_with_youtube_dl(url);; + std::string video_id; + separate_audio_option |= youtube_url_extract_id(url, video_id); + + if(!separate_audio_option) { + download_async_gui(url, file_manager_start_dir.string(), no_video); return; } @@ -2438,7 +2444,7 @@ namespace QuickMedia { if(!selected) return; - download_async_gui(url, file_manager_start_dir.string(), true, audio_only); + download_async_gui(url, file_manager_start_dir.string(), audio_only); } bool Program::video_download_if_non_streamable(std::string &video_url, std::string &audio_url, bool &is_audio_only, bool &has_embedded_audio, PageType previous_page) { @@ -2542,8 +2548,8 @@ namespace QuickMedia { std::unique_ptr<YoutubeMediaProxy> youtube_video_media_proxy; std::unique_ptr<YoutubeMediaProxy> youtube_audio_media_proxy; AsyncTask<void> youtube_downloader_task; - int youtube_video_content_length = 0; - int youtube_audio_content_length = 0; + int64_t youtube_video_content_length = 0; + int64_t youtube_audio_content_length = 0; std::string channel_url; AsyncTask<void> video_tasks; @@ -2585,11 +2591,12 @@ namespace QuickMedia { if(video_page->load(new_title, channel_url, media_chapters) != PluginResult::OK) return false; + std::string ext; if(!no_video) - video_url = video_page->get_video_url(largest_monitor_height, has_embedded_audio); + video_url = video_page->get_video_url(largest_monitor_height, has_embedded_audio, ext); if(video_url.empty() || no_video) { - video_url = video_page->get_audio_url(); + video_url = video_page->get_audio_url(ext); if(video_url.empty()) { video_url = video_page->get_url(); has_embedded_audio = true; @@ -2598,7 +2605,7 @@ namespace QuickMedia { has_embedded_audio = false; } } else if(!has_embedded_audio) { - audio_url = video_page->get_audio_url(); + audio_url = video_page->get_audio_url(ext); } if(!is_youtube && download_if_streaming_fails) { @@ -2677,7 +2684,7 @@ namespace QuickMedia { struct MediaProxyMetadata { std::unique_ptr<YoutubeMediaProxy> *media_proxy; std::string *url; - int content_length; + int64_t content_length; }; MediaProxyMetadata media_proxies[2] = { @@ -2876,7 +2883,7 @@ namespace QuickMedia { } else if(pressed_keysym == XK_f && pressing_ctrl) { window_set_fullscreen(disp, window.getSystemHandle(), WindowFullscreenState::TOGGLE); } else if(pressed_keysym == XK_s && pressing_ctrl) { - video_page_download_video(video_page->get_url(), !is_matrix || is_youtube, video_player_window); + video_page_download_video(video_page->get_url(), video_player_window); } else if(pressed_keysym == XK_F5) { in_seeking = false; double resume_start_time = 0.0; @@ -3954,7 +3961,7 @@ namespace QuickMedia { } else if(event.key.code == sf::Keyboard::S && event.key.control) { BodyItem *selected_item = thread_body->get_selected(); if(selected_item && !selected_item->url.empty()) - download_async_gui(selected_item->url, file_manager_start_dir.string(), false, false); + download_async_gui(selected_item->url, file_manager_start_dir.string(), false); } BodyItem *selected_item = thread_body->get_selected(); @@ -4071,7 +4078,7 @@ namespace QuickMedia { redraw = true; frame_skip_text_entry = true; } else if(event.key.code == sf::Keyboard::S && event.key.control) { - download_async_gui(attached_image_url, file_manager_start_dir.string(), false, false); + download_async_gui(attached_image_url, file_manager_start_dir.string(), false); } } } @@ -5437,7 +5444,7 @@ namespace QuickMedia { avatar_applied = false; return true; } else if(message_type == MessageType::FILE) { - download_async_gui(selected->url, file_manager_start_dir.string(), false, no_video); + download_async_gui(selected->url, file_manager_start_dir.string(), no_video); return true; } @@ -5478,7 +5485,7 @@ namespace QuickMedia { if(selected_item_message) { MessageType message_type = selected_item_message->type; if(!selected->url.empty() && message_type >= MessageType::IMAGE && message_type <= MessageType::FILE) { - download_async_gui(selected->url, file_manager_start_dir.string(), false, no_video); + download_async_gui(selected->url, file_manager_start_dir.string(), no_video); return true; } } @@ -6456,290 +6463,6 @@ namespace QuickMedia { matrix->stop_sync(); } - enum class DownloadUpdateStatus { - DOWNLOADING, - FINISHED, - ERROR - }; - - class Downloader { - public: - Downloader(const std::string &url, const std::string &output_filepath) : url(url), output_filepath(output_filepath) {} - virtual ~Downloader() = default; - - virtual bool start() = 0; - virtual void stop(bool download_completed) = 0; - virtual DownloadUpdateStatus update() = 0; - - virtual float get_progress() = 0; - virtual std::string get_progress_text() = 0; - virtual std::string get_download_speed_text() = 0; - protected: - std::string url; - std::string output_filepath; - }; - - class CurlDownloader : public Downloader { - public: - CurlDownloader(const std::string &url, const std::string &output_filepath) : Downloader(url, output_filepath) { - output_filepath_tmp = output_filepath; - output_filepath_tmp.append(".tmp"); - read_program.pid = -1; - read_program.read_fd = -1; - progress_text = "0 bytes/Unknown"; - download_speed_text = "Unknown/s"; - } - - bool start() override { - remove(output_filepath_tmp.data.c_str()); - - const char *args[] = { "curl", - "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", - "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", - "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(), - "-D", "/dev/stdout", - "--", url.c_str(), nullptr }; - - if(exec_program_pipe(args, &read_program) != 0) - return false; - - // TODO: Remove this async task and make the fd non blocking instead - header_reader = AsyncTask<bool>([this]{ - char tmp_buf[1024]; - while(true) { - ssize_t bytes_available = read(read_program.read_fd, tmp_buf, sizeof(tmp_buf)); - if(bytes_available == -1) { - return false; - } else if(bytes_available > 0 && content_length == (size_t)-1) { - header.append(tmp_buf, bytes_available); - if(header.find("\r\n\r\n") != std::string::npos) { - std::string content_length_str = header_extract_value(header, "content-length"); - if(!content_length_str.empty()) { - errno = 0; - char *endptr; - const long content_length_tmp = strtol(content_length_str.c_str(), &endptr, 10); - if(endptr != content_length_str.c_str() && errno == 0) { - std::lock_guard<std::mutex> lock(content_length_mutex); - content_length = content_length_tmp; - } - } - } - } - } - return true; - }); - - return true; - } - - void stop(bool download_completed) override { - if(read_program.read_fd != -1) - close(read_program.read_fd); - if(read_program.pid != -1) - kill(read_program.pid, SIGTERM); - if(!download_completed) - remove(output_filepath_tmp.data.c_str()); - //header_reader.cancel(); - } - - DownloadUpdateStatus update() override { - int status = 0; - if(wait_program_non_blocking(read_program.pid, &status)) { - read_program.pid = -1; - if(status == 0 && rename_atomic(output_filepath_tmp.data.c_str(), output_filepath.c_str()) == 0) { - return DownloadUpdateStatus::FINISHED; - } else { - return DownloadUpdateStatus::ERROR; - } - } - - if(header_reader.ready()) { - if(!header_reader.get()) - return DownloadUpdateStatus::ERROR; - } - - std::lock_guard<std::mutex> lock(content_length_mutex); - size_t output_file_size = 0; - file_get_size(output_filepath_tmp, &output_file_size); - size_t downloaded_size = std::min(output_file_size, content_length); - - if(content_length == (size_t)-1) { - progress_text = std::to_string(output_file_size / 1024) + "/Unknown"; - } else { - size_t percentage = 0; - if(output_file_size > 0) - percentage = (double)downloaded_size / (double)content_length * 100.0; - progress = (double)percentage / 100.0; - progress_text = file_size_to_human_readable_string(downloaded_size) + "/" + file_size_to_human_readable_string(content_length) + " (" + std::to_string(percentage) + "%)"; - } - - // TODO: Take into consideration time overflow? - size_t downloaded_diff = std::max(0lu, downloaded_size - downloaded_since_last_check); - download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s"; - downloaded_since_last_check = downloaded_size; - - return DownloadUpdateStatus::DOWNLOADING; - } - - float get_progress() override { - return progress; - } - - std::string get_progress_text() override { - return progress_text; - } - - std::string get_download_speed_text() override { - return download_speed_text; - } - private: - Path output_filepath_tmp; - ReadProgram read_program; - AsyncTask<bool> header_reader; - std::string header; - size_t content_length = (size_t)-1; - size_t downloaded_since_last_check = 0; - float progress = 0.0f; - std::mutex content_length_mutex; - std::string progress_text; - std::string download_speed_text; - }; - - class YoutubeDlDownloader : public Downloader { - public: - YoutubeDlDownloader(const std::string &url, const std::string &output_filepath, bool no_video) : Downloader(url, output_filepath), no_video(no_video) { - // youtube-dl requires a file extension for the file - if(this->output_filepath.find('.') == std::string::npos) - this->output_filepath += ".mkv"; - - read_program.pid = -1; - read_program.read_fd = -1; - progress_text = "0.0% of Unknown"; - download_speed_text = "Unknown/s"; - } - - bool start() override { - remove(output_filepath.c_str()); - - std::vector<const char*> args = { "youtube-dl", "--no-warnings", "--no-continue", "--output", output_filepath.c_str(), "--newline" }; - if(no_video) { - args.push_back("-f"); - args.push_back("bestaudio/best"); - args.push_back("-x"); - } else { - args.push_back("-f"); - args.push_back("bestvideo+bestaudio/best"); - } - args.insert(args.end(), { "--", url.c_str(), nullptr }); - - if(exec_program_pipe(args.data(), &read_program) != 0) - return false; - - read_program_file = fdopen(read_program.read_fd, "rb"); - if(!read_program_file) { - wait_program(read_program.pid); - return false; - } - - // TODO: Remove this async task and make the fd non blocking instead - youtube_dl_output_reader = AsyncTask<bool>([this]{ - char line[128]; - char progress_c[10]; - char content_size_c[20]; - char download_speed_c[20]; - - while(true) { - if(fgets(line, sizeof(line), read_program_file)) { - int len = strlen(line); - if(len > 0 && line[len - 1] == '\n') { - line[len - 1] = '\0'; - --len; - } - - if(sscanf(line, "[download] %10s of %20s at %20s", progress_c, content_size_c, download_speed_c) == 3) { - std::lock_guard<std::mutex> lock(progress_update_mutex); - - if(strcmp(progress_c, "Unknown") != 0 && strcmp(content_size_c, "Unknown") != 0) { - std::string progress_str = progress_c; - progress_text = progress_str + " of " + content_size_c; - if(progress_str.back() == '%') { - errno = 0; - char *endptr; - const double progress_tmp = strtod(progress_str.c_str(), &endptr); - if(endptr != progress_str.c_str() && errno == 0) - progress = progress_tmp / 100.0; - } - } - - if(strcmp(download_speed_c, "Unknown") == 0) - download_speed_text = "Unknown/s"; - else - download_speed_text = download_speed_c; - } - } else { - return false; - } - } - - return true; - }); - - return true; - } - - void stop(bool) override { - if(read_program_file) - fclose(read_program_file); - if(read_program.pid != -1) - kill(read_program.pid, SIGTERM); - // TODO: Remove the temporary files created by youtube-dl (if !download_completed) - //header_reader.cancel(); - } - - DownloadUpdateStatus update() override { - int status = 0; - if(wait_program_non_blocking(read_program.pid, &status)) { - read_program.pid = -1; - if(status == 0) { - return DownloadUpdateStatus::FINISHED; - } else { - return DownloadUpdateStatus::ERROR; - } - } - - if(youtube_dl_output_reader.ready()) { - if(!youtube_dl_output_reader.get()) - return DownloadUpdateStatus::ERROR; - } - - return DownloadUpdateStatus::DOWNLOADING; - } - - float get_progress() override { - std::lock_guard<std::mutex> lock(progress_update_mutex); - return progress; - } - - std::string get_progress_text() override { - std::lock_guard<std::mutex> lock(progress_update_mutex); - return progress_text; - } - - std::string get_download_speed_text() override { - std::lock_guard<std::mutex> lock(progress_update_mutex); - return download_speed_text; - } - private: - ReadProgram read_program; - FILE *read_program_file = nullptr; - AsyncTask<bool> youtube_dl_output_reader; - std::mutex progress_update_mutex; - float progress = 0.0f; - std::string progress_text; - std::string download_speed_text; - bool no_video; - }; - static int accumulate_string(char *data, int size, void *userdata) { std::string *str = (std::string*)userdata; if(str->size() + size > 1024 * 1024 * 100) // 100mb sane limit, TODO: make configurable @@ -6748,12 +6471,27 @@ namespace QuickMedia { return 0; } - void Program::download_page(const char *url, bool download_use_youtube_dl) { + void Program::download_page(const std::string &url) { window.setTitle("QuickMedia - Select where you want to save " + std::string(url)); + const bool download_use_youtube_dl = url_should_download_with_youtube_dl(url); std::string filename; + std::string video_id; + const bool url_is_youtube = youtube_url_extract_id(url, video_id); + std::unique_ptr<YoutubeVideoPage> youtube_video_page; + + std::string video_url; + std::string audio_url; + int64_t video_content_length = 0; + int64_t audio_content_length = 0; + TaskResult task_result; if(download_use_youtube_dl) { + if(!is_program_executable_by_name("youtube-dl")) { + show_notification("QuickMedia", "youtube-dl needs to be installed to download the video/music", Urgency::CRITICAL); + abort(); + } + task_result = run_task_with_loading_screen([this, url, &filename]{ std::string json_str; std::vector<const char*> args = { "youtube-dl", "--skip-download", "--print-json", "--no-warnings" }; @@ -6765,7 +6503,7 @@ namespace QuickMedia { args.push_back("-f"); args.push_back("bestvideo+bestaudio/best"); } - args.insert(args.end(), { "--", url, nullptr }); + args.insert(args.end(), { "--", url.c_str(), nullptr }); if(exec_program(args.data(), accumulate_string, &json_str) != 0) return false; @@ -6791,6 +6529,77 @@ namespace QuickMedia { return !filename.empty(); }); + } else if(url_is_youtube) { + youtube_video_page = std::make_unique<YoutubeVideoPage>(this, url); + bool cancelled = false; + bool load_successful = false; + const int largest_monitor_height = get_largest_monitor_height(disp); + + for(int i = 0; i < 3; ++i) { + task_result = run_task_with_loading_screen([this, &youtube_video_page, &filename, &video_url, &audio_url, &video_content_length, &audio_content_length, largest_monitor_height, &cancelled]{ + std::string channel_url; + std::vector<MediaChapter> chapters; + filename.clear(); + if(youtube_video_page->load(filename, channel_url, chapters) != PluginResult::OK) + return false; + + std::string ext; + bool has_embedded_audio = true; + video_url = no_video ? "" : youtube_video_page->get_video_url(largest_monitor_height, has_embedded_audio, ext); + audio_url.clear(); + + if(!has_embedded_audio || no_video) + audio_url = youtube_video_page->get_audio_url(ext); + + if(video_url.empty() && audio_url.empty()) + return false; + + if(!youtube_url_is_live_stream(video_url) && !youtube_url_is_live_stream(audio_url)) { + video_content_length = 0; + audio_content_length = 0; + std::string new_video_url = video_url; + std::string new_audio_url = audio_url; + auto current_thread_id = std::this_thread::get_id(); + if(!youtube_custom_redirect(new_video_url, new_audio_url, video_content_length, audio_content_length, [current_thread_id]{ return !program_is_dead_in_thread(current_thread_id); })) { + if(program_is_dead_in_current_thread()) + cancelled = true; + return false; + } + + video_url = std::move(new_video_url); + audio_url = std::move(new_audio_url); + } + + if(!video_url.empty() && !audio_url.empty()) + filename += ".mkv"; + else + filename += ext; + + return true; + }); + + if(task_result == TaskResult::CANCEL || cancelled) { + exit_code = 1; + return; + } else if(task_result == TaskResult::FALSE) { + continue; + } + + load_successful = true; + break; + } + + if(!load_successful) { + show_notification("QuickMedia", "Download failed", Urgency::CRITICAL); + exit_code = 1; + return; + } + + if(youtube_url_is_live_stream(video_url) || youtube_url_is_live_stream(audio_url)) { + show_notification("QuickMedia", "Downloading youtube live streams is currently not supported", Urgency::CRITICAL); + exit_code = 1; + return; + } } else { task_result = run_task_with_loading_screen([url, &filename]{ return url_get_remote_name(url, filename, true) == DownloadResult::OK; @@ -6800,6 +6609,10 @@ namespace QuickMedia { if(task_result == TaskResult::CANCEL) { exit_code = 1; return; + } else if(task_result == TaskResult::FALSE) { + show_notification("QuickMedia", "Download failed", Urgency::CRITICAL); + exit_code = 1; + return; } std::string output_filepath = file_save_page(filename); @@ -6862,10 +6675,21 @@ namespace QuickMedia { sf::Event event; std::unique_ptr<Downloader> downloader; - if(download_use_youtube_dl) + if(download_use_youtube_dl) { downloader = std::make_unique<YoutubeDlDownloader>(url, output_filepath, no_video); - else + } else if(url_is_youtube) { + MediaMetadata video_metadata; + video_metadata.url = std::move(video_url); + video_metadata.content_length = video_content_length; + + MediaMetadata audio_metadata; + audio_metadata.url = std::move(audio_url); + audio_metadata.content_length = audio_content_length; + + downloader = std::make_unique<YoutubeDownloader>(video_metadata, audio_metadata, output_filepath); + } else { downloader = std::make_unique<CurlDownloader>(url, output_filepath); + } if(!downloader->start()) { show_notification("QuickMedia", std::string("Failed to download ") + url + " to " + output_filepath, Urgency::CRITICAL); @@ -6908,6 +6732,7 @@ namespace QuickMedia { download_completed = true; goto cleanup; case DownloadUpdateStatus::ERROR: + fprintf(stderr, "Download error on update\n"); goto cleanup; } @@ -6962,8 +6787,8 @@ namespace QuickMedia { } cleanup: - downloader->stop(download_completed); - if(download_completed) { + const bool stop_successful = downloader->stop(download_completed); + if(download_completed && stop_successful) { show_notification("QuickMedia", std::string("Download finished! Downloaded ") + Path(filename).filename() + " to " + output_filepath); exit_code = 0; } else { diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index eb95c05..2fd193f 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -266,9 +266,8 @@ R"END( // Sometimes youtube returns a redirect url (not in the header but in the body...). // TODO: Find why this happens and if there is a way bypass it. - static std::string get_playback_url_recursive(std::string playback_url, int &content_length) { + static std::string get_playback_url_recursive(std::string playback_url, int64_t &content_length) { std::vector<CommandArg> additional_args = get_cookies(); - additional_args.push_back({ "-r", "0-4096" }); const int max_redirects = 5; for(int i = 0; i < max_redirects; ++i) { @@ -307,7 +306,7 @@ R"END( return playback_url; } - bool youtube_custom_redirect(std::string &video_url, std::string &audio_url, int &video_content_length, int &audio_content_length, std::function<bool()> active_handler) { + bool youtube_custom_redirect(std::string &video_url, std::string &audio_url, int64_t &video_content_length, int64_t &audio_content_length, std::function<bool()> active_handler) { // TODO: Do this without threads int num_total_tasks = 0; AsyncTask<std::string> tasks[2]; @@ -2164,10 +2163,6 @@ R"END( return url.substr(index, end - index); } - static void print_chosen_format(const YoutubeVideoFormat &format) { - fprintf(stderr, "Choosing youtube video format: width: %d, height: %d, fps: %d, bitrate: %d, mime type: %s\n", format.width, format.height, format.fps, format.base.bitrate, format.base.mime_type.c_str()); - } - static const YoutubeVideoFormat* get_highest_resolution_mp4_non_av1(const std::vector<YoutubeVideoFormat> &video_formats, int max_height) { for(const YoutubeVideoFormat &video_format : video_formats) { if(video_format.height <= max_height && video_format.base.mime_type.find("mp4") != std::string::npos && video_format.base.mime_type.find("av01") == std::string::npos) @@ -2184,14 +2179,14 @@ R"END( return nullptr; } - std::string YoutubeVideoPage::get_video_url(int max_height, bool &has_embedded_audio) { + std::string YoutubeVideoPage::get_video_url(int max_height, bool &has_embedded_audio, std::string &ext) { if(!livestream_url.empty()) { has_embedded_audio = true; return livestream_url; } if(video_formats.empty()) { - has_embedded_audio = true; + has_embedded_audio = false; return ""; } @@ -2208,17 +2203,31 @@ R"END( if(!chosen_video_format) chosen_video_format = &video_formats.back(); - print_chosen_format(*chosen_video_format); + fprintf(stderr, "Choosing youtube video format: width: %d, height: %d, fps: %d, bitrate: %d, mime type: %s\n", chosen_video_format->width, chosen_video_format->height, chosen_video_format->fps, chosen_video_format->base.bitrate, chosen_video_format->base.mime_type.c_str()); has_embedded_audio = chosen_video_format->has_embedded_audio; + + if(chosen_video_format->base.mime_type.find("mp4") != std::string::npos) + ext = ".mp4"; + else if(chosen_video_format->base.mime_type.find("webm") != std::string::npos) + ext = ".webm"; + return chosen_video_format->base.url; } - std::string YoutubeVideoPage::get_audio_url() { + std::string YoutubeVideoPage::get_audio_url(std::string &ext) { if(audio_formats.empty()) return ""; const YoutubeAudioFormat *chosen_audio_format = &audio_formats.front(); fprintf(stderr, "Choosing youtube audio format: bitrate: %d, mime type: %s\n", chosen_audio_format->base.bitrate, chosen_audio_format->base.mime_type.c_str()); + + if(chosen_audio_format->base.mime_type.find("mp4") != std::string::npos) + ext = ".m4a"; + else if(chosen_audio_format->base.mime_type.find("webm") != std::string::npos) + ext = ".webm"; + else if(chosen_audio_format->base.mime_type.find("opus") != std::string::npos) + ext = ".opus"; + return chosen_audio_format->base.url; } diff --git a/src/plugins/youtube/YoutubeMediaProxy.cpp b/src/plugins/youtube/YoutubeMediaProxy.cpp index cba4a1d..2296d1a 100644 --- a/src/plugins/youtube/YoutubeMediaProxy.cpp +++ b/src/plugins/youtube/YoutubeMediaProxy.cpp @@ -77,7 +77,7 @@ namespace QuickMedia { stop(); } - bool YoutubeStaticMediaProxy::start(const std::string &youtube_media_url, int content_length) { + bool YoutubeStaticMediaProxy::start(const std::string &youtube_media_url, int64_t content_length) { if(socket_fd != -1) return false; @@ -251,15 +251,15 @@ namespace QuickMedia { int new_start_range = header_extract_start_range(client_request_buffer); //fprintf(stderr, "got new range from client: %d\n", new_start_range); - if(new_start_range >= 0) { - if(downloader_read_program.pid != -1) { - kill(downloader_read_program.pid, SIGTERM); - wait_program(downloader_read_program.pid); - downloader_read_program.pid = -1; - } - clear_download_state(); - update_download_program_status(false, new_start_range, true); + //if(new_start_range >= 0) { + if(downloader_read_program.pid != -1) { + kill(downloader_read_program.pid, SIGTERM); + wait_program(downloader_read_program.pid); + downloader_read_program.pid = -1; } + clear_download_state(); + return update_download_program_status(false, new_start_range, true); + //} } else { if(client_request_buffer.size() > MAX_BUFFER_SIZE) { client_request_finished = true; @@ -278,13 +278,13 @@ namespace QuickMedia { download_header_written_offset = 0; } - YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::update_download_program_status(bool client_disconnected, int new_range_start, bool restart_download) { + YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::update_download_program_status(bool client_disconnected, int64_t new_range_start, bool restart_download) { int program_status = 0; if(downloader_read_program.pid != -1) { if(client_disconnected) { wait_program(downloader_read_program.pid); } else { - if(!wait_program_non_blocking(downloader_read_program.pid, &program_status)) + if(wait_program_non_blocking(downloader_read_program.pid, &program_status) == 0) return Error::OK; } downloader_read_program.pid = -1; @@ -353,7 +353,7 @@ namespace QuickMedia { return Error::OK; } - static void header_replace_content_length(std::string &header, size_t header_size, int new_content_length) { + static void header_replace_content_length(std::string &header, size_t header_size, int64_t new_content_length) { if(new_content_length < 0) new_content_length = 0; @@ -536,7 +536,7 @@ namespace QuickMedia { stop(); } - bool YoutubeLiveStreamMediaProxy::start(const std::string &youtube_media_url, int) { + bool YoutubeLiveStreamMediaProxy::start(const std::string &youtube_media_url, int64_t) { fd[0] = -1; fd[1] = -1; if(pipe(fd) == -1) { @@ -587,7 +587,7 @@ namespace QuickMedia { YoutubeMediaProxy::Error YoutubeLiveStreamMediaProxy::update_download_program_status() { int program_status = 0; - if(!wait_program_non_blocking(downloader_read_program.pid, &program_status)) + if(wait_program_non_blocking(downloader_read_program.pid, &program_status) == 0) return Error::OK; downloader_read_program.pid = -1; |