diff options
| -rw-r--r-- | TODO | 3 | ||||
| -rw-r--r-- | include/DownloadUtils.hpp | 2 | ||||
| -rw-r--r-- | include/VideoPlayer.hpp | 3 | ||||
| -rw-r--r-- | plugins/Youtube.hpp | 2 | ||||
| -rw-r--r-- | src/DownloadUtils.cpp | 100 | ||||
| -rw-r--r-- | src/QuickMedia.cpp | 67 | ||||
| -rw-r--r-- | src/VideoPlayer.cpp | 29 | ||||
| -rw-r--r-- | src/plugins/Youtube.cpp | 402 | ||||
| -rw-r--r-- | src/plugins/youtube/Signature.cpp | 2 |
9 files changed, 375 insertions, 235 deletions
@@ -168,4 +168,5 @@ Loading image background should be rounded. //Workaround mpv issue where video is frozen after seeking (with and without cache enabled, but more often with cache enabled). This happens because of audio. Reloading audio fixes this but audio will then be gone. Better deal with reading from file errors. This could happen when reading a file while its being modified. See read_file_as_json. Fix youtube comments. -Somehow fix youtube throttling speed limit to as low as 20-80kb which is fixed with a refresh. This should be detected automatically somehow.
\ No newline at end of file +Somehow fix youtube throttling speed limit to as low as 20-80kb which is fixed with a refresh. This should be detected automatically somehow. +Allow ctrl+r for video when the video is loading.
\ No newline at end of file diff --git a/include/DownloadUtils.hpp b/include/DownloadUtils.hpp index 28684ef..498b976 100644 --- a/include/DownloadUtils.hpp +++ b/include/DownloadUtils.hpp @@ -24,7 +24,7 @@ namespace QuickMedia { DownloadResult download_head_to_string(const std::string &url, std::string &result, bool use_browser_useragent = false, bool fail_on_error = true); // Returns the remote name from the content-disposition header or tries to extract the file name from url. Can return empty name DownloadResult url_get_remote_name(const std::string &url, std::string &result, bool use_browser_useragent); - DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_browser_useragent = false, bool fail_on_error = true); + DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_browser_useragent = false, bool fail_on_error = true, std::string *header = nullptr, int download_limit = 1024 * 1024 * 100); // 100mb download limit // Note: This function saves the content to the file atomically DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_browser_useragent = false, DownloadErrorHandler error_handler = nullptr, Path cache_path = ""); // Note: This function saves the content to the file atomically diff --git a/include/VideoPlayer.hpp b/include/VideoPlayer.hpp index 1a1e418..97e4b61 100644 --- a/include/VideoPlayer.hpp +++ b/include/VideoPlayer.hpp @@ -33,7 +33,7 @@ namespace QuickMedia { }; // @event_callback is called from another thread - VideoPlayer(bool no_video, bool use_system_mpv_config, bool keep_open, EventCallbackFunc event_callback, VideoPlayerWindowCreateCallback window_create_callback, const std::string &resource_root, int monitor_height); + VideoPlayer(bool no_video, bool use_system_mpv_config, bool keep_open, EventCallbackFunc event_callback, VideoPlayerWindowCreateCallback window_create_callback, const std::string &resource_root, int monitor_height, std::string plugin_name); ~VideoPlayer(); VideoPlayer(const VideoPlayer&) = delete; VideoPlayer& operator=(const VideoPlayer&) = delete; @@ -55,6 +55,7 @@ namespace QuickMedia { Error launch_video_process(const char *path, const char *audio_path, sf::WindowHandle parent_window, bool is_youtube, const std::string &title, const std::string &start_time); VideoPlayer::Error read_ipc_func(); private: + std::string plugin_name; bool no_video; bool use_system_mpv_config; bool keep_open; diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp index dcccc88..36094f0 100644 --- a/plugins/Youtube.hpp +++ b/plugins/Youtube.hpp @@ -24,6 +24,7 @@ namespace QuickMedia { }; bool youtube_url_extract_id(const std::string &youtube_url, std::string &youtube_video_id); + void youtube_custom_redirect(std::string &video_url, std::string &audio_url); class YoutubeSearchPage : public LazyFetchPage { public: @@ -146,6 +147,7 @@ namespace QuickMedia { PluginResult load(std::string &title, std::string &channel_url, std::vector<MediaChapter> &chapters) override; void mark_watched() override; private: + PluginResult get_video_info(const std::string &video_id, Json::Value &json_root); void parse_format(const Json::Value &format_json, bool is_adaptive); void parse_formats(const Json::Value &streaming_data_json); private: diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp index 0977b78..756da10 100644 --- a/src/DownloadUtils.cpp +++ b/src/DownloadUtils.cpp @@ -10,19 +10,85 @@ #include <rapidjson/document.h> #include <rapidjson/filereadstream.h> -static const bool debug_download = false; +namespace QuickMedia { + struct DownloadUserdata { + std::string *header = nullptr; + std::string *body = nullptr; + int download_limit = 1024 * 1024 * 100; // 100mb + bool header_finished = false; + int total_downloaded_size = 0; + }; + + static const bool debug_download = false; -static int accumulate_string(char *data, int size, void *userdata) { - std::string *str = (std::string*)userdata; - if(str->size() + size > 1024 * 1024 * 100) // 100mb sane limit, TODO: make configurable - return 1; - str->append(data, size); - return 0; -} + static int accumulate_string(char *data, int size, void *userdata) { + std::string *str = (std::string*)userdata; + if(str->size() + size > 1024 * 1024 * 100) // 100mb sane limit, TODO: make configurable + return 1; + str->append(data, size); + return 0; + } -static const char *useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; + // Returns 0 if content length is not found + static long get_content_length(const std::string &header) { + std::string content_length_str = header_extract_value(header, "content-length"); + if(content_length_str.empty()) + return 0; + + errno = 0; + char *endptr; + const long content_length_num = strtol(content_length_str.c_str(), &endptr, 10); + if(endptr != content_length_str.c_str() && errno == 0) + return content_length_num; + + return 0; + } + + static int accumulate_string_with_header(char *data, int size, void *userdata) { + DownloadUserdata *download_userdata = (DownloadUserdata*)userdata; + + if(download_userdata->header_finished || !download_userdata->header) { + download_userdata->body->append(data, size); + } else { + download_userdata->header->append(data, size); + bool end_of_header_found = false; + size_t end_of_headers_index = download_userdata->header->find("\r\n\r\n"); + if(end_of_headers_index != std::string::npos) { + while(true) { + const long content_length = get_content_length(download_userdata->header->substr(0, end_of_headers_index)); // TODO: Do not create a copy of the header string + end_of_headers_index += 4; + if(content_length == 0 && download_userdata->header->size() - end_of_headers_index > 0) { + download_userdata->header->erase(download_userdata->header->begin(), download_userdata->header->begin() + end_of_headers_index); + end_of_headers_index = download_userdata->header->find("\r\n\r\n"); + if(end_of_headers_index == std::string::npos) + break; + } else { + end_of_header_found = true; + break; + } + } + } + + if(end_of_header_found) { + download_userdata->body->append(download_userdata->header->begin() + end_of_headers_index, download_userdata->header->end()); + if(download_userdata->body->find("Content-Type") != std::string::npos) { + fprintf(stderr, "Found header in body!!!!, header: |%s|, body: |%s|\n", download_userdata->header->c_str(), download_userdata->body->c_str()); + abort(); + } + download_userdata->header->erase(download_userdata->header->begin() + end_of_headers_index, download_userdata->header->end()); + download_userdata->header_finished = true; + } + } + + download_userdata->total_downloaded_size += size; + if(download_userdata->total_downloaded_size >= download_userdata->download_limit) + return 1; + + return 0; + } + + static const char *useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; -namespace QuickMedia { DownloadResult download_head_to_string(const std::string &url, std::string &result, bool use_browser_useragent, bool fail_on_error) { result.clear(); sf::Clock timer; @@ -132,7 +198,7 @@ namespace QuickMedia { } // TODO: Add timeout - DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_browser_useragent, bool fail_on_error) { + DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_browser_useragent, bool fail_on_error, std::string *header, int download_limit) { result.clear(); sf::Clock timer; std::vector<const char*> args; @@ -148,9 +214,12 @@ namespace QuickMedia { args.push_back("-H"); args.push_back(useragent_str); } + if(header) + args.push_back("-i"); args.push_back("--"); args.push_back(url.c_str()); args.push_back(nullptr); + if(debug_download) { for(const char *arg : args) { if(arg) @@ -158,8 +227,15 @@ namespace QuickMedia { } fprintf(stderr, "\n"); } - if(exec_program(args.data(), accumulate_string, &result) != 0) + + DownloadUserdata download_userdata; + download_userdata.header = header; + download_userdata.body = &result; + download_userdata.download_limit = download_limit; + + if(exec_program(args.data(), accumulate_string_with_header, &download_userdata) != 0) return DownloadResult::NET_ERR; + fprintf(stderr, "Download duration for %s: %d ms\n", url.c_str(), timer.getElapsedTime().asMilliseconds()); return DownloadResult::OK; } diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index e155fb3..7305788 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -2308,10 +2308,10 @@ namespace QuickMedia { } TaskResult Program::run_task_with_loading_screen(std::function<bool()> callback) { - assert(std::this_thread::get_id() == main_thread_id); if(running_task_with_loading_screen) return callback() ? TaskResult::TRUE : TaskResult::FALSE; running_task_with_loading_screen = true; + assert(std::this_thread::get_id() == main_thread_id); idle_active_handler(); AsyncTask<bool> task = callback; @@ -2497,12 +2497,13 @@ namespace QuickMedia { }; std::string channel_url; - AsyncTask<BodyItems> video_tasks; + AsyncTask<void> video_tasks; std::function<void(const char*)> video_event_callback; bool go_to_previous_page = false; std::string video_url; std::string audio_url; + bool has_embedded_audio = true; bool in_seeking = false; sf::Clock seeking_start_timer; @@ -2513,7 +2514,7 @@ namespace QuickMedia { std::string prev_start_time; std::vector<MediaChapter> media_chapters; - auto load_video_error_check = [this, &prev_start_time, &media_chapters, &in_seeking, &video_url, &audio_url, &video_title, &video_tasks, &channel_url, previous_page, &go_to_previous_page, &video_loaded, video_page, &video_event_callback, &on_window_create, &video_player_window, is_youtube, is_matrix, download_if_streaming_fails](std::string start_time = "", bool reuse_media_source = false) mutable { + auto load_video_error_check = [this, &prev_start_time, &media_chapters, &in_seeking, &video_url, &audio_url, &has_embedded_audio, &video_title, &video_tasks, &channel_url, previous_page, &go_to_previous_page, &video_loaded, video_page, &video_event_callback, &on_window_create, &video_player_window, is_youtube, is_matrix, download_if_streaming_fails](std::string start_time = "", bool reuse_media_source = false) mutable { video_player.reset(); channel_url.clear(); video_loaded = false; @@ -2521,13 +2522,13 @@ namespace QuickMedia { video_player_window = None; bool is_audio_only = no_video; - bool has_embedded_audio = true; const int largest_monitor_height = get_largest_monitor_height(disp); if(!reuse_media_source) { std::string new_title; video_url.clear(); audio_url.clear(); + has_embedded_audio = true; TaskResult load_result = run_task_with_loading_screen([this, video_page, &new_title, &channel_url, &media_chapters, largest_monitor_height, &has_embedded_audio, &video_url, &audio_url, &is_audio_only, &previous_page, is_youtube, download_if_streaming_fails]() { if(video_page->load(new_title, channel_url, media_chapters) != PluginResult::OK) @@ -2579,7 +2580,7 @@ namespace QuickMedia { watched_videos.insert(video_page->get_url()); - video_player = std::make_unique<VideoPlayer>(is_audio_only, use_system_mpv_config, is_matrix && !is_youtube, video_event_callback, on_window_create, resources_root, largest_monitor_height); + video_player = std::make_unique<VideoPlayer>(is_audio_only, use_system_mpv_config, is_matrix && !is_youtube, video_event_callback, on_window_create, resources_root, largest_monitor_height, plugin_name); VideoPlayer::Error err = video_player->load_video(video_url.c_str(), audio_url.c_str(), window.getSystemHandle(), is_youtube, video_title, start_time, media_chapters); if(err != VideoPlayer::Error::OK) { std::string err_msg = "Failed to play url: "; @@ -2593,10 +2594,8 @@ namespace QuickMedia { if(!is_resume_go_back) { std::string url = video_page->get_url(); - video_tasks = AsyncTask<BodyItems>([video_page, url]() { - BodyItems related_videos = video_page->get_related_media(url); + video_tasks = AsyncTask<void>([video_page, url]() { video_page->mark_watched(); - return related_videos; }); } @@ -2744,23 +2743,15 @@ namespace QuickMedia { XUnmapWindow(disp, video_player_window); XSync(disp, False); - TaskResult task_result = run_task_with_loading_screen([&video_tasks, &related_videos]() { - while(true) { - if(program_is_dead_in_current_thread()) - return false; - - if(video_tasks.ready()) { - related_videos = video_tasks.get(); - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - } + TaskResult task_result = run_task_with_loading_screen([video_page, &related_videos]() { + related_videos = video_page->get_related_media(video_page->get_url()); + return true; }); XMapWindow(disp, video_player_window); XSync(disp, False); - if(task_result == TaskResult::CANCEL || task_result == TaskResult::FALSE) + if(task_result == TaskResult::CANCEL) cancelled = true; } @@ -2852,20 +2843,12 @@ namespace QuickMedia { std::string new_video_url; if(video_tasks.valid()) { - TaskResult task_result = run_task_with_loading_screen([&video_tasks, &related_videos]() { - while(true) { - if(program_is_dead_in_current_thread()) - return false; - - if(video_tasks.ready()) { - related_videos = video_tasks.get(); - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - } + TaskResult task_result = run_task_with_loading_screen([video_page, &related_videos]() { + related_videos = video_page->get_related_media(video_page->get_url()); + return true; }); - if(task_result == TaskResult::CANCEL || task_result == TaskResult::FALSE) { + if(task_result == TaskResult::CANCEL) { current_page = previous_page; go_to_previous_page = true; break; @@ -2943,14 +2926,22 @@ namespace QuickMedia { load_video_error_check(); } else if(update_err != VideoPlayer::Error::OK) { ++load_try; - if(load_try < num_load_tries_max) { + if(load_try == 1 && num_load_tries_max > 1 && is_youtube) { fprintf(stderr, "Failed to play the media, retrying (try %d out of %d)\n", 1 + load_try, num_load_tries_max); - load_video_error_check(prev_start_time); + std::string prev_video_url = video_url; + std::string prev_audio_url = audio_url; + youtube_custom_redirect(video_url, audio_url); + load_video_error_check(prev_start_time, video_url != prev_video_url || audio_url != prev_audio_url); } else { - show_notification("QuickMedia", "Failed to play the video (error code " + std::to_string((int)update_err) + ")", Urgency::CRITICAL); - current_page = previous_page; - go_to_previous_page = true; - break; + if(load_try < num_load_tries_max) { + fprintf(stderr, "Failed to play the media, retrying (try %d out of %d)\n", 1 + load_try, num_load_tries_max); + load_video_error_check(prev_start_time); + } else { + show_notification("QuickMedia", "Failed to play the video (error code " + std::to_string((int)update_err) + ")", Urgency::CRITICAL); + current_page = previous_page; + go_to_previous_page = true; + break; + } } } diff --git a/src/VideoPlayer.cpp b/src/VideoPlayer.cpp index caeaa15..2407955 100644 --- a/src/VideoPlayer.cpp +++ b/src/VideoPlayer.cpp @@ -64,8 +64,9 @@ namespace QuickMedia { } } - VideoPlayer::VideoPlayer(bool no_video, bool use_system_mpv_config, bool keep_open, EventCallbackFunc _event_callback, VideoPlayerWindowCreateCallback _window_create_callback, const std::string &resource_root, int monitor_height) : + VideoPlayer::VideoPlayer(bool no_video, bool use_system_mpv_config, bool keep_open, EventCallbackFunc _event_callback, VideoPlayerWindowCreateCallback _window_create_callback, const std::string &resource_root, int monitor_height, std::string plugin_name) : exit_status(0), + plugin_name(std::move(plugin_name)), no_video(no_video), use_system_mpv_config(use_system_mpv_config), keep_open(keep_open), @@ -134,12 +135,6 @@ namespace QuickMedia { std::string input_conf = "--input-conf=" + resource_root + "input.conf"; - std::string ytdl_format; - if(no_video) - ytdl_format = "--ytdl-format=bestaudio/best"; - else - ytdl_format = "--ytdl-format=bestvideo[height<=?" + std::to_string(monitor_height) + "]+bestaudio/best"; - // TODO: Resume playback if the last video played matches the first video played next time QuickMedia is launched args.insert(args.end(), { "mpv", @@ -148,7 +143,6 @@ namespace QuickMedia { "--no-terminal", "--save-position-on-quit=no", "--profile=pseudo-gui", // For gui when playing audio, requires a version of mpv that isn't ancient - ytdl_format.c_str(), "--no-resume-playback", // TODO: Disable hr seek on low power devices? "--hr-seek=yes", @@ -168,8 +162,27 @@ namespace QuickMedia { if(keep_open) args.push_back("--keep-open=yes"); + Path cookies_filepath; + std::string cookies_arg; + if(get_cookies_filepath(cookies_filepath, is_youtube ? "youtube" : plugin_name) == 0) { + cookies_arg = "--cookies-file="; + cookies_arg += cookies_filepath.data; + args.push_back("--cookies"); + args.push_back(cookies_arg.c_str()); + } else { + fprintf(stderr, "Failed to create %s cookies filepath\n", is_youtube ? "youtube" : plugin_name.c_str()); + } + + std::string ytdl_format; + if(no_video) + ytdl_format = "--ytdl-format=bestaudio/best"; + else + ytdl_format = "--ytdl-format=bestvideo[height<=?" + std::to_string(monitor_height) + "]+bestaudio/best"; + if(is_youtube) args.push_back("--no-ytdl"); + else + args.push_back(ytdl_format.c_str()); if(!use_system_mpv_config) { args.insert(args.end(), { diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index 79eb3d5..4df1358 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -101,6 +101,170 @@ R"END( return false; } + static std::mutex cookies_mutex; + static std::string cookies_filepath; + static std::string api_key; + + static bool is_whitespace(char c) { + return (c >= 8 && c <= 13) || c == ' '; + } + + // TODO: Cache this and redownload it when a network request fails with this api key? Do that in the same place as the signature, which means it would be done asynchronously + static std::string youtube_page_find_api_key() { + size_t api_key_index; + size_t api_key_index_end; + size_t api_key_length; + std::string website_result; + std::string::iterator api_key_start; + + if(download_to_string("https://www.youtube.com/?gl=US&hl=en", website_result, {}, true) != DownloadResult::OK) + goto fallback; + + api_key_index = website_result.find("INNERTUBE_API_KEY"); + if(api_key_index == std::string::npos) + goto fallback; + + api_key_index += 17; + api_key_start = std::find_if(website_result.begin() + api_key_index, website_result.end(), [](char c) { + return c != '"' && c != ':' && !is_whitespace(c); + }); + + if(api_key_start == website_result.end()) + goto fallback; + + api_key_index = api_key_start - website_result.begin(); + api_key_index_end = website_result.find('"', api_key_index); + if(api_key_index_end == std::string::npos) + goto fallback; + + api_key_length = api_key_index_end - api_key_index; + if(api_key_length > 512) // sanity check + goto fallback; + + return website_result.substr(api_key_index, api_key_length); + + fallback: + fprintf(stderr, "Failed to fetch youtube api key, fallback to %s\n", api_key.c_str()); + return "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; + } + + static std::string cpn; + + static bool generate_random_characters(char *buffer, int buffer_size, const char *alphabet, size_t alphabet_size) { + int fd = open("/dev/urandom", O_RDONLY); + if(fd == -1) { + perror("/dev/urandom"); + return false; + } + + if(read(fd, buffer, buffer_size) < buffer_size) { + fprintf(stderr, "Failed to read %d bytes from /dev/urandom\n", buffer_size); + close(fd); + return false; + } + + for(int i = 0; i < buffer_size; ++i) { + unsigned char c = *(unsigned char*)&buffer[i]; + buffer[i] = alphabet[c % alphabet_size]; + } + close(fd); + return true; + } + + static std::vector<CommandArg> get_cookies() { + std::lock_guard<std::mutex> lock(cookies_mutex); + if(cookies_filepath.empty()) { + YoutubeSignatureDecryptor::get_instance(); + + cpn.resize(16); + generate_random_characters(cpn.data(), cpn.size(), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_", 64); + + Path cookies_filepath_p; + if(get_cookies_filepath(cookies_filepath_p, "youtube") != 0) { + show_notification("QuickMedia", "Failed to create youtube cookies file", Urgency::CRITICAL); + return {}; + } + + // TODO: Re-enable this if the api key ever changes in the future. + // Maybe also put signature decryption in the same request? since it requests the same page. + #if 0 + //api_key = youtube_page_find_api_key(); + #else + api_key = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; + #endif + + if(get_file_type(cookies_filepath_p) == FileType::REGULAR) { + cookies_filepath = cookies_filepath_p.data; + } else { + Path cookies_filepath_tmp = cookies_filepath_p; + cookies_filepath_tmp.append(".tmp"); + + // TODO: This response also contains INNERTUBE_API_KEY which is the api key above. Maybe that should be parsed? + // TODO: Is there any way to bypass this? this is needed to set VISITOR_INFO1_LIVE which is required to read comments + const char *args[] = { "curl", "-I", "-s", "-f", "-L", "-b", cookies_filepath_tmp.data.c_str(), "-c", cookies_filepath_tmp.data.c_str(), "https://www.youtube.com/embed/watch?v=jNQXAC9IVRw&gl=US&hl=en", nullptr }; + if(exec_program(args, nullptr, nullptr) == 0) { + rename_atomic(cookies_filepath_tmp.data.c_str(), cookies_filepath_p.data.c_str()); + cookies_filepath = cookies_filepath_p.data; + } else { + show_notification("QuickMedia", "Failed to fetch cookies to view youtube comments", Urgency::CRITICAL); + return {}; + } + } + } + + return { + CommandArg{ "-b", cookies_filepath }, + CommandArg{ "-c", cookies_filepath } + }; + } + + // Sometimes youtube returns a redirect url (not in the header but in the body...). + // TODO: Find why this happens and if there is a way bypass it. + static std::string get_playback_url_recursive(std::string playback_url) { + std::vector<CommandArg> additional_args = get_cookies(); + additional_args.push_back({ "-r", "0-4096" }); + + const int max_redirects = 5; + for(int i = 0; i < max_redirects; ++i) { + std::string response_body; + std::string response_headers; + download_to_string(playback_url, response_body, additional_args, true, true, &response_headers, 4096); + + std::string content_type = header_extract_value(response_headers, "content-type"); + if(content_type.empty()) { + fprintf(stderr, "Failed to find content-type in youtube video header. Trying to play the video anyways\n"); + return playback_url; + } + + if(string_starts_with(content_type, "video") || string_starts_with(content_type, "audio")) + return playback_url; + + if(response_body.empty()) { + fprintf(stderr, "Failed to redirect youtube video. Trying to play the video anyways\n"); + return playback_url; + } + + playback_url = std::move(response_body); + } + + return playback_url; + } + + void youtube_custom_redirect(std::string &video_url, std::string &audio_url) { + // TODO: Do this without threads + AsyncTask<std::string> tasks[2]; + if(!video_url.empty()) + tasks[0] = AsyncTask<std::string>([video_url]() { return get_playback_url_recursive(std::move(video_url)); }); + if(!audio_url.empty()) + tasks[1] = AsyncTask<std::string>([audio_url]() { return get_playback_url_recursive(std::move(audio_url)); }); + + std::string *strings[2] = { &video_url, &audio_url }; + for(int i = 0; i < 2; ++i) { + if(tasks[i].valid()) + *strings[i] = tasks[i].get(); + } + } + // This is a common setup of text in the youtube json static std::optional<std::string> yt_json_get_text(const Json::Value &json, const char *root_name) { if(!json.isObject()) @@ -427,131 +591,6 @@ R"END( } } - static std::mutex cookies_mutex; - static std::string cookies_filepath; - static std::string api_key; - - static bool is_whitespace(char c) { - return (c >= 8 && c <= 13) || c == ' '; - } - - // TODO: Cache this and redownload it when a network request fails with this api key? Do that in the same place as the signature, which means it would be done asynchronously - static std::string youtube_page_find_api_key() { - size_t api_key_index; - size_t api_key_index_end; - size_t api_key_length; - std::string website_result; - std::string::iterator api_key_start; - - if(download_to_string("https://www.youtube.com/?gl=US&hl=en", website_result, {}, true) != DownloadResult::OK) - goto fallback; - - api_key_index = website_result.find("INNERTUBE_API_KEY"); - if(api_key_index == std::string::npos) - goto fallback; - - api_key_index += 17; - api_key_start = std::find_if(website_result.begin() + api_key_index, website_result.end(), [](char c) { - return c != '"' && c != ':' && !is_whitespace(c); - }); - - if(api_key_start == website_result.end()) - goto fallback; - - api_key_index = api_key_start - website_result.begin(); - api_key_index_end = website_result.find('"', api_key_index); - if(api_key_index_end == std::string::npos) - goto fallback; - - api_key_length = api_key_index_end - api_key_index; - if(api_key_length > 512) // sanity check - goto fallback; - - return website_result.substr(api_key_index, api_key_length); - - fallback: - fprintf(stderr, "Failed to fetch youtube api key, fallback to %s\n", api_key.c_str()); - return "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; - } - - static std::string cpn; - - static bool generate_random_characters(char *buffer, int buffer_size, const char *alphabet, size_t alphabet_size) { - int fd = open("/dev/urandom", O_RDONLY); - if(fd == -1) { - perror("/dev/urandom"); - return false; - } - - if(read(fd, buffer, buffer_size) < buffer_size) { - fprintf(stderr, "Failed to read %d bytes from /dev/urandom\n", buffer_size); - close(fd); - return false; - } - - for(int i = 0; i < buffer_size; ++i) { - unsigned char c = *(unsigned char*)&buffer[i]; - buffer[i] = alphabet[c % alphabet_size]; - } - close(fd); - return true; - } - - static std::vector<CommandArg> get_cookies() { - std::lock_guard<std::mutex> lock(cookies_mutex); - if(cookies_filepath.empty()) { - YoutubeSignatureDecryptor::get_instance(); - - cpn.resize(16); - generate_random_characters(cpn.data(), cpn.size(), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_", 64); - - Path cookies_filepath_p; - if(get_cookies_filepath(cookies_filepath_p, "youtube") != 0) { - show_notification("QuickMedia", "Failed to create youtube cookies file", Urgency::CRITICAL); - return {}; - } - - // TODO: Re-enable this if the api key ever changes in the future. - // Maybe also put signature decryption in the same request? since it requests the same page. - #if 0 - //api_key = youtube_page_find_api_key(); - #else - api_key = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; - #endif - - if(get_file_type(cookies_filepath_p) == FileType::REGULAR) { - cookies_filepath = cookies_filepath_p.data; - } else { - Path cookies_filepath_tmp = cookies_filepath_p; - cookies_filepath_tmp.append(".tmp"); - - // TODO: This response also contains INNERTUBE_API_KEY which is the api key above. Maybe that should be parsed? - // TODO: Is there any way to bypass this? this is needed to set VISITOR_INFO1_LIVE which is required to read comments - const char *args[] = { "curl", "-I", "-s", "-f", "-L", "-b", cookies_filepath_tmp.data.c_str(), "-c", cookies_filepath_tmp.data.c_str(), "https://www.youtube.com/embed/watch?v=jNQXAC9IVRw&gl=US&hl=en", nullptr }; - if(exec_program(args, nullptr, nullptr) == 0) { - rename_atomic(cookies_filepath_tmp.data.c_str(), cookies_filepath_p.data.c_str()); - cookies_filepath = cookies_filepath_p.data; - } else { - show_notification("QuickMedia", "Failed to fetch cookies to view youtube comments", Urgency::CRITICAL); - return {}; - } - } - } - - return { - CommandArg{ "-b", cookies_filepath }, - CommandArg{ "-c", cookies_filepath } - }; - } - - static std::string remove_index_from_playlist_url(const std::string &url) { - std::string result = url; - size_t index = result.rfind("&index="); - if(index == std::string::npos) - return result; - return result.substr(0, index); - } - static std::shared_ptr<BodyItem> parse_compact_video_renderer_json(const Json::Value &item_json, std::unordered_set<std::string> &added_videos) { const Json::Value &compact_video_renderer_json = item_json["compactVideoRenderer"]; if(!compact_video_renderer_json.isObject()) @@ -1982,34 +2021,6 @@ R"END( return nullptr; } - // Sometimes youtube returns a redirect url (not in the header but in the body...). - // TODO: Find why this happens and if there is a way bypass it. - // Or maybe move this logic to QuickMedia video_content_page when mpv fails to start up (mpv will exit with exit code 2 and the message "Failed to recognize file format." when this happens). - // But that might be too slow for pinephone. - static std::string get_playback_url_recursive(std::string playback_url) { - const int max_redirects = 5; - for(int i = 0; i < max_redirects; ++i) { - std::string response_headers; - if(download_head_to_string(playback_url, response_headers, true) != DownloadResult::OK) - return ""; - - std::string content_type = header_extract_value(response_headers, "content-type"); - if(content_type.empty()) - return ""; - - if(string_starts_with(content_type, "video") || string_starts_with(content_type, "audio")) - return playback_url; - - // TODO: Download head and body in one request - std::string new_url; - if(download_to_string(playback_url, new_url, {}, true) != DownloadResult::OK) - return ""; - - playback_url = std::move(new_url); - } - return playback_url; - } - std::string YoutubeVideoPage::get_video_url(int max_height, bool &has_embedded_audio) { if(!hls_manifest_url.empty()) { has_embedded_audio = true; @@ -2036,7 +2047,7 @@ R"END( print_chosen_format(*chosen_video_format); has_embedded_audio = chosen_video_format->has_embedded_audio; - return get_playback_url_recursive(chosen_video_format->base.url); + return chosen_video_format->base.url; } std::string YoutubeVideoPage::get_audio_url() { @@ -2046,7 +2057,7 @@ R"END( // TODO: The "worst" (but still good) quality audio is chosen right now because youtube seeking freezes for up to 15 seconds when choosing the best quality const YoutubeAudioFormat *chosen_audio_format = &audio_formats.back(); fprintf(stderr, "Choosing youtube audio format: bitrate: %d, mime type: %s\n", chosen_audio_format->base.bitrate, chosen_audio_format->base.mime_type.c_str()); - return get_playback_url_recursive(chosen_audio_format->base.url); + return chosen_audio_format->base.url; } // Returns -1 if timestamp is in an invalid format @@ -2089,17 +2100,7 @@ R"END( return result; } - PluginResult YoutubeVideoPage::load(std::string &title, std::string &channel_url, std::vector<MediaChapter> &chapters) { - hls_manifest_url.clear(); - video_formats.clear(); - audio_formats.clear(); - - std::string video_id; - if(!youtube_url_extract_id(url, video_id)) { - fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); - return PluginResult::ERR; - } - + PluginResult YoutubeVideoPage::get_video_info(const std::string &video_id, Json::Value &json_root) { std::vector<CommandArg> additional_args = get_cookies(); std::string response; @@ -2109,7 +2110,6 @@ R"END( std::string player_response_param = url_extract_param(response, "player_response"); player_response_param = url_param_decode(player_response_param); - Json::Value json_root; Json::CharReaderBuilder json_builder; std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader()); std::string json_errors; @@ -2118,27 +2118,83 @@ R"END( return PluginResult::ERR; } + return PluginResult::OK; + } + + PluginResult YoutubeVideoPage::load(std::string &title, std::string &channel_url, std::vector<MediaChapter> &chapters) { + hls_manifest_url.clear(); + video_formats.clear(); + audio_formats.clear(); + + std::string video_id; + if(!youtube_url_extract_id(url, video_id)) { + fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); + return PluginResult::ERR; + } + + #if 0 + std::string request_data = key_api_request_data; + string_replace_all(request_data, "%VIDEO_ID%", video_id); + + std::vector<CommandArg> additional_args = { + { "-H", "Content-Type: application/json" }, + { "-H", "x-youtube-client-name: 1" }, + { "-H", youtube_client_version }, + { "--data-raw", std::move(request_data) } + }; + + std::vector<CommandArg> cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + + Json::Value json_root; + DownloadResult download_result = download_json(json_root, "https://www.youtube.com/youtubei/v1/player?key=" + api_key + "&gl=US&hl=en", additional_args, true); + if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result); + if(!json_root.isObject()) return PluginResult::ERR; - const Json::Value &streaming_data_json = json_root["streamingData"]; - if(!streaming_data_json.isObject()) { + const Json::Value *streaming_data_json = &json_root["streamingData"]; + if(!streaming_data_json->isObject()) { const Json::Value &playability_status_json = json_root["playabilityStatus"]; if(playability_status_json.isObject()) { const Json::Value &status_json = playability_status_json["status"]; const Json::Value &reason_json = playability_status_json["reason"]; - if(status_json.isString()) - fprintf(stderr, "Youtube video loading failed, reason: (status: %s, reason: %s)\n", status_json.asCString(), reason_json.isString() ? reason_json.asCString() : "unknown"); + fprintf(stderr, "Warning: youtube video loading failed, reason: (status: %s, reason: %s), trying with get_video_info endpoint instead\n", status_json.isString() ? status_json.asCString() : "unknown", reason_json.isString() ? reason_json.asCString() : "unknown"); + + json_root = Json::Value(Json::nullValue); + PluginResult result = get_video_info(video_id, json_root); + if(result != PluginResult::OK) + return result; + + if(!json_root.isObject()) + return PluginResult::ERR; + + streaming_data_json = &json_root["streamingData"]; + if(!streaming_data_json->isObject()) + return PluginResult::ERR; } return PluginResult::ERR; } + #else + Json::Value json_root; + PluginResult result = get_video_info(video_id, json_root); + if(result != PluginResult::OK) + return result; + + if(!json_root.isObject()) + return PluginResult::ERR; + + const Json::Value *streaming_data_json = &json_root["streamingData"]; + if(!streaming_data_json->isObject()) + return PluginResult::ERR; + #endif // TODO: Verify if this always works (what about copyrighted live streams?), also what about choosing video quality for live stream? Maybe use mpv --hls-bitrate option? - const Json::Value &hls_manifest_url_json = streaming_data_json["hlsManifestUrl"]; + const Json::Value &hls_manifest_url_json = (*streaming_data_json)["hlsManifestUrl"]; if(hls_manifest_url_json.isString()) { hls_manifest_url = hls_manifest_url_json.asString(); } else { - parse_formats(streaming_data_json); + parse_formats(*streaming_data_json); if(video_formats.empty() && audio_formats.empty()) return PluginResult::ERR; } diff --git a/src/plugins/youtube/Signature.cpp b/src/plugins/youtube/Signature.cpp index 65d4e2e..7631182 100644 --- a/src/plugins/youtube/Signature.cpp +++ b/src/plugins/youtube/Signature.cpp @@ -260,7 +260,7 @@ namespace QuickMedia { int YoutubeSignatureDecryptor::update_decrypt_function() { std::string response; - DownloadResult download_result = download_to_string("https://www.youtube.com/?gl=US&hl=en", response, {}, true); + DownloadResult download_result = download_to_string("https://www.youtube.com/watch?v=jNQXAC9IVRw&gl=US&hl=en", response, {}, true); if(download_result != DownloadResult::OK) { fprintf(stderr, "YoutubeSignatureDecryptor::update_decrypt_function failed. Failed to get youtube page\n"); return U_DEC_FUN_NET_ERR; |