From e4792f46d545263d16db21bd0caf71345a69b63f Mon Sep 17 00:00:00 2001 From: dec05eba Date: Tue, 22 Jun 2021 16:07:20 +0200 Subject: Only do youtube redirect on failure to load video --- src/DownloadUtils.cpp | 100 ++++++++-- src/QuickMedia.cpp | 67 +++---- src/VideoPlayer.cpp | 29 ++- src/plugins/Youtube.cpp | 402 ++++++++++++++++++++++---------------- src/plugins/youtube/Signature.cpp | 2 +- 5 files changed, 368 insertions(+), 232 deletions(-) (limited to 'src') diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp index 0977b78..756da10 100644 --- a/src/DownloadUtils.cpp +++ b/src/DownloadUtils.cpp @@ -10,19 +10,85 @@ #include #include -static const bool debug_download = false; +namespace QuickMedia { + struct DownloadUserdata { + std::string *header = nullptr; + std::string *body = nullptr; + int download_limit = 1024 * 1024 * 100; // 100mb + bool header_finished = false; + int total_downloaded_size = 0; + }; + + static const bool debug_download = false; -static int accumulate_string(char *data, int size, void *userdata) { - std::string *str = (std::string*)userdata; - if(str->size() + size > 1024 * 1024 * 100) // 100mb sane limit, TODO: make configurable - return 1; - str->append(data, size); - return 0; -} + static int accumulate_string(char *data, int size, void *userdata) { + std::string *str = (std::string*)userdata; + if(str->size() + size > 1024 * 1024 * 100) // 100mb sane limit, TODO: make configurable + return 1; + str->append(data, size); + return 0; + } -static const char *useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; + // Returns 0 if content length is not found + static long get_content_length(const std::string &header) { + std::string content_length_str = header_extract_value(header, "content-length"); + if(content_length_str.empty()) + return 0; + + errno = 0; + char *endptr; + const long content_length_num = strtol(content_length_str.c_str(), &endptr, 10); + if(endptr != content_length_str.c_str() && errno == 0) + return content_length_num; + + return 0; + } + + static int accumulate_string_with_header(char *data, int size, void *userdata) { + DownloadUserdata *download_userdata = (DownloadUserdata*)userdata; + + if(download_userdata->header_finished || !download_userdata->header) { + download_userdata->body->append(data, size); + } else { + download_userdata->header->append(data, size); + bool end_of_header_found = false; + size_t end_of_headers_index = download_userdata->header->find("\r\n\r\n"); + if(end_of_headers_index != std::string::npos) { + while(true) { + const long content_length = get_content_length(download_userdata->header->substr(0, end_of_headers_index)); // TODO: Do not create a copy of the header string + end_of_headers_index += 4; + if(content_length == 0 && download_userdata->header->size() - end_of_headers_index > 0) { + download_userdata->header->erase(download_userdata->header->begin(), download_userdata->header->begin() + end_of_headers_index); + end_of_headers_index = download_userdata->header->find("\r\n\r\n"); + if(end_of_headers_index == std::string::npos) + break; + } else { + end_of_header_found = true; + break; + } + } + } + + if(end_of_header_found) { + download_userdata->body->append(download_userdata->header->begin() + end_of_headers_index, download_userdata->header->end()); + if(download_userdata->body->find("Content-Type") != std::string::npos) { + fprintf(stderr, "Found header in body!!!!, header: |%s|, body: |%s|\n", download_userdata->header->c_str(), download_userdata->body->c_str()); + abort(); + } + download_userdata->header->erase(download_userdata->header->begin() + end_of_headers_index, download_userdata->header->end()); + download_userdata->header_finished = true; + } + } + + download_userdata->total_downloaded_size += size; + if(download_userdata->total_downloaded_size >= download_userdata->download_limit) + return 1; + + return 0; + } + + static const char *useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; -namespace QuickMedia { DownloadResult download_head_to_string(const std::string &url, std::string &result, bool use_browser_useragent, bool fail_on_error) { result.clear(); sf::Clock timer; @@ -132,7 +198,7 @@ namespace QuickMedia { } // TODO: Add timeout - DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector &additional_args, bool use_browser_useragent, bool fail_on_error) { + DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector &additional_args, bool use_browser_useragent, bool fail_on_error, std::string *header, int download_limit) { result.clear(); sf::Clock timer; std::vector args; @@ -148,9 +214,12 @@ namespace QuickMedia { args.push_back("-H"); args.push_back(useragent_str); } + if(header) + args.push_back("-i"); args.push_back("--"); args.push_back(url.c_str()); args.push_back(nullptr); + if(debug_download) { for(const char *arg : args) { if(arg) @@ -158,8 +227,15 @@ namespace QuickMedia { } fprintf(stderr, "\n"); } - if(exec_program(args.data(), accumulate_string, &result) != 0) + + DownloadUserdata download_userdata; + download_userdata.header = header; + download_userdata.body = &result; + download_userdata.download_limit = download_limit; + + if(exec_program(args.data(), accumulate_string_with_header, &download_userdata) != 0) return DownloadResult::NET_ERR; + fprintf(stderr, "Download duration for %s: %d ms\n", url.c_str(), timer.getElapsedTime().asMilliseconds()); return DownloadResult::OK; } diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index e155fb3..7305788 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -2308,10 +2308,10 @@ namespace QuickMedia { } TaskResult Program::run_task_with_loading_screen(std::function callback) { - assert(std::this_thread::get_id() == main_thread_id); if(running_task_with_loading_screen) return callback() ? TaskResult::TRUE : TaskResult::FALSE; running_task_with_loading_screen = true; + assert(std::this_thread::get_id() == main_thread_id); idle_active_handler(); AsyncTask task = callback; @@ -2497,12 +2497,13 @@ namespace QuickMedia { }; std::string channel_url; - AsyncTask video_tasks; + AsyncTask video_tasks; std::function video_event_callback; bool go_to_previous_page = false; std::string video_url; std::string audio_url; + bool has_embedded_audio = true; bool in_seeking = false; sf::Clock seeking_start_timer; @@ -2513,7 +2514,7 @@ namespace QuickMedia { std::string prev_start_time; std::vector media_chapters; - auto load_video_error_check = [this, &prev_start_time, &media_chapters, &in_seeking, &video_url, &audio_url, &video_title, &video_tasks, &channel_url, previous_page, &go_to_previous_page, &video_loaded, video_page, &video_event_callback, &on_window_create, &video_player_window, is_youtube, is_matrix, download_if_streaming_fails](std::string start_time = "", bool reuse_media_source = false) mutable { + auto load_video_error_check = [this, &prev_start_time, &media_chapters, &in_seeking, &video_url, &audio_url, &has_embedded_audio, &video_title, &video_tasks, &channel_url, previous_page, &go_to_previous_page, &video_loaded, video_page, &video_event_callback, &on_window_create, &video_player_window, is_youtube, is_matrix, download_if_streaming_fails](std::string start_time = "", bool reuse_media_source = false) mutable { video_player.reset(); channel_url.clear(); video_loaded = false; @@ -2521,13 +2522,13 @@ namespace QuickMedia { video_player_window = None; bool is_audio_only = no_video; - bool has_embedded_audio = true; const int largest_monitor_height = get_largest_monitor_height(disp); if(!reuse_media_source) { std::string new_title; video_url.clear(); audio_url.clear(); + has_embedded_audio = true; TaskResult load_result = run_task_with_loading_screen([this, video_page, &new_title, &channel_url, &media_chapters, largest_monitor_height, &has_embedded_audio, &video_url, &audio_url, &is_audio_only, &previous_page, is_youtube, download_if_streaming_fails]() { if(video_page->load(new_title, channel_url, media_chapters) != PluginResult::OK) @@ -2579,7 +2580,7 @@ namespace QuickMedia { watched_videos.insert(video_page->get_url()); - video_player = std::make_unique(is_audio_only, use_system_mpv_config, is_matrix && !is_youtube, video_event_callback, on_window_create, resources_root, largest_monitor_height); + video_player = std::make_unique(is_audio_only, use_system_mpv_config, is_matrix && !is_youtube, video_event_callback, on_window_create, resources_root, largest_monitor_height, plugin_name); VideoPlayer::Error err = video_player->load_video(video_url.c_str(), audio_url.c_str(), window.getSystemHandle(), is_youtube, video_title, start_time, media_chapters); if(err != VideoPlayer::Error::OK) { std::string err_msg = "Failed to play url: "; @@ -2593,10 +2594,8 @@ namespace QuickMedia { if(!is_resume_go_back) { std::string url = video_page->get_url(); - video_tasks = AsyncTask([video_page, url]() { - BodyItems related_videos = video_page->get_related_media(url); + video_tasks = AsyncTask([video_page, url]() { video_page->mark_watched(); - return related_videos; }); } @@ -2744,23 +2743,15 @@ namespace QuickMedia { XUnmapWindow(disp, video_player_window); XSync(disp, False); - TaskResult task_result = run_task_with_loading_screen([&video_tasks, &related_videos]() { - while(true) { - if(program_is_dead_in_current_thread()) - return false; - - if(video_tasks.ready()) { - related_videos = video_tasks.get(); - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - } + TaskResult task_result = run_task_with_loading_screen([video_page, &related_videos]() { + related_videos = video_page->get_related_media(video_page->get_url()); + return true; }); XMapWindow(disp, video_player_window); XSync(disp, False); - if(task_result == TaskResult::CANCEL || task_result == TaskResult::FALSE) + if(task_result == TaskResult::CANCEL) cancelled = true; } @@ -2852,20 +2843,12 @@ namespace QuickMedia { std::string new_video_url; if(video_tasks.valid()) { - TaskResult task_result = run_task_with_loading_screen([&video_tasks, &related_videos]() { - while(true) { - if(program_is_dead_in_current_thread()) - return false; - - if(video_tasks.ready()) { - related_videos = video_tasks.get(); - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - } + TaskResult task_result = run_task_with_loading_screen([video_page, &related_videos]() { + related_videos = video_page->get_related_media(video_page->get_url()); + return true; }); - if(task_result == TaskResult::CANCEL || task_result == TaskResult::FALSE) { + if(task_result == TaskResult::CANCEL) { current_page = previous_page; go_to_previous_page = true; break; @@ -2943,14 +2926,22 @@ namespace QuickMedia { load_video_error_check(); } else if(update_err != VideoPlayer::Error::OK) { ++load_try; - if(load_try < num_load_tries_max) { + if(load_try == 1 && num_load_tries_max > 1 && is_youtube) { fprintf(stderr, "Failed to play the media, retrying (try %d out of %d)\n", 1 + load_try, num_load_tries_max); - load_video_error_check(prev_start_time); + std::string prev_video_url = video_url; + std::string prev_audio_url = audio_url; + youtube_custom_redirect(video_url, audio_url); + load_video_error_check(prev_start_time, video_url != prev_video_url || audio_url != prev_audio_url); } else { - show_notification("QuickMedia", "Failed to play the video (error code " + std::to_string((int)update_err) + ")", Urgency::CRITICAL); - current_page = previous_page; - go_to_previous_page = true; - break; + if(load_try < num_load_tries_max) { + fprintf(stderr, "Failed to play the media, retrying (try %d out of %d)\n", 1 + load_try, num_load_tries_max); + load_video_error_check(prev_start_time); + } else { + show_notification("QuickMedia", "Failed to play the video (error code " + std::to_string((int)update_err) + ")", Urgency::CRITICAL); + current_page = previous_page; + go_to_previous_page = true; + break; + } } } diff --git a/src/VideoPlayer.cpp b/src/VideoPlayer.cpp index caeaa15..2407955 100644 --- a/src/VideoPlayer.cpp +++ b/src/VideoPlayer.cpp @@ -64,8 +64,9 @@ namespace QuickMedia { } } - VideoPlayer::VideoPlayer(bool no_video, bool use_system_mpv_config, bool keep_open, EventCallbackFunc _event_callback, VideoPlayerWindowCreateCallback _window_create_callback, const std::string &resource_root, int monitor_height) : + VideoPlayer::VideoPlayer(bool no_video, bool use_system_mpv_config, bool keep_open, EventCallbackFunc _event_callback, VideoPlayerWindowCreateCallback _window_create_callback, const std::string &resource_root, int monitor_height, std::string plugin_name) : exit_status(0), + plugin_name(std::move(plugin_name)), no_video(no_video), use_system_mpv_config(use_system_mpv_config), keep_open(keep_open), @@ -134,12 +135,6 @@ namespace QuickMedia { std::string input_conf = "--input-conf=" + resource_root + "input.conf"; - std::string ytdl_format; - if(no_video) - ytdl_format = "--ytdl-format=bestaudio/best"; - else - ytdl_format = "--ytdl-format=bestvideo[height<=?" + std::to_string(monitor_height) + "]+bestaudio/best"; - // TODO: Resume playback if the last video played matches the first video played next time QuickMedia is launched args.insert(args.end(), { "mpv", @@ -148,7 +143,6 @@ namespace QuickMedia { "--no-terminal", "--save-position-on-quit=no", "--profile=pseudo-gui", // For gui when playing audio, requires a version of mpv that isn't ancient - ytdl_format.c_str(), "--no-resume-playback", // TODO: Disable hr seek on low power devices? "--hr-seek=yes", @@ -168,8 +162,27 @@ namespace QuickMedia { if(keep_open) args.push_back("--keep-open=yes"); + Path cookies_filepath; + std::string cookies_arg; + if(get_cookies_filepath(cookies_filepath, is_youtube ? "youtube" : plugin_name) == 0) { + cookies_arg = "--cookies-file="; + cookies_arg += cookies_filepath.data; + args.push_back("--cookies"); + args.push_back(cookies_arg.c_str()); + } else { + fprintf(stderr, "Failed to create %s cookies filepath\n", is_youtube ? "youtube" : plugin_name.c_str()); + } + + std::string ytdl_format; + if(no_video) + ytdl_format = "--ytdl-format=bestaudio/best"; + else + ytdl_format = "--ytdl-format=bestvideo[height<=?" + std::to_string(monitor_height) + "]+bestaudio/best"; + if(is_youtube) args.push_back("--no-ytdl"); + else + args.push_back(ytdl_format.c_str()); if(!use_system_mpv_config) { args.insert(args.end(), { diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index 79eb3d5..4df1358 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -101,6 +101,170 @@ R"END( return false; } + static std::mutex cookies_mutex; + static std::string cookies_filepath; + static std::string api_key; + + static bool is_whitespace(char c) { + return (c >= 8 && c <= 13) || c == ' '; + } + + // TODO: Cache this and redownload it when a network request fails with this api key? Do that in the same place as the signature, which means it would be done asynchronously + static std::string youtube_page_find_api_key() { + size_t api_key_index; + size_t api_key_index_end; + size_t api_key_length; + std::string website_result; + std::string::iterator api_key_start; + + if(download_to_string("https://www.youtube.com/?gl=US&hl=en", website_result, {}, true) != DownloadResult::OK) + goto fallback; + + api_key_index = website_result.find("INNERTUBE_API_KEY"); + if(api_key_index == std::string::npos) + goto fallback; + + api_key_index += 17; + api_key_start = std::find_if(website_result.begin() + api_key_index, website_result.end(), [](char c) { + return c != '"' && c != ':' && !is_whitespace(c); + }); + + if(api_key_start == website_result.end()) + goto fallback; + + api_key_index = api_key_start - website_result.begin(); + api_key_index_end = website_result.find('"', api_key_index); + if(api_key_index_end == std::string::npos) + goto fallback; + + api_key_length = api_key_index_end - api_key_index; + if(api_key_length > 512) // sanity check + goto fallback; + + return website_result.substr(api_key_index, api_key_length); + + fallback: + fprintf(stderr, "Failed to fetch youtube api key, fallback to %s\n", api_key.c_str()); + return "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; + } + + static std::string cpn; + + static bool generate_random_characters(char *buffer, int buffer_size, const char *alphabet, size_t alphabet_size) { + int fd = open("/dev/urandom", O_RDONLY); + if(fd == -1) { + perror("/dev/urandom"); + return false; + } + + if(read(fd, buffer, buffer_size) < buffer_size) { + fprintf(stderr, "Failed to read %d bytes from /dev/urandom\n", buffer_size); + close(fd); + return false; + } + + for(int i = 0; i < buffer_size; ++i) { + unsigned char c = *(unsigned char*)&buffer[i]; + buffer[i] = alphabet[c % alphabet_size]; + } + close(fd); + return true; + } + + static std::vector get_cookies() { + std::lock_guard lock(cookies_mutex); + if(cookies_filepath.empty()) { + YoutubeSignatureDecryptor::get_instance(); + + cpn.resize(16); + generate_random_characters(cpn.data(), cpn.size(), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_", 64); + + Path cookies_filepath_p; + if(get_cookies_filepath(cookies_filepath_p, "youtube") != 0) { + show_notification("QuickMedia", "Failed to create youtube cookies file", Urgency::CRITICAL); + return {}; + } + + // TODO: Re-enable this if the api key ever changes in the future. + // Maybe also put signature decryption in the same request? since it requests the same page. + #if 0 + //api_key = youtube_page_find_api_key(); + #else + api_key = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; + #endif + + if(get_file_type(cookies_filepath_p) == FileType::REGULAR) { + cookies_filepath = cookies_filepath_p.data; + } else { + Path cookies_filepath_tmp = cookies_filepath_p; + cookies_filepath_tmp.append(".tmp"); + + // TODO: This response also contains INNERTUBE_API_KEY which is the api key above. Maybe that should be parsed? + // TODO: Is there any way to bypass this? this is needed to set VISITOR_INFO1_LIVE which is required to read comments + const char *args[] = { "curl", "-I", "-s", "-f", "-L", "-b", cookies_filepath_tmp.data.c_str(), "-c", cookies_filepath_tmp.data.c_str(), "https://www.youtube.com/embed/watch?v=jNQXAC9IVRw&gl=US&hl=en", nullptr }; + if(exec_program(args, nullptr, nullptr) == 0) { + rename_atomic(cookies_filepath_tmp.data.c_str(), cookies_filepath_p.data.c_str()); + cookies_filepath = cookies_filepath_p.data; + } else { + show_notification("QuickMedia", "Failed to fetch cookies to view youtube comments", Urgency::CRITICAL); + return {}; + } + } + } + + return { + CommandArg{ "-b", cookies_filepath }, + CommandArg{ "-c", cookies_filepath } + }; + } + + // Sometimes youtube returns a redirect url (not in the header but in the body...). + // TODO: Find why this happens and if there is a way bypass it. + static std::string get_playback_url_recursive(std::string playback_url) { + std::vector additional_args = get_cookies(); + additional_args.push_back({ "-r", "0-4096" }); + + const int max_redirects = 5; + for(int i = 0; i < max_redirects; ++i) { + std::string response_body; + std::string response_headers; + download_to_string(playback_url, response_body, additional_args, true, true, &response_headers, 4096); + + std::string content_type = header_extract_value(response_headers, "content-type"); + if(content_type.empty()) { + fprintf(stderr, "Failed to find content-type in youtube video header. Trying to play the video anyways\n"); + return playback_url; + } + + if(string_starts_with(content_type, "video") || string_starts_with(content_type, "audio")) + return playback_url; + + if(response_body.empty()) { + fprintf(stderr, "Failed to redirect youtube video. Trying to play the video anyways\n"); + return playback_url; + } + + playback_url = std::move(response_body); + } + + return playback_url; + } + + void youtube_custom_redirect(std::string &video_url, std::string &audio_url) { + // TODO: Do this without threads + AsyncTask tasks[2]; + if(!video_url.empty()) + tasks[0] = AsyncTask([video_url]() { return get_playback_url_recursive(std::move(video_url)); }); + if(!audio_url.empty()) + tasks[1] = AsyncTask([audio_url]() { return get_playback_url_recursive(std::move(audio_url)); }); + + std::string *strings[2] = { &video_url, &audio_url }; + for(int i = 0; i < 2; ++i) { + if(tasks[i].valid()) + *strings[i] = tasks[i].get(); + } + } + // This is a common setup of text in the youtube json static std::optional yt_json_get_text(const Json::Value &json, const char *root_name) { if(!json.isObject()) @@ -427,131 +591,6 @@ R"END( } } - static std::mutex cookies_mutex; - static std::string cookies_filepath; - static std::string api_key; - - static bool is_whitespace(char c) { - return (c >= 8 && c <= 13) || c == ' '; - } - - // TODO: Cache this and redownload it when a network request fails with this api key? Do that in the same place as the signature, which means it would be done asynchronously - static std::string youtube_page_find_api_key() { - size_t api_key_index; - size_t api_key_index_end; - size_t api_key_length; - std::string website_result; - std::string::iterator api_key_start; - - if(download_to_string("https://www.youtube.com/?gl=US&hl=en", website_result, {}, true) != DownloadResult::OK) - goto fallback; - - api_key_index = website_result.find("INNERTUBE_API_KEY"); - if(api_key_index == std::string::npos) - goto fallback; - - api_key_index += 17; - api_key_start = std::find_if(website_result.begin() + api_key_index, website_result.end(), [](char c) { - return c != '"' && c != ':' && !is_whitespace(c); - }); - - if(api_key_start == website_result.end()) - goto fallback; - - api_key_index = api_key_start - website_result.begin(); - api_key_index_end = website_result.find('"', api_key_index); - if(api_key_index_end == std::string::npos) - goto fallback; - - api_key_length = api_key_index_end - api_key_index; - if(api_key_length > 512) // sanity check - goto fallback; - - return website_result.substr(api_key_index, api_key_length); - - fallback: - fprintf(stderr, "Failed to fetch youtube api key, fallback to %s\n", api_key.c_str()); - return "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; - } - - static std::string cpn; - - static bool generate_random_characters(char *buffer, int buffer_size, const char *alphabet, size_t alphabet_size) { - int fd = open("/dev/urandom", O_RDONLY); - if(fd == -1) { - perror("/dev/urandom"); - return false; - } - - if(read(fd, buffer, buffer_size) < buffer_size) { - fprintf(stderr, "Failed to read %d bytes from /dev/urandom\n", buffer_size); - close(fd); - return false; - } - - for(int i = 0; i < buffer_size; ++i) { - unsigned char c = *(unsigned char*)&buffer[i]; - buffer[i] = alphabet[c % alphabet_size]; - } - close(fd); - return true; - } - - static std::vector get_cookies() { - std::lock_guard lock(cookies_mutex); - if(cookies_filepath.empty()) { - YoutubeSignatureDecryptor::get_instance(); - - cpn.resize(16); - generate_random_characters(cpn.data(), cpn.size(), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_", 64); - - Path cookies_filepath_p; - if(get_cookies_filepath(cookies_filepath_p, "youtube") != 0) { - show_notification("QuickMedia", "Failed to create youtube cookies file", Urgency::CRITICAL); - return {}; - } - - // TODO: Re-enable this if the api key ever changes in the future. - // Maybe also put signature decryption in the same request? since it requests the same page. - #if 0 - //api_key = youtube_page_find_api_key(); - #else - api_key = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; - #endif - - if(get_file_type(cookies_filepath_p) == FileType::REGULAR) { - cookies_filepath = cookies_filepath_p.data; - } else { - Path cookies_filepath_tmp = cookies_filepath_p; - cookies_filepath_tmp.append(".tmp"); - - // TODO: This response also contains INNERTUBE_API_KEY which is the api key above. Maybe that should be parsed? - // TODO: Is there any way to bypass this? this is needed to set VISITOR_INFO1_LIVE which is required to read comments - const char *args[] = { "curl", "-I", "-s", "-f", "-L", "-b", cookies_filepath_tmp.data.c_str(), "-c", cookies_filepath_tmp.data.c_str(), "https://www.youtube.com/embed/watch?v=jNQXAC9IVRw&gl=US&hl=en", nullptr }; - if(exec_program(args, nullptr, nullptr) == 0) { - rename_atomic(cookies_filepath_tmp.data.c_str(), cookies_filepath_p.data.c_str()); - cookies_filepath = cookies_filepath_p.data; - } else { - show_notification("QuickMedia", "Failed to fetch cookies to view youtube comments", Urgency::CRITICAL); - return {}; - } - } - } - - return { - CommandArg{ "-b", cookies_filepath }, - CommandArg{ "-c", cookies_filepath } - }; - } - - static std::string remove_index_from_playlist_url(const std::string &url) { - std::string result = url; - size_t index = result.rfind("&index="); - if(index == std::string::npos) - return result; - return result.substr(0, index); - } - static std::shared_ptr parse_compact_video_renderer_json(const Json::Value &item_json, std::unordered_set &added_videos) { const Json::Value &compact_video_renderer_json = item_json["compactVideoRenderer"]; if(!compact_video_renderer_json.isObject()) @@ -1982,34 +2021,6 @@ R"END( return nullptr; } - // Sometimes youtube returns a redirect url (not in the header but in the body...). - // TODO: Find why this happens and if there is a way bypass it. - // Or maybe move this logic to QuickMedia video_content_page when mpv fails to start up (mpv will exit with exit code 2 and the message "Failed to recognize file format." when this happens). - // But that might be too slow for pinephone. - static std::string get_playback_url_recursive(std::string playback_url) { - const int max_redirects = 5; - for(int i = 0; i < max_redirects; ++i) { - std::string response_headers; - if(download_head_to_string(playback_url, response_headers, true) != DownloadResult::OK) - return ""; - - std::string content_type = header_extract_value(response_headers, "content-type"); - if(content_type.empty()) - return ""; - - if(string_starts_with(content_type, "video") || string_starts_with(content_type, "audio")) - return playback_url; - - // TODO: Download head and body in one request - std::string new_url; - if(download_to_string(playback_url, new_url, {}, true) != DownloadResult::OK) - return ""; - - playback_url = std::move(new_url); - } - return playback_url; - } - std::string YoutubeVideoPage::get_video_url(int max_height, bool &has_embedded_audio) { if(!hls_manifest_url.empty()) { has_embedded_audio = true; @@ -2036,7 +2047,7 @@ R"END( print_chosen_format(*chosen_video_format); has_embedded_audio = chosen_video_format->has_embedded_audio; - return get_playback_url_recursive(chosen_video_format->base.url); + return chosen_video_format->base.url; } std::string YoutubeVideoPage::get_audio_url() { @@ -2046,7 +2057,7 @@ R"END( // TODO: The "worst" (but still good) quality audio is chosen right now because youtube seeking freezes for up to 15 seconds when choosing the best quality const YoutubeAudioFormat *chosen_audio_format = &audio_formats.back(); fprintf(stderr, "Choosing youtube audio format: bitrate: %d, mime type: %s\n", chosen_audio_format->base.bitrate, chosen_audio_format->base.mime_type.c_str()); - return get_playback_url_recursive(chosen_audio_format->base.url); + return chosen_audio_format->base.url; } // Returns -1 if timestamp is in an invalid format @@ -2089,17 +2100,7 @@ R"END( return result; } - PluginResult YoutubeVideoPage::load(std::string &title, std::string &channel_url, std::vector &chapters) { - hls_manifest_url.clear(); - video_formats.clear(); - audio_formats.clear(); - - std::string video_id; - if(!youtube_url_extract_id(url, video_id)) { - fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); - return PluginResult::ERR; - } - + PluginResult YoutubeVideoPage::get_video_info(const std::string &video_id, Json::Value &json_root) { std::vector additional_args = get_cookies(); std::string response; @@ -2109,7 +2110,6 @@ R"END( std::string player_response_param = url_extract_param(response, "player_response"); player_response_param = url_param_decode(player_response_param); - Json::Value json_root; Json::CharReaderBuilder json_builder; std::unique_ptr json_reader(json_builder.newCharReader()); std::string json_errors; @@ -2118,27 +2118,83 @@ R"END( return PluginResult::ERR; } + return PluginResult::OK; + } + + PluginResult YoutubeVideoPage::load(std::string &title, std::string &channel_url, std::vector &chapters) { + hls_manifest_url.clear(); + video_formats.clear(); + audio_formats.clear(); + + std::string video_id; + if(!youtube_url_extract_id(url, video_id)) { + fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); + return PluginResult::ERR; + } + + #if 0 + std::string request_data = key_api_request_data; + string_replace_all(request_data, "%VIDEO_ID%", video_id); + + std::vector additional_args = { + { "-H", "Content-Type: application/json" }, + { "-H", "x-youtube-client-name: 1" }, + { "-H", youtube_client_version }, + { "--data-raw", std::move(request_data) } + }; + + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + + Json::Value json_root; + DownloadResult download_result = download_json(json_root, "https://www.youtube.com/youtubei/v1/player?key=" + api_key + "&gl=US&hl=en", additional_args, true); + if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result); + if(!json_root.isObject()) return PluginResult::ERR; - const Json::Value &streaming_data_json = json_root["streamingData"]; - if(!streaming_data_json.isObject()) { + const Json::Value *streaming_data_json = &json_root["streamingData"]; + if(!streaming_data_json->isObject()) { const Json::Value &playability_status_json = json_root["playabilityStatus"]; if(playability_status_json.isObject()) { const Json::Value &status_json = playability_status_json["status"]; const Json::Value &reason_json = playability_status_json["reason"]; - if(status_json.isString()) - fprintf(stderr, "Youtube video loading failed, reason: (status: %s, reason: %s)\n", status_json.asCString(), reason_json.isString() ? reason_json.asCString() : "unknown"); + fprintf(stderr, "Warning: youtube video loading failed, reason: (status: %s, reason: %s), trying with get_video_info endpoint instead\n", status_json.isString() ? status_json.asCString() : "unknown", reason_json.isString() ? reason_json.asCString() : "unknown"); + + json_root = Json::Value(Json::nullValue); + PluginResult result = get_video_info(video_id, json_root); + if(result != PluginResult::OK) + return result; + + if(!json_root.isObject()) + return PluginResult::ERR; + + streaming_data_json = &json_root["streamingData"]; + if(!streaming_data_json->isObject()) + return PluginResult::ERR; } return PluginResult::ERR; } + #else + Json::Value json_root; + PluginResult result = get_video_info(video_id, json_root); + if(result != PluginResult::OK) + return result; + + if(!json_root.isObject()) + return PluginResult::ERR; + + const Json::Value *streaming_data_json = &json_root["streamingData"]; + if(!streaming_data_json->isObject()) + return PluginResult::ERR; + #endif // TODO: Verify if this always works (what about copyrighted live streams?), also what about choosing video quality for live stream? Maybe use mpv --hls-bitrate option? - const Json::Value &hls_manifest_url_json = streaming_data_json["hlsManifestUrl"]; + const Json::Value &hls_manifest_url_json = (*streaming_data_json)["hlsManifestUrl"]; if(hls_manifest_url_json.isString()) { hls_manifest_url = hls_manifest_url_json.asString(); } else { - parse_formats(streaming_data_json); + parse_formats(*streaming_data_json); if(video_formats.empty() && audio_formats.empty()) return PluginResult::ERR; } diff --git a/src/plugins/youtube/Signature.cpp b/src/plugins/youtube/Signature.cpp index 65d4e2e..7631182 100644 --- a/src/plugins/youtube/Signature.cpp +++ b/src/plugins/youtube/Signature.cpp @@ -260,7 +260,7 @@ namespace QuickMedia { int YoutubeSignatureDecryptor::update_decrypt_function() { std::string response; - DownloadResult download_result = download_to_string("https://www.youtube.com/?gl=US&hl=en", response, {}, true); + DownloadResult download_result = download_to_string("https://www.youtube.com/watch?v=jNQXAC9IVRw&gl=US&hl=en", response, {}, true); if(download_result != DownloadResult::OK) { fprintf(stderr, "YoutubeSignatureDecryptor::update_decrypt_function failed. Failed to get youtube page\n"); return U_DEC_FUN_NET_ERR; -- cgit v1.2.3