From b2adec14313749f67b8caae7f4952e0eae04a84d Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sun, 27 Jun 2021 03:09:55 +0200 Subject: Fix youtube comments --- src/plugins/Youtube.cpp | 166 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 111 insertions(+), 55 deletions(-) (limited to 'src/plugins') diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index f20f38e..276fba5 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -104,6 +104,8 @@ R"END( static std::mutex cookies_mutex; static std::string cookies_filepath; static std::string api_key; + static std::string ysc; + static std::string visitor_info1_live; static bool is_whitespace(char c) { return (c >= 8 && c <= 13) || c == ' '; @@ -171,6 +173,20 @@ R"END( return true; } + static std::string header_get_cookie(const char *str, size_t size, const char *cookies_key) { + const int cookie_key_len = strlen(cookies_key); + const char *cookie_p = (const char*)memmem(str, size, cookies_key, cookie_key_len); + if(!cookie_p) + return ""; + + cookie_p += cookie_key_len; + const void *end_p = memchr(cookie_p, ';', (size_t)(str + size - cookie_p)); + if(!end_p) + end_p = str + size; + + return std::string(cookie_p, (const char*)end_p); + } + static std::vector get_cookies() { std::lock_guard lock(cookies_mutex); if(cookies_filepath.empty()) { @@ -180,7 +196,7 @@ R"END( generate_random_characters(cpn.data(), cpn.size(), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_", 64); Path cookies_filepath_p; - if(get_cookies_filepath(cookies_filepath_p, "youtube") != 0) { + if(get_cookies_filepath(cookies_filepath_p, "youtube-custom") != 0) { show_notification("QuickMedia", "Failed to create youtube cookies file", Urgency::CRITICAL); return {}; } @@ -195,16 +211,47 @@ R"END( if(get_file_type(cookies_filepath_p) == FileType::REGULAR) { cookies_filepath = cookies_filepath_p.data; - } else { - Path cookies_filepath_tmp = cookies_filepath_p; - cookies_filepath_tmp.append(".tmp"); + std::string file_content; + if(file_get_content(cookies_filepath_p, file_content) != 0) { + show_notification("QuickMedia", "Failed to load cookies to view youtube comments", Urgency::CRITICAL); + return {}; + } + + const size_t line_end_index = file_content.find('\n'); + if(line_end_index == std::string::npos) { + show_notification("QuickMedia", "Failed to load cookies to view youtube comments", Urgency::CRITICAL); + return {}; + } + + ysc = file_content.substr(0, line_end_index); + visitor_info1_live = file_content.substr(line_end_index + 1); + } else { // TODO: This response also contains INNERTUBE_API_KEY which is the api key above. Maybe that should be parsed? // TODO: Is there any way to bypass this? this is needed to set VISITOR_INFO1_LIVE which is required to read comments - const char *args[] = { "curl", "-I", "-s", "-f", "-L", "-b", cookies_filepath_tmp.data.c_str(), "-c", cookies_filepath_tmp.data.c_str(), "https://www.youtube.com/embed/watch?v=jNQXAC9IVRw&gl=US&hl=en", nullptr }; - if(exec_program(args, nullptr, nullptr) == 0) { - rename_atomic(cookies_filepath_tmp.data.c_str(), cookies_filepath_p.data.c_str()); - cookies_filepath = cookies_filepath_p.data; + std::string response; + if(download_head_to_string("https://www.youtube.com/embed/watch?v=jNQXAC9IVRw&gl=US&hl=en", response, true) == DownloadResult::OK) { + string_split(response, "\r\n", [](const char *str, size_t size){ + if(size > 11 && memcmp(str, "set-cookie:", 11) == 0) { + if(ysc.empty()) { + std::string ysc_cookie = header_get_cookie(str + 11, size - 11, "YSC="); + if(!ysc_cookie.empty()) + ysc = std::move(ysc_cookie); + } + + if(visitor_info1_live.empty()) { + std::string visitor_info = header_get_cookie(str + 11, size - 11, "VISITOR_INFO1_LIVE="); + if(!visitor_info.empty()) + visitor_info1_live = std::move(visitor_info); + } + } + return true; + }); + + if(ysc.empty() || visitor_info1_live.empty() || file_overwrite_atomic(cookies_filepath_p, ysc + "\n" + visitor_info1_live) != 0) { + show_notification("QuickMedia", "Failed to fetch cookies to view youtube comments", Urgency::CRITICAL); + return {}; + } } else { show_notification("QuickMedia", "Failed to fetch cookies to view youtube comments", Urgency::CRITICAL); return {}; @@ -213,8 +260,7 @@ R"END( } return { - CommandArg{ "-b", cookies_filepath }, - CommandArg{ "-c", cookies_filepath } + CommandArg{ "-H", "cookie: YSC=" + ysc + "; VISITOR_INFO1_LIVE=" + visitor_info1_live + "; CONSENT=YES+SE.sv+V10" } }; } @@ -1907,6 +1953,8 @@ R"END( } BodyItems YoutubeVideoPage::get_related_media(const std::string &url) { + xsrf_token.clear(); + comments_continuation_token.clear(); BodyItems result_items; std::string video_id; @@ -1915,76 +1963,84 @@ R"END( return result_items; } - std::string request_data = key_api_request_data; - string_replace_all(request_data, "%VIDEO_ID%", video_id); - std::vector additional_args = { - { "-H", "Content-Type: application/json" }, { "-H", "x-youtube-client-name: 1" }, { "-H", youtube_client_version }, - { "--data-raw", std::move(request_data) } }; std::vector cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; - DownloadResult download_result = download_json(json_root, "https://www.youtube.com/youtubei/v1/next?key=" + api_key + "&gl=US&hl=en", additional_args, true); + DownloadResult download_result = download_json(json_root, "https://www.youtube.com/watch?v=" + video_id + "&pbj=1&gl=US&hl=en", additional_args, true); if(download_result != DownloadResult::OK) return result_items; - if(!json_root.isObject()) + if(!json_root.isArray()) return result_items; std::unordered_set added_videos; - xsrf_token.clear(); // TODO: Get xsrf token somehow - comments_continuation_token.clear(); - const Json::Value &contents_json = json_root["contents"]; - if(!contents_json.isObject()) - return result_items; + for(const Json::Value &json_item : json_root) { + if(!json_item.isObject()) + continue; - const Json::Value &tcwnr_json = contents_json["twoColumnWatchNextResults"]; - if(!tcwnr_json.isObject()) - return result_items; + if(xsrf_token.empty()) { + const Json::Value &xsrf_token_json = json_item["xsrf_token"]; + if(xsrf_token_json.isString()) + xsrf_token = xsrf_token_json.asString(); + } + + const Json::Value &response_json = json_item["response"]; + if(!response_json.isObject()) + continue; - if(comments_continuation_token.empty()) - comments_continuation_token = two_column_watch_next_results_get_comments_continuation_token(tcwnr_json); + const Json::Value &contents_json = response_json["contents"]; + if(!contents_json.isObject()) + return result_items; - const Json::Value &secondary_results_json = tcwnr_json["secondaryResults"]; - if(!secondary_results_json.isObject()) - return result_items; + const Json::Value &tcwnr_json = contents_json["twoColumnWatchNextResults"]; + if(!tcwnr_json.isObject()) + return result_items; - const Json::Value &secondary_results2_json = secondary_results_json["secondaryResults"]; - if(!secondary_results2_json.isObject()) - return result_items; - - const Json::Value &results_json = secondary_results2_json["results"]; - if(!results_json.isArray()) - return result_items; + if(comments_continuation_token.empty()) + comments_continuation_token = two_column_watch_next_results_get_comments_continuation_token(tcwnr_json); - for(const Json::Value &item_json : results_json) { - if(!item_json.isObject()) - continue; + const Json::Value &secondary_results_json = tcwnr_json["secondaryResults"]; + if(!secondary_results_json.isObject()) + return result_items; - auto body_item = parse_compact_video_renderer_json(item_json, added_videos); - if(body_item) - result_items.push_back(std::move(body_item)); - - const Json::Value &compact_autoplay_renderer_json = item_json["compactAutoplayRenderer"]; - if(!compact_autoplay_renderer_json.isObject()) - continue; - - const Json::Value &item_contents_json = compact_autoplay_renderer_json["contents"]; - if(!item_contents_json.isArray()) - continue; + const Json::Value &secondary_results2_json = secondary_results_json["secondaryResults"]; + if(!secondary_results2_json.isObject()) + return result_items; - for(const Json::Value &content_item_json : item_contents_json) { - if(!content_item_json.isObject()) + const Json::Value &results_json = secondary_results2_json["results"]; + if(!results_json.isArray()) + return result_items; + + for(const Json::Value &item_json : results_json) { + if(!item_json.isObject()) continue; - - auto body_item = parse_compact_video_renderer_json(content_item_json, added_videos); + + auto body_item = parse_compact_video_renderer_json(item_json, added_videos); if(body_item) result_items.push_back(std::move(body_item)); + + const Json::Value &compact_autoplay_renderer_json = item_json["compactAutoplayRenderer"]; + if(!compact_autoplay_renderer_json.isObject()) + continue; + + const Json::Value &item_contents_json = compact_autoplay_renderer_json["contents"]; + if(!item_contents_json.isArray()) + continue; + + for(const Json::Value &content_item_json : item_contents_json) { + if(!content_item_json.isObject()) + continue; + + auto body_item = parse_compact_video_renderer_json(content_item_json, added_videos); + if(body_item) + result_items.push_back(std::move(body_item)); + } } } -- cgit v1.2.3