diff options
| -rw-r--r-- | plugins/Youtube.hpp | 11 | ||||
| -rw-r--r-- | src/plugins/Youtube.cpp | 343 |
2 files changed, 210 insertions, 144 deletions
diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp index 1b17c7b..2958c44 100644 --- a/plugins/Youtube.hpp +++ b/plugins/Youtube.hpp @@ -49,27 +49,27 @@ namespace QuickMedia { class YoutubeCommentsPage : public LazyFetchPage { public: - YoutubeCommentsPage(Program *program, const std::string &xsrf_token, const std::string &continuation_token) : LazyFetchPage(program), xsrf_token(xsrf_token), continuation_token(continuation_token) {} + YoutubeCommentsPage(Program *program, const std::string &video_url, const std::string &continuation_token) : LazyFetchPage(program), video_url(video_url), continuation_token(continuation_token) {} const char* get_title() const override { return "Comments"; } PluginResult get_page(const std::string &str, int page, BodyItems &result_items) override; PluginResult submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) override; PluginResult lazy_fetch(BodyItems &result_items) override; private: int current_page = 0; - std::string xsrf_token; + std::string video_url; std::string continuation_token; }; class YoutubeCommentRepliesPage : public LazyFetchPage { public: - YoutubeCommentRepliesPage(Program *program, const std::string &xsrf_token, const std::string &continuation_token) : LazyFetchPage(program), xsrf_token(xsrf_token), continuation_token(continuation_token) {} + YoutubeCommentRepliesPage(Program *program, const std::string &video_url, const std::string &continuation_token) : LazyFetchPage(program), video_url(video_url), continuation_token(continuation_token) {} const char* get_title() const override { return "Comment replies"; } PluginResult get_page(const std::string &str, int page, BodyItems &result_items) override; PluginResult submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) override; PluginResult lazy_fetch(BodyItems &result_items) override; private: int current_page = 0; - std::string xsrf_token; + std::string video_url; std::string continuation_token; }; @@ -155,7 +155,6 @@ namespace QuickMedia { void parse_formats(const Json::Value &streaming_data_json); private: std::string timestamp; - std::string xsrf_token; std::string comments_continuation_token; std::string livestream_url; std::vector<YoutubeVideoFormat> video_formats; @@ -165,4 +164,4 @@ namespace QuickMedia { std::string watchtime_url; std::string tracking_url; }; -}
\ No newline at end of file +} diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index 13655c7..eb95c05 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -687,21 +687,18 @@ R"END( if(!item_section_renderer.isObject()) return ""; - const Json::Value &continuations_json = item_section_renderer["continuations"]; - if(!continuations_json.isArray()) + const Json::Value &contents_json = item_section_renderer["contents"]; + if(!contents_json.isArray()) return ""; - for(const Json::Value &json_item : continuations_json) { + std::string continuation_token; + for(const Json::Value &json_item : contents_json) { if(!json_item.isObject()) continue; - const Json::Value &next_continuation_data_json = json_item["nextContinuationData"]; - if(!next_continuation_data_json.isObject()) - continue; - - const Json::Value &continuation_json = next_continuation_data_json["continuation"]; - if(continuation_json.isString()) - return continuation_json.asString(); + continuation_token = item_section_renderer_get_continuation_token(json_item); + if(!continuation_token.empty()) + return continuation_token; } return ""; @@ -797,9 +794,7 @@ R"END( } } - if(!new_continuation_token.empty()) - continuation_token = std::move(new_continuation_token); - + continuation_token = std::move(new_continuation_token); return body_items; } @@ -905,6 +900,9 @@ R"END( } PluginResult YoutubeSearchPage::search_get_continuation(const std::string &url, const std::string ¤t_continuation_token, BodyItems &result_items) { + if(current_continuation_token.empty()) + return PluginResult::OK; + std::string next_url = url + "&pbj=1&gl=US&hl=en&ctoken=" + current_continuation_token; std::vector<CommandArg> additional_args = { @@ -968,9 +966,7 @@ R"END( } } - if(!new_continuation_token.empty()) - continuation_token = std::move(new_continuation_token); - + continuation_token = std::move(new_continuation_token); return PluginResult::OK; } @@ -986,10 +982,34 @@ R"END( PluginResult YoutubeCommentsPage::submit(const std::string&, const std::string &url, std::vector<Tab> &result_tabs) { if(url.empty()) return PluginResult::OK; - result_tabs.push_back(Tab{create_body(), std::make_unique<YoutubeCommentRepliesPage>(program, xsrf_token, url), nullptr}); + result_tabs.push_back(Tab{create_body(), std::make_unique<YoutubeCommentRepliesPage>(program, video_url, url), nullptr}); return PluginResult::OK; } + static std::string item_section_continuation_get_continuation_token(const Json::Value &item_section_continuation_json) { + if(!item_section_continuation_json.isObject()) + return ""; + + const Json::Value &continuations_json = item_section_continuation_json["continuations"]; + if(!continuations_json.isArray()) + return ""; + + for(const Json::Value &json_item : continuations_json) { + if(!json_item.isObject()) + continue; + + const Json::Value &next_continuation_data_json = json_item["nextContinuationData"]; + if(!next_continuation_data_json.isObject()) + continue; + + const Json::Value &continuation_json = next_continuation_data_json["continuation"]; + if(continuation_json.isString()) + return continuation_json.asString(); + } + + return ""; + } + static std::string comment_thread_renderer_get_replies_continuation(const Json::Value &comment_thread_renderer_json) { if(!comment_thread_renderer_json.isObject()) return ""; @@ -998,12 +1018,12 @@ R"END( if(!replies_json.isObject()) return ""; - const Json::Value &comment_replies_renderer = replies_json["commentRepliesRenderer"]; - if(!comment_replies_renderer.isObject()) - return ""; + const Json::Value &comment_replies_renderer_json = replies_json["commentRepliesRenderer"]; + std::string continuation = item_section_renderer_get_continuation(comment_replies_renderer_json); + if(!continuation.empty()) + return continuation; - // item_section_renderer_get_continuation is compatible with commentRepliesRenderer - return item_section_renderer_get_continuation(comment_replies_renderer); + return item_section_continuation_get_continuation_token(comment_replies_renderer_json); } // Returns empty string if comment is not hearted @@ -1092,152 +1112,208 @@ R"END( return body_item; } - PluginResult YoutubeCommentsPage::lazy_fetch(BodyItems &result_items) { - if(continuation_token.empty()) - return PluginResult::OK; - - std::string next_url = "https://www.youtube.com/comment_service_ajax?action_get_comments=1&pbj=1&ctoken="; - next_url += url_param_encode(continuation_token); - //next_url += "&continuation="; - //next_url += url_param_encode(comments_continuation_token); - next_url += "&type=next&gl=US&hl=en"; + static std::string continuation_item_renderer_get_continuation_token(const Json::Value &continuation_item_renderer_json) { + if(!continuation_item_renderer_json.isObject()) + return ""; - std::vector<CommandArg> additional_args = { - { "-H", "x-youtube-client-name: 1" }, - { "-H", youtube_client_version }, - { "-F", "session_token=" + xsrf_token } - }; + const Json::Value &button_json = continuation_item_renderer_json["button"]; + if(!button_json.isObject()) + return ""; - std::vector<CommandArg> cookies = get_cookies(); - additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + const Json::Value &button_renderer_json = button_json["buttonRenderer"]; + if(!button_renderer_json.isObject()) + return ""; - Json::Value json_root; - DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); - if(result != DownloadResult::OK) return download_result_to_plugin_result(result); + const Json::Value &command_json = button_renderer_json["command"]; + if(!command_json.isObject()) + return ""; - if(!json_root.isObject()) - return PluginResult::ERR; + const Json::Value &continuation_command_json = command_json["continuationCommand"]; + if(!continuation_command_json.isObject()) + return ""; - const Json::Value &xsrf_token_json = json_root["xsrf_token"]; - if(xsrf_token_json.isString()) - xsrf_token = xsrf_token_json.asString(); + const Json::Value &token_json = continuation_command_json["token"]; + if(!token_json.isString()) + return ""; - const Json::Value &response_json = json_root["response"]; - if(!response_json.isObject()) - return PluginResult::ERR; + return token_json.asString(); + } - const Json::Value &continuation_contents_json = response_json["continuationContents"]; - if(!continuation_contents_json.isObject()) + static PluginResult fetch_comments_received_endpoints(const Json::Value &json_root, BodyItems &result_items, std::string &continuation_token) { + const Json::Value &on_response_received_endpoints_json = json_root["onResponseReceivedEndpoints"]; + if(!on_response_received_endpoints_json.isArray()) return PluginResult::ERR; - const Json::Value &item_section_continuation_json = continuation_contents_json["itemSectionContinuation"]; - if(!item_section_continuation_json.isObject()) - return PluginResult::ERR; + std::string new_continuation_token; + for(const Json::Value &json_item : on_response_received_endpoints_json) { + if(!json_item.isObject()) + continue; - const Json::Value &contents_json = item_section_continuation_json["contents"]; - if(contents_json.isArray()) { - for(const Json::Value &json_item : contents_json) { - if(!json_item.isObject()) + const Json::Value *append_continuation_items_action_json = &json_item["reloadContinuationItemsCommand"]; + if(!append_continuation_items_action_json->isObject()) { + append_continuation_items_action_json = &json_item["appendContinuationItemsAction"]; + if(!append_continuation_items_action_json->isObject()) continue; + } - const Json::Value &comment_thread_renderer = json_item["commentThreadRenderer"]; - if(!comment_thread_renderer.isObject()) - continue; + const Json::Value &continuation_items_json = (*append_continuation_items_action_json)["continuationItems"]; + if(!continuation_items_json.isArray()) + continue; - const Json::Value &comment_json = comment_thread_renderer["comment"]; - if(!comment_json.isObject()) + for(const Json::Value &continuation_item_json : continuation_items_json) { + if(!continuation_item_json.isObject()) continue; - auto body_item = comment_renderer_to_body_item(comment_json["commentRenderer"]); - if(body_item) { - body_item->url = comment_thread_renderer_get_replies_continuation(comment_thread_renderer); + if(new_continuation_token.empty()) + new_continuation_token = item_section_renderer_get_continuation_token(continuation_item_json); + + const Json::Value &comment_thread_renderer_json = continuation_item_json["commentThreadRenderer"]; + if(comment_thread_renderer_json.isObject()) { + const Json::Value &comment_json = comment_thread_renderer_json["comment"]; + if(!comment_json.isObject()) + continue; + + auto body_item = comment_renderer_to_body_item(comment_json["commentRenderer"]); + if(!body_item) + continue; + + body_item->url = comment_thread_renderer_get_replies_continuation(comment_thread_renderer_json); result_items.push_back(std::move(body_item)); + } else { + auto body_item = comment_renderer_to_body_item(continuation_item_json["commentRenderer"]); + if(body_item) + result_items.push_back(std::move(body_item)); + + if(new_continuation_token.empty()) + new_continuation_token = continuation_item_renderer_get_continuation_token(continuation_item_json["continuationItemRenderer"]); } } } - continuation_token = item_section_renderer_get_continuation(item_section_continuation_json); - + continuation_token = std::move(new_continuation_token); return PluginResult::OK; } - PluginResult YoutubeCommentRepliesPage::get_page(const std::string&, int page, BodyItems &result_items) { - while(current_page < page) { - PluginResult plugin_result = lazy_fetch(result_items); - if(plugin_result != PluginResult::OK) return plugin_result; - ++current_page; + static PluginResult fetch_comments_continuation_contents(const Json::Value &json_root, BodyItems &result_items, std::string &continuation_token) { + const Json::Value &continuation_contents_json = json_root["continuationContents"]; + if(!continuation_contents_json.isObject()) + return PluginResult::ERR; + + const Json::Value *item_section_continuation_json = &continuation_contents_json["itemSectionContinuation"]; + if(!item_section_continuation_json->isObject()) { + item_section_continuation_json = &continuation_contents_json["commentRepliesContinuation"]; + if(!item_section_continuation_json->isObject()) + return PluginResult::ERR; } - return PluginResult::OK; - } - PluginResult YoutubeCommentRepliesPage::submit(const std::string&, const std::string&, std::vector<Tab>&) { + const Json::Value &contents_json = (*item_section_continuation_json)["contents"]; + if(!contents_json.isArray()) + return PluginResult::ERR; + + std::string new_continuation_token; + for(const Json::Value &json_item : contents_json) { + if(!json_item.isObject()) + continue; + + const Json::Value &comment_thread_renderer_json = json_item["commentThreadRenderer"]; + if(comment_thread_renderer_json.isObject()) { + const Json::Value &comment_json = comment_thread_renderer_json["comment"]; + if(!comment_json.isObject()) + continue; + + auto body_item = comment_renderer_to_body_item(comment_json["commentRenderer"]); + if(!body_item) + continue; + + body_item->url = comment_thread_renderer_get_replies_continuation(comment_thread_renderer_json); + result_items.push_back(std::move(body_item)); + } else { + auto body_item = comment_renderer_to_body_item(json_item["commentRenderer"]); + if(body_item) + result_items.push_back(std::move(body_item)); + + if(new_continuation_token.empty()) + new_continuation_token = continuation_item_renderer_get_continuation_token(json_item["continuationItemRenderer"]); + } + } + + if(new_continuation_token.empty()) + new_continuation_token = item_section_continuation_get_continuation_token(*item_section_continuation_json); + + continuation_token = std::move(new_continuation_token); return PluginResult::OK; } - PluginResult YoutubeCommentRepliesPage::lazy_fetch(BodyItems &result_items) { + static PluginResult fetch_comments(Page *page, const std::string &video_url, std::string &continuation_token, BodyItems &result_items) { if(continuation_token.empty()) return PluginResult::OK; - std::string next_url = "https://www.youtube.com/comment_service_ajax?action_get_comment_replies=1&pbj=1&ctoken="; - next_url += url_param_encode(continuation_token); - //next_url += "&continuation="; - //next_url += url_param_encode(comments_continuation_token); - next_url += "&type=next&gl=US&hl=en"; + std::vector<CommandArg> cookies = get_cookies(); + std::string next_url = "https://www.youtube.com/youtubei/v1/next?key=" + url_param_encode(api_key) + "&gl=US&hl=en"; + + Json::Value request_json(Json::objectValue); + Json::Value context_json(Json::objectValue); + Json::Value client_json(Json::objectValue); + client_json["hl"] = "en"; + client_json["gl"] = "US"; + client_json["deviceMake"] = ""; + client_json["deviceModel"] = ""; + client_json["userAgent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; + client_json["clientName"] = "WEB"; + client_json["clientVersion"] = "2.20210701.07.00"; + client_json["osName"] = "X11"; + client_json["osVersion"] = ""; + client_json["originalUrl"] = video_url; + context_json["client"] = std::move(client_json); + request_json["context"] = std::move(context_json); + request_json["continuation"] = continuation_token; + + Json::StreamWriterBuilder json_builder; + json_builder["commentStyle"] = "None"; + json_builder["indentation"] = ""; std::vector<CommandArg> additional_args = { + { "-H", "content-type: application/json" }, { "-H", "x-youtube-client-name: 1" }, { "-H", youtube_client_version }, - { "-F", "session_token=" + xsrf_token } + { "--data-raw", Json::writeString(json_builder, request_json) } }; - std::vector<CommandArg> cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; - DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); + DownloadResult result = page->download_json(json_root, next_url, std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_plugin_result(result); - if(!json_root.isArray()) + if(!json_root.isObject()) return PluginResult::ERR; + + PluginResult res = fetch_comments_received_endpoints(json_root, result_items, continuation_token); + if(res == PluginResult::OK) + return res; - for(const Json::Value &json_item : json_root) { - if(!json_item.isObject()) - continue; - - const Json::Value &xsrf_token_json = json_item["xsrf_token"]; - if(xsrf_token_json.isString()) - xsrf_token = xsrf_token_json.asString(); - - const Json::Value &response_json = json_item["response"]; - if(!response_json.isObject()) - continue; - - const Json::Value &continuation_contents_json = response_json["continuationContents"]; - if(!continuation_contents_json.isObject()) - continue; - - const Json::Value &comment_replies_continuation_json = continuation_contents_json["commentRepliesContinuation"]; - if(!comment_replies_continuation_json.isObject()) - continue; - - const Json::Value &contents_json = comment_replies_continuation_json["contents"]; - if(contents_json.isArray()) { - for(const Json::Value &content_item_json : contents_json) { - if(!content_item_json.isObject()) - continue; + return fetch_comments_continuation_contents(json_root, result_items, continuation_token); + } - auto body_item = comment_renderer_to_body_item(content_item_json["commentRenderer"]); - if(body_item) - result_items.push_back(std::move(body_item)); - } - } + PluginResult YoutubeCommentsPage::lazy_fetch(BodyItems &result_items) { + return fetch_comments(this, video_url, continuation_token, result_items); + } - // item_section_renderer_get_continuation is compatible with commentRepliesContinuation - continuation_token = item_section_renderer_get_continuation(comment_replies_continuation_json); - return PluginResult::OK; + PluginResult YoutubeCommentRepliesPage::get_page(const std::string&, int page, BodyItems &result_items) { + while(current_page < page) { + PluginResult plugin_result = lazy_fetch(result_items); + if(plugin_result != PluginResult::OK) return plugin_result; + ++current_page; } + return PluginResult::OK; + } + + PluginResult YoutubeCommentRepliesPage::submit(const std::string&, const std::string&, std::vector<Tab>&) { + return PluginResult::OK; + } - return PluginResult::ERR; + PluginResult YoutubeCommentRepliesPage::lazy_fetch(BodyItems &result_items) { + return fetch_comments(this, video_url, continuation_token, result_items); } static std::string channel_url_extract_id(const std::string &channel_url) { @@ -1427,9 +1503,7 @@ R"END( } } - if(!new_continuation_token.empty()) - continuation_token = std::move(new_continuation_token); - + continuation_token = std::move(new_continuation_token); return PluginResult::OK; } @@ -1708,6 +1782,9 @@ R"END( } PluginResult YoutubeRecommendedPage::search_get_continuation(const std::string ¤t_continuation_token, BodyItems &result_items) { + if(current_continuation_token.empty()) + return PluginResult::OK; + std::string next_url = "https://www.youtube.com/?pbj=1&gl=US&hl=en&ctoken=" + current_continuation_token; std::vector<CommandArg> additional_args = { @@ -1779,9 +1856,7 @@ R"END( } } - if(!new_continuation_token.empty()) - continuation_token = std::move(new_continuation_token); - + continuation_token = std::move(new_continuation_token); return PluginResult::OK; } @@ -1877,9 +1952,7 @@ R"END( } } - if(!new_continuation_token.empty()) - continuation_token = std::move(new_continuation_token); - + continuation_token = std::move(new_continuation_token); return PluginResult::OK; } @@ -1955,7 +2028,6 @@ R"END( } BodyItems YoutubeVideoPage::get_related_media(const std::string &url) { - xsrf_token.clear(); comments_continuation_token.clear(); BodyItems result_items; @@ -1973,6 +2045,7 @@ R"END( std::vector<CommandArg> cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + // TODO: Remove this code completely and replace with existing player? api Json::Value json_root; DownloadResult download_result = download_json(json_root, "https://www.youtube.com/watch?v=" + video_id + "&pbj=1&gl=US&hl=en", additional_args, true); if(download_result != DownloadResult::OK) return result_items; @@ -1986,12 +2059,6 @@ R"END( if(!json_item.isObject()) continue; - if(xsrf_token.empty()) { - const Json::Value &xsrf_token_json = json_item["xsrf_token"]; - if(xsrf_token_json.isString()) - xsrf_token = xsrf_token_json.asString(); - } - const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) continue; @@ -2055,7 +2122,7 @@ R"END( } std::unique_ptr<Page> YoutubeVideoPage::create_comments_page(Program *program) { - return std::make_unique<YoutubeCommentsPage>(program, xsrf_token, comments_continuation_token); + return std::make_unique<YoutubeCommentsPage>(program, url, comments_continuation_token); } std::unique_ptr<RelatedVideosPage> YoutubeVideoPage::create_related_videos_page(Program *program) { @@ -2494,4 +2561,4 @@ R"END( const Json::Value &adaptive_formats_json = streaming_data_json["adaptiveFormats"]; parse_format(adaptive_formats_json, true); } -}
\ No newline at end of file +} |