From 83a4df36832156d08fbf04294164979efcaa06ab Mon Sep 17 00:00:00 2001 From: dec05eba Date: Thu, 15 Oct 2020 17:18:05 +0200 Subject: Youtube: fetch next page when reaching bottom of search (also fixes continuation) --- plugins/Youtube.hpp | 7 ++- src/QuickMedia.cpp | 9 ++-- src/plugins/Youtube.cpp | 113 +++++++++++++++++++++++++++++++----------------- 3 files changed, 85 insertions(+), 44 deletions(-) diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp index 0922b9d..007f398 100644 --- a/plugins/Youtube.hpp +++ b/plugins/Youtube.hpp @@ -9,9 +9,14 @@ namespace QuickMedia { const char* get_title() const override { return "All"; } bool search_is_filter() override { return false; } SearchResult search(const std::string &str, BodyItems &result_items) override; + PluginResult get_page(const std::string &str, int page, BodyItems &result_items) override; PluginResult submit(const std::string &title, const std::string &url, std::vector &result_tabs) override; private: - void search_suggestions_get_continuation(const std::string &url, const std::string &continuation_token, BodyItems &result_items); + PluginResult search_get_continuation(const std::string &url, const std::string &continuation_token, BodyItems &result_items); + private: + std::string search_url; + std::string continuation_token; + int current_page = 0; }; class YoutubeVideoPage : public Page { diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 3b059d9..4c41e28 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -1231,6 +1231,7 @@ namespace QuickMedia { BodyItems result_items = associated_data.search_future.get(); tabs[i].body->items = std::move(result_items); tabs[i].body->select_first_item(); + associated_data.fetched_page = 0; if(tabs[i].body->items.empty()) associated_data.search_result_text.setString("No results found"); else @@ -1656,7 +1657,7 @@ namespace QuickMedia { } else if(event.key.code == sf::Keyboard::R && event.key.control) { related_media_window_visible = false; related_media_window->setVisible(related_media_window_visible); - related_media_body->clear_thumbnails(); + related_media_body->clear_cache(); } else if(event.key.code == sf::Keyboard::F && event.key.control) { window_set_fullscreen(disp, window.getSystemHandle(), WindowFullscreenState::TOGGLE); } else if(event.key.code == sf::Keyboard::Enter) { @@ -3231,7 +3232,7 @@ namespace QuickMedia { } else if(event.key.code == sf::Keyboard::Escape) { current_page = PageType::EXIT; } else if(event.key.code == sf::Keyboard::Left && synced) { - tabs[selected_tab].body->clear_thumbnails(); + tabs[selected_tab].body->clear_cache(); selected_tab = std::max(0, selected_tab - 1); read_marker_timer.restart(); redraw = true; @@ -3241,7 +3242,7 @@ namespace QuickMedia { typing_futures.push_back(std::async(typing_async_func, false, current_room_id)); } } else if(event.key.code == sf::Keyboard::Right && synced) { - tabs[selected_tab].body->clear_thumbnails(); + tabs[selected_tab].body->clear_cache(); selected_tab = std::min((int)tabs.size() - 1, selected_tab + 1); read_marker_timer.restart(); redraw = true; @@ -3458,7 +3459,7 @@ namespace QuickMedia { case PageType::CHAT_LOGIN: { new_page = PageType::CHAT; matrix->logout(); - tabs[MESSAGES_TAB_INDEX].body->clear_thumbnails(); + tabs[MESSAGES_TAB_INDEX].body->clear_cache(); // TODO: Instead of doing this, exit this current function and navigate to chat login page instead. // This doesn't currently work because at the end of this function there are futures that need to wait // and one of them is /sync, which has a timeout of 30 seconds. That timeout has to be killed somehow. diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index bff0592..1711e41 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -114,29 +114,26 @@ namespace QuickMedia { // Returns empty string if continuation token can't be found static std::string item_section_renderer_get_continuation_token(const Json::Value &item_section_renderer_json) { - const Json::Value &continuations_json = item_section_renderer_json["continuations"]; - if(!continuations_json.isArray() || continuations_json.empty()) + const Json::Value &continuation_item_renderer_json = item_section_renderer_json["continuationItemRenderer"]; + if(!continuation_item_renderer_json.isObject()) return ""; - - const Json::Value &first_continuation_json = continuations_json[0]; - if(!first_continuation_json.isObject()) + + const Json::Value &continuation_endpoint_json = continuation_item_renderer_json["continuationEndpoint"]; + if(!continuation_endpoint_json.isObject()) return ""; - - const Json::Value &next_continuation_data_json = first_continuation_json["nextContinuationData"]; - if(!next_continuation_data_json.isObject()) + + const Json::Value &continuation_command_json = continuation_endpoint_json["continuationCommand"]; + if(!continuation_command_json.isObject()) return ""; - - const Json::Value &continuation_json = next_continuation_data_json["continuation"]; - if(!continuation_json.isString()) + + const Json::Value &token_json = continuation_command_json["token"]; + if(!token_json.isString()) return ""; - return continuation_json.asString(); + return token_json.asString(); } - static void parse_item_section_renderer(const Json::Value &item_section_renderer_json, std::string &continuation_token, std::unordered_set &added_videos, BodyItems &result_items) { - if(continuation_token.empty()) - continuation_token = item_section_renderer_get_continuation_token(item_section_renderer_json); - + static void parse_item_section_renderer(const Json::Value &item_section_renderer_json, std::unordered_set &added_videos, BodyItems &result_items) { const Json::Value &item_contents_json = item_section_renderer_json["contents"]; if(!item_contents_json.isArray()) return; @@ -195,27 +192,29 @@ namespace QuickMedia { } SearchResult YoutubeSearchPage::search(const std::string &str, BodyItems &result_items) { - std::string url = "https://youtube.com/results?search_query="; - url += url_param_encode(str); + continuation_token.clear(); + current_page = 0; + + search_url = "https://youtube.com/results?search_query="; + search_url += url_param_encode(str); std::vector additional_args = { - { "-H", "x-spf-referer: " + url }, + { "-H", "x-spf-referer: " + search_url }, { "-H", "x-youtube-client-name: 1" }, { "-H", "x-youtube-client-version: 2.20200626.03.00" }, - { "-H", "referer: " + url } + { "-H", "referer: " + search_url } }; //std::vector cookies = get_cookies(); //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; - DownloadResult result = download_json(json_root, url + "&pbj=1", std::move(additional_args), true); + DownloadResult result = download_json(json_root, search_url + "&pbj=1", std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_search_result(result); if(!json_root.isArray()) return SearchResult::ERR; - std::string continuation_token; std::unordered_set added_videos; /* The input contains duplicates, filter them out! */ for(const Json::Value &json_item : json_root) { @@ -249,22 +248,31 @@ namespace QuickMedia { for(const Json::Value &item_json : contents2_json) { if(!item_json.isObject()) continue; + + if(continuation_token.empty()) + continuation_token = item_section_renderer_get_continuation_token(item_json); const Json::Value &item_section_renderer_json = item_json["itemSectionRenderer"]; if(!item_section_renderer_json.isObject()) continue; - parse_item_section_renderer(item_section_renderer_json, continuation_token, added_videos, result_items); + parse_item_section_renderer(item_section_renderer_json, added_videos, result_items); } } - // The continuation data can also contain continuation, but we ignore that for now. Only get the first continuation data - if(!continuation_token.empty()) - search_suggestions_get_continuation(url, continuation_token, result_items); - return SearchResult::OK; } + PluginResult YoutubeSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) { + (void)str; + while(current_page < page) { + PluginResult plugin_result = search_get_continuation(search_url, continuation_token, result_items); + if(plugin_result != PluginResult::OK) return plugin_result; + ++current_page; + } + return PluginResult::OK; + } + PluginResult YoutubeSearchPage::submit(const std::string &title, const std::string &url, std::vector &result_tabs) { (void)title; (void)url; @@ -272,8 +280,8 @@ namespace QuickMedia { return PluginResult::OK; } - void YoutubeSearchPage::search_suggestions_get_continuation(const std::string &url, const std::string &continuation_token, BodyItems &result_items) { - std::string next_url = url + "&pbj=1&ctoken=" + continuation_token; + PluginResult YoutubeSearchPage::search_get_continuation(const std::string &url, const std::string ¤t_continuation_token, BodyItems &result_items) { + std::string next_url = url + "&pbj=1&ctoken=" + current_continuation_token; std::vector additional_args = { { "-H", "x-spf-referer: " + url }, @@ -288,13 +296,13 @@ namespace QuickMedia { Json::Value json_root; DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); - if(result != DownloadResult::OK) return; + if(result != DownloadResult::OK) return download_result_to_plugin_result(result); if(!json_root.isArray()) - return; + return PluginResult::ERR; - std::string next_continuation_token; std::unordered_set added_videos; + std::string new_continuation_token; for(const Json::Value &json_item : json_root) { if(!json_item.isObject()) @@ -304,17 +312,44 @@ namespace QuickMedia { if(!response_json.isObject()) continue; - const Json::Value &continuation_contents_json = response_json["continuationContents"]; - if(!continuation_contents_json.isObject()) + const Json::Value &on_response_received_commands_json = response_json["onResponseReceivedCommands"]; + if(!on_response_received_commands_json.isArray()) continue; - const Json::Value &item_section_continuation_json = continuation_contents_json["itemSectionContinuation"]; - if(!item_section_continuation_json.isObject()) - continue; + for(const Json::Value &response_received_command : on_response_received_commands_json) { + if(!response_received_command.isObject()) + continue; - // Note: item_section_continuation json object is compatible with item_section_renderer json object - parse_item_section_renderer(item_section_continuation_json, next_continuation_token, added_videos, result_items); + const Json::Value &append_continuation_items_action_json = response_received_command["appendContinuationItemsAction"]; + if(!append_continuation_items_action_json.isObject()) + continue; + + const Json::Value &continuation_items_json = append_continuation_items_action_json["continuationItems"]; + if(!continuation_items_json.isArray()) + continue; + + for(const Json::Value &continuation_item : continuation_items_json) { + if(!continuation_item.isObject()) + continue; + + if(new_continuation_token.empty()) { + // Note: item_section_renderer is compatible with continuation_item + new_continuation_token = item_section_renderer_get_continuation_token(continuation_item); + } + + const Json::Value &item_section_renderer_json = continuation_item["itemSectionRenderer"]; + if(!item_section_renderer_json.isObject()) + continue; + + parse_item_section_renderer(item_section_renderer_json, added_videos, result_items); + } + } } + + if(new_continuation_token.empty()) + continuation_token = std::move(new_continuation_token); + + return PluginResult::OK; } // TODO: Make this faster by using string search instead of parsing html. -- cgit v1.2.3