Youtube: fetch next page when reaching bottom of search (also fixes continuation)

3 files changed, 85 insertions, 44 deletions

diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp
index 0922b9d..007f398 100644
--- a/plugins/Youtube.hpp
+++ b/plugins/Youtube.hpp
@@ -9,9 +9,14 @@ namespace QuickMedia {
         const char* get_title() const override { return "All"; }
         bool search_is_filter() override { return false; }
         SearchResult search(const std::string &str, BodyItems &result_items) override;
+        PluginResult get_page(const std::string &str, int page, BodyItems &result_items) override;
         PluginResult submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) override;
     private:
-        void search_suggestions_get_continuation(const std::string &url, const std::string &continuation_token, BodyItems &result_items);
+        PluginResult search_get_continuation(const std::string &url, const std::string &continuation_token, BodyItems &result_items);
+    private:
+        std::string search_url;
+        std::string continuation_token;
+        int current_page = 0;
     };
 
     class YoutubeVideoPage : public Page {
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 3b059d9..4c41e28 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -1231,6 +1231,7 @@ namespace QuickMedia {
                         BodyItems result_items = associated_data.search_future.get();
                         tabs[i].body->items = std::move(result_items);
                         tabs[i].body->select_first_item();
+                        associated_data.fetched_page = 0;
                         if(tabs[i].body->items.empty())
                             associated_data.search_result_text.setString("No results found");
                         else
@@ -1656,7 +1657,7 @@ namespace QuickMedia {
                     } else if(event.key.code == sf::Keyboard::R && event.key.control) {
                         related_media_window_visible = false;
                         related_media_window->setVisible(related_media_window_visible);
-                        related_media_body->clear_thumbnails();
+                        related_media_body->clear_cache();
                     } else if(event.key.code == sf::Keyboard::F && event.key.control) {
                         window_set_fullscreen(disp, window.getSystemHandle(), WindowFullscreenState::TOGGLE);
                     } else if(event.key.code == sf::Keyboard::Enter) {
@@ -3231,7 +3232,7 @@ namespace QuickMedia {
                     } else if(event.key.code == sf::Keyboard::Escape) {
                         current_page = PageType::EXIT;
                     } else if(event.key.code == sf::Keyboard::Left && synced) {
-                        tabs[selected_tab].body->clear_thumbnails();
+                        tabs[selected_tab].body->clear_cache();
                         selected_tab = std::max(0, selected_tab - 1);
                         read_marker_timer.restart();
                         redraw = true;
@@ -3241,7 +3242,7 @@ namespace QuickMedia {
                             typing_futures.push_back(std::async(typing_async_func, false, current_room_id));
                         }
                     } else if(event.key.code == sf::Keyboard::Right && synced) {
-                        tabs[selected_tab].body->clear_thumbnails();
+                        tabs[selected_tab].body->clear_cache();
                         selected_tab = std::min((int)tabs.size() - 1, selected_tab + 1);
                         read_marker_timer.restart();
                         redraw = true;
@@ -3458,7 +3459,7 @@ namespace QuickMedia {
                 case PageType::CHAT_LOGIN: {
                     new_page = PageType::CHAT;
                     matrix->logout();
-                    tabs[MESSAGES_TAB_INDEX].body->clear_thumbnails();
+                    tabs[MESSAGES_TAB_INDEX].body->clear_cache();
                     // TODO: Instead of doing this, exit this current function and navigate to chat login page instead.
                     // This doesn't currently work because at the end of this function there are futures that need to wait
                     // and one of them is /sync, which has a timeout of 30 seconds. That timeout has to be killed somehow.
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index bff0592..1711e41 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -114,29 +114,26 @@ namespace QuickMedia {
 
     // Returns empty string if continuation token can't be found
     static std::string item_section_renderer_get_continuation_token(const Json::Value &item_section_renderer_json) {
-        const Json::Value &continuations_json = item_section_renderer_json["continuations"];
-        if(!continuations_json.isArray() || continuations_json.empty())
+        const Json::Value &continuation_item_renderer_json = item_section_renderer_json["continuationItemRenderer"];
+        if(!continuation_item_renderer_json.isObject())
             return "";
-        
-        const Json::Value &first_continuation_json = continuations_json[0];
-        if(!first_continuation_json.isObject())
+
+        const Json::Value &continuation_endpoint_json = continuation_item_renderer_json["continuationEndpoint"];
+        if(!continuation_endpoint_json.isObject())
             return "";
-        
-        const Json::Value &next_continuation_data_json = first_continuation_json["nextContinuationData"];
-        if(!next_continuation_data_json.isObject())
+
+        const Json::Value &continuation_command_json = continuation_endpoint_json["continuationCommand"];
+        if(!continuation_command_json.isObject())
             return "";
-        
-        const Json::Value &continuation_json = next_continuation_data_json["continuation"];
-        if(!continuation_json.isString())
+
+        const Json::Value &token_json = continuation_command_json["token"];
+        if(!token_json.isString())
             return "";
         
-        return continuation_json.asString();
+        return token_json.asString();
     }
 
-    static void parse_item_section_renderer(const Json::Value &item_section_renderer_json, std::string &continuation_token, std::unordered_set<std::string> &added_videos, BodyItems &result_items) {
-        if(continuation_token.empty())
-            continuation_token = item_section_renderer_get_continuation_token(item_section_renderer_json);
-        
+    static void parse_item_section_renderer(const Json::Value &item_section_renderer_json, std::unordered_set<std::string> &added_videos, BodyItems &result_items) {
         const Json::Value &item_contents_json = item_section_renderer_json["contents"];
         if(!item_contents_json.isArray())
             return;
@@ -195,27 +192,29 @@ namespace QuickMedia {
     }
 
     SearchResult YoutubeSearchPage::search(const std::string &str, BodyItems &result_items) {
-        std::string url = "https://youtube.com/results?search_query=";
-        url += url_param_encode(str);
+        continuation_token.clear();
+        current_page = 0;
+
+        search_url = "https://youtube.com/results?search_query=";
+        search_url += url_param_encode(str);
 
         std::vector<CommandArg> additional_args = {
-            { "-H", "x-spf-referer: " + url },
+            { "-H", "x-spf-referer: " + search_url },
             { "-H", "x-youtube-client-name: 1" },
             { "-H", "x-youtube-client-version: 2.20200626.03.00" },
-            { "-H", "referer: " + url }
+            { "-H", "referer: " + search_url }
         };
 
         //std::vector<CommandArg> cookies = get_cookies();
         //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         Json::Value json_root;
-        DownloadResult result = download_json(json_root, url + "&pbj=1", std::move(additional_args), true);
+        DownloadResult result = download_json(json_root, search_url + "&pbj=1", std::move(additional_args), true);
         if(result != DownloadResult::OK) return download_result_to_search_result(result);
 
         if(!json_root.isArray())
             return SearchResult::ERR;
 
-        std::string continuation_token;
         std::unordered_set<std::string> added_videos; /* The input contains duplicates, filter them out! */
 
         for(const Json::Value &json_item : json_root) {
@@ -249,22 +248,31 @@ namespace QuickMedia {
             for(const Json::Value &item_json : contents2_json) {
                 if(!item_json.isObject())
                     continue;
+
+                if(continuation_token.empty())
+                    continuation_token = item_section_renderer_get_continuation_token(item_json);
                 
                 const Json::Value &item_section_renderer_json = item_json["itemSectionRenderer"];
                 if(!item_section_renderer_json.isObject())
                     continue;
 
-                parse_item_section_renderer(item_section_renderer_json, continuation_token, added_videos, result_items);
+                parse_item_section_renderer(item_section_renderer_json, added_videos, result_items);
             }
         }
 
-        // The continuation data can also contain continuation, but we ignore that for now. Only get the first continuation data
-        if(!continuation_token.empty())
-            search_suggestions_get_continuation(url, continuation_token, result_items);
-
         return SearchResult::OK;
     }
 
+    PluginResult YoutubeSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) {
+        (void)str;
+        while(current_page < page) {
+            PluginResult plugin_result = search_get_continuation(search_url, continuation_token, result_items);
+            if(plugin_result != PluginResult::OK) return plugin_result;
+            ++current_page;
+        }
+        return PluginResult::OK;
+    }
+
     PluginResult YoutubeSearchPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
         (void)title;
         (void)url;
@@ -272,8 +280,8 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    void YoutubeSearchPage::search_suggestions_get_continuation(const std::string &url, const std::string &continuation_token, BodyItems &result_items) {
-        std::string next_url = url + "&pbj=1&ctoken=" + continuation_token;
+    PluginResult YoutubeSearchPage::search_get_continuation(const std::string &url, const std::string &current_continuation_token, BodyItems &result_items) {
+        std::string next_url = url + "&pbj=1&ctoken=" + current_continuation_token;
 
         std::vector<CommandArg> additional_args = {
             { "-H", "x-spf-referer: " + url },
@@ -288,13 +296,13 @@ namespace QuickMedia {
 
         Json::Value json_root;
         DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
-        if(result != DownloadResult::OK) return;
+        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
 
         if(!json_root.isArray())
-            return;
+            return PluginResult::ERR;
 
-        std::string next_continuation_token;
         std::unordered_set<std::string> added_videos;
+        std::string new_continuation_token;
 
         for(const Json::Value &json_item : json_root) {
             if(!json_item.isObject())
@@ -304,17 +312,44 @@ namespace QuickMedia {
             if(!response_json.isObject())
                 continue;
 
-            const Json::Value &continuation_contents_json = response_json["continuationContents"];
-            if(!continuation_contents_json.isObject())
+            const Json::Value &on_response_received_commands_json = response_json["onResponseReceivedCommands"];
+            if(!on_response_received_commands_json.isArray())
                 continue;
 
-            const Json::Value &item_section_continuation_json = continuation_contents_json["itemSectionContinuation"];
-            if(!item_section_continuation_json.isObject())
-                continue;
+            for(const Json::Value &response_received_command : on_response_received_commands_json) {
+                if(!response_received_command.isObject())
+                    continue;
 
-            // Note: item_section_continuation json object is compatible with item_section_renderer json object
-            parse_item_section_renderer(item_section_continuation_json, next_continuation_token, added_videos, result_items);
+                const Json::Value &append_continuation_items_action_json = response_received_command["appendContinuationItemsAction"];
+                if(!append_continuation_items_action_json.isObject())
+                    continue;
+
+                const Json::Value &continuation_items_json = append_continuation_items_action_json["continuationItems"];
+                if(!continuation_items_json.isArray())
+                    continue;
+
+                for(const Json::Value &continuation_item : continuation_items_json) {
+                    if(!continuation_item.isObject())
+                        continue;
+
+                    if(new_continuation_token.empty()) {
+                        // Note: item_section_renderer is compatible with continuation_item
+                        new_continuation_token = item_section_renderer_get_continuation_token(continuation_item);
+                    }
+
+                    const Json::Value &item_section_renderer_json = continuation_item["itemSectionRenderer"];
+                    if(!item_section_renderer_json.isObject())
+                        continue;
+
+                    parse_item_section_renderer(item_section_renderer_json, added_videos, result_items);
+                }
+            }
         }
+
+        if(new_continuation_token.empty())
+            continuation_token = std::move(new_continuation_token);
+
+        return PluginResult::OK;
     }
 
     // TODO: Make this faster by using string search instead of parsing html.