Youtube: fix comments by replacing removed comments ajax api with next api

2 files changed, 210 insertions, 144 deletions

diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp
index 1b17c7b..2958c44 100644
--- a/plugins/Youtube.hpp
+++ b/plugins/Youtube.hpp
@@ -49,27 +49,27 @@ namespace QuickMedia {
 
     class YoutubeCommentsPage : public LazyFetchPage {
     public:
-        YoutubeCommentsPage(Program *program, const std::string &xsrf_token, const std::string &continuation_token) : LazyFetchPage(program), xsrf_token(xsrf_token), continuation_token(continuation_token) {}
+        YoutubeCommentsPage(Program *program, const std::string &video_url, const std::string &continuation_token) : LazyFetchPage(program), video_url(video_url), continuation_token(continuation_token) {}
         const char* get_title() const override { return "Comments"; }
         PluginResult get_page(const std::string &str, int page, BodyItems &result_items) override;
         PluginResult submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) override;
         PluginResult lazy_fetch(BodyItems &result_items) override;
     private:
         int current_page = 0;
-        std::string xsrf_token;
+        std::string video_url;
         std::string continuation_token;
     };
 
     class YoutubeCommentRepliesPage : public LazyFetchPage {
     public:
-        YoutubeCommentRepliesPage(Program *program, const std::string &xsrf_token, const std::string &continuation_token) : LazyFetchPage(program), xsrf_token(xsrf_token), continuation_token(continuation_token) {}
+        YoutubeCommentRepliesPage(Program *program, const std::string &video_url, const std::string &continuation_token) : LazyFetchPage(program), video_url(video_url), continuation_token(continuation_token) {}
         const char* get_title() const override { return "Comment replies"; }
         PluginResult get_page(const std::string &str, int page, BodyItems &result_items) override;
         PluginResult submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) override;
         PluginResult lazy_fetch(BodyItems &result_items) override;
     private:
         int current_page = 0;
-        std::string xsrf_token;
+        std::string video_url;
         std::string continuation_token;
     };
 
@@ -155,7 +155,6 @@ namespace QuickMedia {
         void parse_formats(const Json::Value &streaming_data_json);
     private:
         std::string timestamp;
-        std::string xsrf_token;
         std::string comments_continuation_token;
         std::string livestream_url;
         std::vector<YoutubeVideoFormat> video_formats;
@@ -165,4 +164,4 @@ namespace QuickMedia {
         std::string watchtime_url;
         std::string tracking_url;
     };
-}
\ No newline at end of file
+}
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index 13655c7..eb95c05 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -687,21 +687,18 @@ R"END(
         if(!item_section_renderer.isObject())
             return "";
 
-        const Json::Value &continuations_json = item_section_renderer["continuations"];
-        if(!continuations_json.isArray())
+        const Json::Value &contents_json = item_section_renderer["contents"];
+        if(!contents_json.isArray())
             return "";
 
-        for(const Json::Value &json_item : continuations_json) {
+        std::string continuation_token;
+        for(const Json::Value &json_item : contents_json) {
             if(!json_item.isObject())
                 continue;
 
-            const Json::Value &next_continuation_data_json = json_item["nextContinuationData"];
-            if(!next_continuation_data_json.isObject())
-                continue;
-
-            const Json::Value &continuation_json = next_continuation_data_json["continuation"];
-            if(continuation_json.isString())
-                return continuation_json.asString();
+            continuation_token = item_section_renderer_get_continuation_token(json_item);
+            if(!continuation_token.empty())
+                return continuation_token;
         }
 
         return "";
@@ -797,9 +794,7 @@ R"END(
             }
         }
 
-        if(!new_continuation_token.empty())
-            continuation_token = std::move(new_continuation_token);
-
+        continuation_token = std::move(new_continuation_token);
         return body_items;
     }
 
@@ -905,6 +900,9 @@ R"END(
     }
 
     PluginResult YoutubeSearchPage::search_get_continuation(const std::string &url, const std::string &current_continuation_token, BodyItems &result_items) {
+        if(current_continuation_token.empty())
+            return PluginResult::OK;
+
         std::string next_url = url + "&pbj=1&gl=US&hl=en&ctoken=" + current_continuation_token;
 
         std::vector<CommandArg> additional_args = {
@@ -968,9 +966,7 @@ R"END(
             }
         }
 
-        if(!new_continuation_token.empty())
-            continuation_token = std::move(new_continuation_token);
-
+        continuation_token = std::move(new_continuation_token);
         return PluginResult::OK;
     }
 
@@ -986,10 +982,34 @@ R"END(
     PluginResult YoutubeCommentsPage::submit(const std::string&, const std::string &url, std::vector<Tab> &result_tabs) {
         if(url.empty())
             return PluginResult::OK;
-        result_tabs.push_back(Tab{create_body(), std::make_unique<YoutubeCommentRepliesPage>(program, xsrf_token, url), nullptr});
+        result_tabs.push_back(Tab{create_body(), std::make_unique<YoutubeCommentRepliesPage>(program, video_url, url), nullptr});
         return PluginResult::OK;
     }
 
+    static std::string item_section_continuation_get_continuation_token(const Json::Value &item_section_continuation_json) {
+        if(!item_section_continuation_json.isObject())
+            return "";
+
+        const Json::Value &continuations_json = item_section_continuation_json["continuations"];
+        if(!continuations_json.isArray())
+            return "";
+
+        for(const Json::Value &json_item : continuations_json) {
+            if(!json_item.isObject())
+                continue;
+            
+            const Json::Value &next_continuation_data_json = json_item["nextContinuationData"];
+            if(!next_continuation_data_json.isObject())
+                continue;
+
+            const Json::Value &continuation_json = next_continuation_data_json["continuation"];
+            if(continuation_json.isString())
+                return continuation_json.asString();
+        }
+
+        return "";
+    }
+
     static std::string comment_thread_renderer_get_replies_continuation(const Json::Value &comment_thread_renderer_json) {
         if(!comment_thread_renderer_json.isObject())
             return "";
@@ -998,12 +1018,12 @@ R"END(
         if(!replies_json.isObject())
             return "";
 
-        const Json::Value &comment_replies_renderer = replies_json["commentRepliesRenderer"];
-        if(!comment_replies_renderer.isObject())
-            return "";
+        const Json::Value &comment_replies_renderer_json = replies_json["commentRepliesRenderer"];
+        std::string continuation = item_section_renderer_get_continuation(comment_replies_renderer_json);
+        if(!continuation.empty())
+            return continuation;
 
-        // item_section_renderer_get_continuation is compatible with commentRepliesRenderer
-        return item_section_renderer_get_continuation(comment_replies_renderer);
+        return item_section_continuation_get_continuation_token(comment_replies_renderer_json);
     }
 
     // Returns empty string if comment is not hearted
@@ -1092,152 +1112,208 @@ R"END(
         return body_item;
     }
 
-    PluginResult YoutubeCommentsPage::lazy_fetch(BodyItems &result_items) {
-        if(continuation_token.empty())
-            return PluginResult::OK;
-
-        std::string next_url = "https://www.youtube.com/comment_service_ajax?action_get_comments=1&pbj=1&ctoken=";
-        next_url += url_param_encode(continuation_token);
-        //next_url += "&continuation=";
-        //next_url += url_param_encode(comments_continuation_token);
-        next_url += "&type=next&gl=US&hl=en";
+    static std::string continuation_item_renderer_get_continuation_token(const Json::Value &continuation_item_renderer_json) {
+        if(!continuation_item_renderer_json.isObject())
+            return "";
 
-        std::vector<CommandArg> additional_args = {
-            { "-H", "x-youtube-client-name: 1" },
-            { "-H", youtube_client_version },
-            { "-F", "session_token=" + xsrf_token }
-        };
+        const Json::Value &button_json = continuation_item_renderer_json["button"];
+        if(!button_json.isObject())
+            return "";
 
-        std::vector<CommandArg> cookies = get_cookies();
-        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+        const Json::Value &button_renderer_json = button_json["buttonRenderer"];
+        if(!button_renderer_json.isObject())
+            return "";
 
-        Json::Value json_root;
-        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
-        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
+        const Json::Value &command_json = button_renderer_json["command"];
+        if(!command_json.isObject())
+            return "";
 
-        if(!json_root.isObject())
-            return PluginResult::ERR;
+        const Json::Value &continuation_command_json = command_json["continuationCommand"];
+        if(!continuation_command_json.isObject())
+            return "";
 
-        const Json::Value &xsrf_token_json = json_root["xsrf_token"];
-        if(xsrf_token_json.isString())
-            xsrf_token = xsrf_token_json.asString();
+        const Json::Value &token_json = continuation_command_json["token"];
+        if(!token_json.isString())
+            return "";
 
-        const Json::Value &response_json = json_root["response"];
-        if(!response_json.isObject())
-            return PluginResult::ERR;
+        return token_json.asString();
+    }
 
-        const Json::Value &continuation_contents_json = response_json["continuationContents"];
-        if(!continuation_contents_json.isObject())
+    static PluginResult fetch_comments_received_endpoints(const Json::Value &json_root, BodyItems &result_items, std::string &continuation_token) {
+        const Json::Value &on_response_received_endpoints_json = json_root["onResponseReceivedEndpoints"];
+        if(!on_response_received_endpoints_json.isArray())
             return PluginResult::ERR;
 
-        const Json::Value &item_section_continuation_json = continuation_contents_json["itemSectionContinuation"];
-        if(!item_section_continuation_json.isObject())
-            return PluginResult::ERR;
+        std::string new_continuation_token;
+        for(const Json::Value &json_item : on_response_received_endpoints_json) {
+            if(!json_item.isObject())
+                continue;
 
-        const Json::Value &contents_json = item_section_continuation_json["contents"];
-        if(contents_json.isArray()) {
-            for(const Json::Value &json_item : contents_json) {
-                if(!json_item.isObject())
+            const Json::Value *append_continuation_items_action_json = &json_item["reloadContinuationItemsCommand"];
+            if(!append_continuation_items_action_json->isObject()) {
+                append_continuation_items_action_json = &json_item["appendContinuationItemsAction"];
+                if(!append_continuation_items_action_json->isObject())
                     continue;
+            }
 
-                const Json::Value &comment_thread_renderer = json_item["commentThreadRenderer"];
-                if(!comment_thread_renderer.isObject())
-                    continue;
+            const Json::Value &continuation_items_json = (*append_continuation_items_action_json)["continuationItems"];
+            if(!continuation_items_json.isArray())
+                continue;
 
-                const Json::Value &comment_json = comment_thread_renderer["comment"];
-                if(!comment_json.isObject())
+            for(const Json::Value &continuation_item_json : continuation_items_json) {
+                if(!continuation_item_json.isObject())
                     continue;
 
-                auto body_item = comment_renderer_to_body_item(comment_json["commentRenderer"]);
-                if(body_item) {
-                    body_item->url = comment_thread_renderer_get_replies_continuation(comment_thread_renderer);
+                if(new_continuation_token.empty())
+                    new_continuation_token = item_section_renderer_get_continuation_token(continuation_item_json);
+
+                const Json::Value &comment_thread_renderer_json = continuation_item_json["commentThreadRenderer"];
+                if(comment_thread_renderer_json.isObject()) {
+                    const Json::Value &comment_json = comment_thread_renderer_json["comment"];
+                    if(!comment_json.isObject())
+                        continue;
+
+                    auto body_item = comment_renderer_to_body_item(comment_json["commentRenderer"]);
+                    if(!body_item)
+                        continue;
+
+                    body_item->url = comment_thread_renderer_get_replies_continuation(comment_thread_renderer_json);
                     result_items.push_back(std::move(body_item));
+                } else {
+                    auto body_item = comment_renderer_to_body_item(continuation_item_json["commentRenderer"]);
+                    if(body_item)
+                        result_items.push_back(std::move(body_item));
+
+                    if(new_continuation_token.empty())
+                        new_continuation_token = continuation_item_renderer_get_continuation_token(continuation_item_json["continuationItemRenderer"]);
                 }
             }
         }
 
-        continuation_token = item_section_renderer_get_continuation(item_section_continuation_json);
-
+        continuation_token = std::move(new_continuation_token);
         return PluginResult::OK;
     }
 
-    PluginResult YoutubeCommentRepliesPage::get_page(const std::string&, int page, BodyItems &result_items) {
-        while(current_page < page) {
-            PluginResult plugin_result = lazy_fetch(result_items);
-            if(plugin_result != PluginResult::OK) return plugin_result;
-            ++current_page;
+    static PluginResult fetch_comments_continuation_contents(const Json::Value &json_root, BodyItems &result_items, std::string &continuation_token) {
+        const Json::Value &continuation_contents_json = json_root["continuationContents"];
+        if(!continuation_contents_json.isObject())
+            return PluginResult::ERR;
+
+        const Json::Value *item_section_continuation_json = &continuation_contents_json["itemSectionContinuation"];
+        if(!item_section_continuation_json->isObject()) {
+            item_section_continuation_json = &continuation_contents_json["commentRepliesContinuation"];
+            if(!item_section_continuation_json->isObject())
+                return PluginResult::ERR;
         }
-        return PluginResult::OK;
-    }
 
-    PluginResult YoutubeCommentRepliesPage::submit(const std::string&, const std::string&, std::vector<Tab>&) {
+        const Json::Value &contents_json = (*item_section_continuation_json)["contents"];
+        if(!contents_json.isArray())
+            return PluginResult::ERR;
+
+        std::string new_continuation_token;
+        for(const Json::Value &json_item : contents_json) {
+            if(!json_item.isObject())
+                continue;
+
+            const Json::Value &comment_thread_renderer_json = json_item["commentThreadRenderer"];
+            if(comment_thread_renderer_json.isObject()) {
+                const Json::Value &comment_json = comment_thread_renderer_json["comment"];
+                if(!comment_json.isObject())
+                    continue;
+
+                auto body_item = comment_renderer_to_body_item(comment_json["commentRenderer"]);
+                if(!body_item)
+                    continue;
+
+                body_item->url = comment_thread_renderer_get_replies_continuation(comment_thread_renderer_json);
+                result_items.push_back(std::move(body_item));
+            } else {
+                auto body_item = comment_renderer_to_body_item(json_item["commentRenderer"]);
+                if(body_item)
+                    result_items.push_back(std::move(body_item));
+
+                if(new_continuation_token.empty())
+                    new_continuation_token = continuation_item_renderer_get_continuation_token(json_item["continuationItemRenderer"]);
+            }
+        }
+
+        if(new_continuation_token.empty())
+            new_continuation_token = item_section_continuation_get_continuation_token(*item_section_continuation_json);
+
+        continuation_token = std::move(new_continuation_token);
         return PluginResult::OK;
     }
 
-    PluginResult YoutubeCommentRepliesPage::lazy_fetch(BodyItems &result_items) {
+    static PluginResult fetch_comments(Page *page, const std::string &video_url, std::string &continuation_token, BodyItems &result_items) {
         if(continuation_token.empty())
             return PluginResult::OK;
 
-        std::string next_url = "https://www.youtube.com/comment_service_ajax?action_get_comment_replies=1&pbj=1&ctoken=";
-        next_url += url_param_encode(continuation_token);
-        //next_url += "&continuation=";
-        //next_url += url_param_encode(comments_continuation_token);
-        next_url += "&type=next&gl=US&hl=en";
+        std::vector<CommandArg> cookies = get_cookies();
+        std::string next_url = "https://www.youtube.com/youtubei/v1/next?key=" + url_param_encode(api_key) + "&gl=US&hl=en";
+
+        Json::Value request_json(Json::objectValue);
+        Json::Value context_json(Json::objectValue);
+        Json::Value client_json(Json::objectValue);
+        client_json["hl"] = "en";
+        client_json["gl"] = "US";
+        client_json["deviceMake"] = "";
+        client_json["deviceModel"] = "";
+        client_json["userAgent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36";
+        client_json["clientName"] = "WEB";
+        client_json["clientVersion"] = "2.20210701.07.00";
+        client_json["osName"] = "X11";
+        client_json["osVersion"] = "";
+        client_json["originalUrl"] = video_url;
+        context_json["client"] = std::move(client_json);
+        request_json["context"] = std::move(context_json);
+        request_json["continuation"] = continuation_token;
+
+        Json::StreamWriterBuilder json_builder;
+        json_builder["commentStyle"] = "None";
+        json_builder["indentation"] = "";
 
         std::vector<CommandArg> additional_args = {
+            { "-H", "content-type: application/json" },
             { "-H", "x-youtube-client-name: 1" },
             { "-H", youtube_client_version },
-            { "-F", "session_token=" + xsrf_token }
+            { "--data-raw", Json::writeString(json_builder, request_json) }
         };
 
-        std::vector<CommandArg> cookies = get_cookies();
         additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         Json::Value json_root;
-        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
+        DownloadResult result = page->download_json(json_root, next_url, std::move(additional_args), true);
         if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
 
-        if(!json_root.isArray())
+        if(!json_root.isObject())
             return PluginResult::ERR;
+        
+        PluginResult res = fetch_comments_received_endpoints(json_root, result_items, continuation_token);
+        if(res == PluginResult::OK)
+            return res;
 
-        for(const Json::Value &json_item : json_root) {
-            if(!json_item.isObject())
-                continue;
-
-            const Json::Value &xsrf_token_json = json_item["xsrf_token"];
-            if(xsrf_token_json.isString())
-                xsrf_token = xsrf_token_json.asString();
-
-            const Json::Value &response_json = json_item["response"];
-            if(!response_json.isObject())
-                continue;
-
-            const Json::Value &continuation_contents_json = response_json["continuationContents"];
-            if(!continuation_contents_json.isObject())
-                continue;
-
-            const Json::Value &comment_replies_continuation_json = continuation_contents_json["commentRepliesContinuation"];
-            if(!comment_replies_continuation_json.isObject())
-                continue;
-
-            const Json::Value &contents_json = comment_replies_continuation_json["contents"];
-            if(contents_json.isArray()) {
-                for(const Json::Value &content_item_json : contents_json) {
-                    if(!content_item_json.isObject())
-                        continue;
+        return fetch_comments_continuation_contents(json_root, result_items, continuation_token);
+    }
 
-                    auto body_item = comment_renderer_to_body_item(content_item_json["commentRenderer"]);
-                    if(body_item)
-                        result_items.push_back(std::move(body_item));
-                }
-            }
+    PluginResult YoutubeCommentsPage::lazy_fetch(BodyItems &result_items) {
+        return fetch_comments(this, video_url, continuation_token, result_items);
+    }
 
-            // item_section_renderer_get_continuation is compatible with commentRepliesContinuation
-            continuation_token = item_section_renderer_get_continuation(comment_replies_continuation_json);
-            return PluginResult::OK;
+    PluginResult YoutubeCommentRepliesPage::get_page(const std::string&, int page, BodyItems &result_items) {
+        while(current_page < page) {
+            PluginResult plugin_result = lazy_fetch(result_items);
+            if(plugin_result != PluginResult::OK) return plugin_result;
+            ++current_page;
         }
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubeCommentRepliesPage::submit(const std::string&, const std::string&, std::vector<Tab>&) {
+        return PluginResult::OK;
+    }
 
-        return PluginResult::ERR;
+    PluginResult YoutubeCommentRepliesPage::lazy_fetch(BodyItems &result_items) {
+        return fetch_comments(this, video_url, continuation_token, result_items);
     }
 
     static std::string channel_url_extract_id(const std::string &channel_url) {
@@ -1427,9 +1503,7 @@ R"END(
             }
         }
 
-        if(!new_continuation_token.empty())
-            continuation_token = std::move(new_continuation_token);
-
+        continuation_token = std::move(new_continuation_token);
         return PluginResult::OK;
     }
 
@@ -1708,6 +1782,9 @@ R"END(
     }
 
     PluginResult YoutubeRecommendedPage::search_get_continuation(const std::string &current_continuation_token, BodyItems &result_items) {
+        if(current_continuation_token.empty())
+            return PluginResult::OK;
+
         std::string next_url = "https://www.youtube.com/?pbj=1&gl=US&hl=en&ctoken=" + current_continuation_token;
 
         std::vector<CommandArg> additional_args = {
@@ -1779,9 +1856,7 @@ R"END(
             }
         }
 
-        if(!new_continuation_token.empty())
-            continuation_token = std::move(new_continuation_token);
-
+        continuation_token = std::move(new_continuation_token);
         return PluginResult::OK;
     }
 
@@ -1877,9 +1952,7 @@ R"END(
             }
         }
 
-        if(!new_continuation_token.empty())
-            continuation_token = std::move(new_continuation_token);
-
+        continuation_token = std::move(new_continuation_token);
         return PluginResult::OK;
     }
 
@@ -1955,7 +2028,6 @@ R"END(
     }
 
     BodyItems YoutubeVideoPage::get_related_media(const std::string &url) {
-        xsrf_token.clear();
         comments_continuation_token.clear();
         BodyItems result_items;
 
@@ -1973,6 +2045,7 @@ R"END(
         std::vector<CommandArg> cookies = get_cookies();
         additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
+        // TODO: Remove this code completely and replace with existing player? api
         Json::Value json_root;
         DownloadResult download_result = download_json(json_root, "https://www.youtube.com/watch?v=" + video_id + "&pbj=1&gl=US&hl=en", additional_args, true);
         if(download_result != DownloadResult::OK) return result_items;
@@ -1986,12 +2059,6 @@ R"END(
             if(!json_item.isObject())
                 continue;
 
-            if(xsrf_token.empty()) {
-                const Json::Value &xsrf_token_json = json_item["xsrf_token"];
-                if(xsrf_token_json.isString())
-                    xsrf_token = xsrf_token_json.asString();
-            }
-
             const Json::Value &response_json = json_item["response"];
             if(!response_json.isObject())
                 continue;
@@ -2055,7 +2122,7 @@ R"END(
     }
 
     std::unique_ptr<Page> YoutubeVideoPage::create_comments_page(Program *program) {
-        return std::make_unique<YoutubeCommentsPage>(program, xsrf_token, comments_continuation_token);
+        return std::make_unique<YoutubeCommentsPage>(program, url, comments_continuation_token);
     }
 
     std::unique_ptr<RelatedVideosPage> YoutubeVideoPage::create_related_videos_page(Program *program) {
@@ -2494,4 +2561,4 @@ R"END(
         const Json::Value &adaptive_formats_json = streaming_data_json["adaptiveFormats"];
         parse_format(adaptive_formats_json, true);
     }
-}
\ No newline at end of file
+}