Fix youtube comments/replies sometimes missing after youtube update

1 files changed, 176 insertions, 6 deletions

diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index 683845b..fef5fce 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -1586,6 +1586,88 @@ namespace QuickMedia {
         return body_item;
     }
 
+    static std::shared_ptr<BodyItem> comment_entity_payload_to_body_item(const Json::Value &comment_entity_payload_json, std::string &heart_active_tooltip_text) {
+        if(!comment_entity_payload_json.isObject())
+            return nullptr;
+
+        const Json::Value &key_json = comment_entity_payload_json["key"];
+        if(!key_json.isString())
+            return nullptr;
+
+        const Json::Value &properties_json = comment_entity_payload_json["properties"];
+        if(!properties_json.isObject())
+            return nullptr;
+
+        const Json::Value &author_json = comment_entity_payload_json["author"];
+        if(!author_json.isObject())
+            return nullptr;
+
+        const Json::Value &toolbar_json = comment_entity_payload_json["toolbar"];
+        if(!toolbar_json.isObject())
+            return nullptr;
+
+        const Json::Value &content1_json = properties_json["content"];
+        if(!content1_json.isObject())
+            return nullptr;
+
+        const Json::Value &content2_json = content1_json["content"];
+        if(!content2_json.isString())
+            return nullptr;
+
+        const Json::Value &display_name_json = author_json["displayName"];
+        if(!display_name_json.isString())
+            return nullptr;
+
+        std::string author = display_name_json.asString();
+        const Json::Value &published_time_json = properties_json["publishedTime"];
+        if(published_time_json.isString())
+            author += " - " + published_time_json.asString();
+
+        auto body_item = BodyItem::create("");
+        body_item->set_author(std::move(author));
+        body_item->url = key_json.asString();
+
+        const Json::Value &is_creator_json = author_json["isCreator"];
+        if(is_creator_json.isBool() && is_creator_json.asBool())
+            body_item->set_author_color(mgl::Color(150, 255, 150));
+
+        std::string description = content2_json.asString();
+        
+        const Json::Value &avatar_thumbnail_url = author_json["avatarThumbnailUrl"];
+        if(avatar_thumbnail_url.isString()) {
+            body_item->thumbnail_url = avatar_thumbnail_url.asString();
+            body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE;
+            body_item->thumbnail_size.x = 48;
+            body_item->thumbnail_size.y = 48;
+        }
+
+        const Json::Value &like_count_liked_json = toolbar_json["likeCountLiked"];
+        if(!description.empty())
+            description += '\n';
+        description += std::string("👍 ") + (like_count_liked_json.isString() ? like_count_liked_json.asString() : "0");
+
+        const Json::Value &reply_count_json = toolbar_json["replyCount"];
+        if(reply_count_json.isString() && reply_count_json.asCString()[0] != '\0' && reply_count_json.asCString()[0] != '0') {
+            if(!description.empty())
+                description += '\n';
+
+            std::string reply_count_str = reply_count_json.asString();
+            if(reply_count_str == "1")
+                description += "1 reply";
+            else
+                description += std::move(reply_count_str) + " replies";
+        }
+
+        const Json::Value &heart_active_tooltip_json = toolbar_json["heartActiveTooltip"];
+        if(heart_active_tooltip_json.isString()) {
+            heart_active_tooltip_text = heart_active_tooltip_json.asString();
+        }
+
+        body_item->set_description(std::move(description));
+        body_item->userdata = body_item.get();
+        return body_item;
+    }
+
     static std::string continuation_item_renderer_get_continuation_token(const Json::Value &continuation_item_renderer_json) {
         if(!continuation_item_renderer_json.isObject())
             return "";
@@ -1613,7 +1695,27 @@ namespace QuickMedia {
         return token_json.asString();
     }
 
-    static PluginResult fetch_comments_received_endpoints(const Json::Value &json_root, BodyItems &result_items, std::string &continuation_token) {
+    static std::string get_comment_key(const Json::Value &comment_thread_renderer_json) {
+        std::string result;
+        if(!comment_thread_renderer_json.isObject())
+            return result;
+
+        const Json::Value &comment_view_model_json = comment_thread_renderer_json["commentViewModel"];
+        if(!comment_view_model_json.isObject())
+            return result;
+
+        const Json::Value &comment_view_model2_json = comment_view_model_json["commentViewModel"];
+        if(!comment_view_model2_json.isObject())
+            return result;
+
+        const Json::Value &comment_key_json = comment_view_model2_json["commentKey"];
+        if(comment_key_json.isString())
+            result = comment_key_json.asString();
+
+        return result;
+    }
+
+    static PluginResult fetch_comments_received_endpoints(const Json::Value &json_root, BodyItems &result_items, std::string &continuation_token, std::unordered_map<std::string, std::string> &comment_reply_tokens_by_key) {
         const Json::Value &on_response_received_endpoints_json = json_root["onResponseReceivedEndpoints"];
         if(!on_response_received_endpoints_json.isArray())
             return PluginResult::ERR;
@@ -1643,6 +1745,12 @@ namespace QuickMedia {
 
                 const Json::Value &comment_thread_renderer_json = continuation_item_json["commentThreadRenderer"];
                 if(comment_thread_renderer_json.isObject()) {
+                    std::string comment_key = get_comment_key(comment_thread_renderer_json);
+                    if(!comment_key.empty()) {
+                        comment_reply_tokens_by_key[comment_key] = comment_thread_renderer_get_replies_continuation(comment_thread_renderer_json);
+                        continue;
+                    }
+
                     const Json::Value &comment_json = comment_thread_renderer_json["comment"];
                     if(!comment_json.isObject())
                         continue;
@@ -1668,7 +1776,7 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    static PluginResult fetch_comments_continuation_contents(const Json::Value &json_root, BodyItems &result_items, std::string &continuation_token) {
+    static PluginResult fetch_comments_continuation_contents(const Json::Value &json_root, BodyItems &result_items, std::string &continuation_token, std::unordered_map<std::string, std::string> &comment_reply_tokens_by_key) {
         const Json::Value &continuation_contents_json = json_root["continuationContents"];
         if(!continuation_contents_json.isObject())
             return PluginResult::ERR;
@@ -1691,6 +1799,12 @@ namespace QuickMedia {
 
             const Json::Value &comment_thread_renderer_json = json_item["commentThreadRenderer"];
             if(comment_thread_renderer_json.isObject()) {
+                std::string comment_key = get_comment_key(comment_thread_renderer_json);
+                if(!comment_key.empty()) {
+                    comment_reply_tokens_by_key[comment_key] = comment_thread_renderer_get_replies_continuation(comment_thread_renderer_json);
+                    continue;
+                }
+
                 const Json::Value &comment_json = comment_thread_renderer_json["comment"];
                 if(!comment_json.isObject())
                     continue;
@@ -1718,6 +1832,61 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
+    static PluginResult fetch_comments_framework_updates(const Json::Value &json_root, BodyItems &result_items, const std::unordered_map<std::string, std::string> &comment_reply_tokens_by_key) {
+        const Json::Value &framework_updates_json = json_root["frameworkUpdates"];
+        if(!framework_updates_json.isObject())
+            return PluginResult::ERR;
+        
+        const Json::Value &entity_batch_update_json = framework_updates_json["entityBatchUpdate"];
+        if(!entity_batch_update_json.isObject())
+            return PluginResult::ERR;
+
+        const Json::Value &mutations_json = entity_batch_update_json["mutations"];
+        if(!mutations_json.isArray())
+            return PluginResult::ERR;
+
+        std::string heart_active_tooltip_text;
+        for(const Json::Value &item_json : mutations_json) {
+            if(!item_json.isObject())
+                continue;
+
+            const Json::Value &payload_json = item_json["payload"];
+            if(!payload_json.isObject())
+                continue;
+
+            const Json::Value &eng_toolbar_state_ent_payload_json = payload_json["engagementToolbarStateEntityPayload"];
+            if(eng_toolbar_state_ent_payload_json.isObject()) {
+                const Json::Value &heart_state_json = eng_toolbar_state_ent_payload_json["heartState"];
+                if(heart_state_json.isString() && strcmp(heart_state_json.asCString(), "TOOLBAR_HEART_STATE_HEARTED") == 0 && !result_items.empty()) {
+                    std::string description = result_items.back()->get_description();
+                    if(!description.empty())
+                        description += " - ";
+                    description += heart_active_tooltip_text;
+                    result_items.back()->set_description(std::move(description));
+                    continue;
+                }
+            }
+
+            const Json::Value &comment_entity_payload_json = payload_json["commentEntityPayload"];
+            if(!comment_entity_payload_json.isObject())
+                continue;
+
+            auto body_item = comment_entity_payload_to_body_item(comment_entity_payload_json, heart_active_tooltip_text);
+            if(!body_item)
+                continue;
+
+            auto it = comment_reply_tokens_by_key.find(body_item->url);
+            if(it == comment_reply_tokens_by_key.end()) {
+                body_item->url.clear();
+            } else {
+                body_item->url = it->second;
+            }
+            result_items.push_back(std::move(body_item));
+        }
+
+        return PluginResult::OK;
+    }
+
     static PluginResult fetch_comments(Page *page, const std::string &video_url, std::string &continuation_token, BodyItems &result_items) {
         if(continuation_token.empty())
             return PluginResult::OK;
@@ -1762,11 +1931,12 @@ namespace QuickMedia {
         if(!json_root.isObject())
             return PluginResult::ERR;
         
-        PluginResult res = fetch_comments_received_endpoints(json_root, result_items, continuation_token);
-        if(res == PluginResult::OK)
-            return res;
+        std::unordered_map<std::string, std::string> comment_reply_tokens_by_key;
+        fetch_comments_received_endpoints(json_root, result_items, continuation_token, comment_reply_tokens_by_key);
+        fetch_comments_continuation_contents(json_root, result_items, continuation_token, comment_reply_tokens_by_key);
+        fetch_comments_framework_updates(json_root, result_items, comment_reply_tokens_by_key);
 
-        return fetch_comments_continuation_contents(json_root, result_items, continuation_token);
+        return PluginResult::OK;
     }
 
     PluginResult YoutubeCommentsPage::lazy_fetch(BodyItems &result_items) {