Youtube: add youtube comments to ctrl+r

2 files changed, 390 insertions, 24 deletions

diff --git a/src/plugins/Spotify.cpp b/src/plugins/Spotify.cpp
index 3ce640a..14f9831 100644
--- a/src/plugins/Spotify.cpp
+++ b/src/plugins/Spotify.cpp
@@ -167,7 +167,7 @@ namespace QuickMedia {
         if(result != PluginResult::OK)
             return result;
 
-        result_tabs.push_back(Tab{std::move(body), std::move(episode_list_page), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+        result_tabs.push_back(Tab{std::move(body), std::move(episode_list_page), nullptr});
         return result;
     }
 
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index b3c9a60..ced6c45 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -5,6 +5,7 @@
 #include "../../include/Scale.hpp"
 #include <json/writer.h>
 #include <string.h>
+#include <unistd.h>
 
 namespace QuickMedia {
     // This is a common setup of text in the youtube json
@@ -18,7 +19,7 @@ namespace QuickMedia {
 
         const Json::Value &simple_text_json = text_json["simpleText"];
         if(simple_text_json.isString()) {
-            return simple_text_json.asCString();
+            return simple_text_json.asString();
         } else {
             const Json::Value &runs_json = text_json["runs"];
             if(!runs_json.isArray() || runs_json.empty())
@@ -46,8 +47,13 @@ namespace QuickMedia {
         int height;
     };
 
+    enum class ThumbnailSize {
+        SMALLEST,
+        LARGEST
+    };
+
     // TODO: Use this in |parse_common_video_item| when QuickMedia supports webp
-    static std::optional<Thumbnail> yt_json_get_largest_thumbnail(const Json::Value &thumbnail_json) {
+    static std::optional<Thumbnail> yt_json_get_thumbnail(const Json::Value &thumbnail_json, ThumbnailSize thumbnail_size) {
         if(!thumbnail_json.isObject())
             return std::nullopt;
 
@@ -75,11 +81,22 @@ namespace QuickMedia {
             thumbnails.push_back({ url_json.asCString(), width_json.asInt(), height_json.asInt() });
         }
 
-        return *std::max_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) {
-            int size1 = thumbnail1.width * thumbnail1.height;
-            int size2 = thumbnail2.width * thumbnail2.height;
-            return size1 < size2;
-        });
+        switch(thumbnail_size) {
+            case ThumbnailSize::SMALLEST:
+                return *std::min_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) {
+                    int size1 = thumbnail1.width * thumbnail1.height;
+                    int size2 = thumbnail2.width * thumbnail2.height;
+                    return size1 < size2;
+                });
+            case ThumbnailSize::LARGEST:
+                return *std::max_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) {
+                    int size1 = thumbnail1.width * thumbnail1.height;
+                    int size2 = thumbnail2.width * thumbnail2.height;
+                    return size1 < size2;
+                });
+        }
+
+        return std::nullopt;
     }
 
     static std::shared_ptr<BodyItem> parse_common_video_item(const Json::Value &video_item_json, std::unordered_set<std::string> &added_videos) {
@@ -196,7 +213,7 @@ namespace QuickMedia {
         std::optional<std::string> subscribers = yt_json_get_text(channel_renderer_json, "subscriberCountText");
 
         const Json::Value &thumbnail_json = channel_renderer_json["thumbnail"];
-        std::optional<Thumbnail> thumbnail = yt_json_get_largest_thumbnail(thumbnail_json);
+        std::optional<Thumbnail> thumbnail = yt_json_get_thumbnail(thumbnail_json, ThumbnailSize::LARGEST);
 
         auto body_item = BodyItem::create(title.value());
         std::string desc;
@@ -300,6 +317,37 @@ namespace QuickMedia {
         }
     }
 
+    static std::mutex cookies_mutex;
+    static std::string cookies_filepath;
+
+    static void remove_cookies_file_at_exit() {
+        std::lock_guard<std::mutex> lock(cookies_mutex);
+        if(!cookies_filepath.empty())
+            remove(cookies_filepath.c_str());
+    }
+
+    static std::vector<CommandArg> get_cookies() {
+        std::lock_guard<std::mutex> lock(cookies_mutex);
+        if(cookies_filepath.empty()) {
+            char filename[] = "quickmedia.youtube.cookie.XXXXXX";
+            int fd = mkstemp(filename);
+            if(fd == -1)
+                return {};
+            close(fd);
+
+            cookies_filepath = "/tmp/";
+            cookies_filepath += filename;
+            cookies_filepath += ".txt";
+
+            atexit(remove_cookies_file_at_exit);
+        }
+
+        return {
+            CommandArg{ "-b", cookies_filepath },
+            CommandArg{ "-c", cookies_filepath }
+        };
+    }
+
     static std::string remove_index_from_playlist_url(const std::string &url) {
         std::string result = url;
         size_t index = result.rfind("&index=");
@@ -316,6 +364,30 @@ namespace QuickMedia {
         return parse_common_video_item(compact_video_renderer_json, added_videos);
     }
 
+    static std::string item_section_renderer_get_continuation(const Json::Value &item_section_renderer) {
+        if(!item_section_renderer.isObject())
+            return "";
+
+        const Json::Value &continuations_json = item_section_renderer["continuations"];
+        if(!continuations_json.isArray())
+            return "";
+
+        for(const Json::Value &json_item : continuations_json) {
+            if(!json_item.isObject())
+                continue;
+
+            const Json::Value &next_continuation_data_json = json_item["nextContinuationData"];
+            if(!next_continuation_data_json.isObject())
+                continue;
+
+            const Json::Value &continuation_json = next_continuation_data_json["continuation"];
+            if(continuation_json.isString())
+                return continuation_json.asString();
+        }
+
+        return "";
+    }
+
     static BodyItems parse_channel_videos(const Json::Value &json_root, std::string &continuation_token, std::unordered_set<std::string> &added_videos) {
         BodyItems body_items;
         if(!json_root.isArray())
@@ -450,8 +522,8 @@ namespace QuickMedia {
             { "-H", "referer: " + search_url }
         };
 
-        //std::vector<CommandArg> cookies = get_cookies();
-        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         Json::Value json_root;
         DownloadResult result = download_json(json_root, search_url + "&pbj=1", std::move(additional_args), true);
@@ -519,8 +591,8 @@ namespace QuickMedia {
             { "-H", "referer: " + url }
         };
 
-        //std::vector<CommandArg> cookies = get_cookies();
-        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         Json::Value json_root;
         DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
@@ -578,6 +650,263 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
+    PluginResult YoutubeCommentsPage::get_page(const std::string&, int page, BodyItems &result_items) {
+        while(current_page < page) {
+            PluginResult plugin_result = lazy_fetch(result_items);
+            if(plugin_result != PluginResult::OK) return plugin_result;
+            ++current_page;
+        }
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubeCommentsPage::submit(const std::string&, const std::string &url, std::vector<Tab> &result_tabs) {
+        if(url.empty())
+            return PluginResult::OK;
+        result_tabs.push_back(Tab{create_body(), std::make_unique<YoutubeCommentRepliesPage>(program, xsrf_token, url), nullptr});
+        return PluginResult::OK;
+    }
+
+    static std::string comment_thread_renderer_get_replies_continuation(const Json::Value &comment_thread_renderer_json) {
+        if(!comment_thread_renderer_json.isObject())
+            return "";
+
+        const Json::Value &replies_json = comment_thread_renderer_json["replies"];
+        if(!replies_json.isObject())
+            return "";
+
+        const Json::Value &comment_replies_renderer = replies_json["commentRepliesRenderer"];
+        if(!comment_replies_renderer.isObject())
+            return "";
+
+        // item_section_renderer_get_continuation is compatible with commentRepliesRenderer
+        return item_section_renderer_get_continuation(comment_replies_renderer);
+    }
+
+    // Returns empty string if comment is not hearted
+    static std::string comment_renderer_get_hearted_tooltip(const Json::Value &comment_renderer_json) {
+        const Json::Value &action_buttons_json = comment_renderer_json["actionButtons"];
+        if(!action_buttons_json.isObject())
+            return "";
+
+        const Json::Value &comment_action_buttons_renderer_json = action_buttons_json["commentActionButtonsRenderer"];
+        if(!comment_action_buttons_renderer_json.isObject())
+            return "";
+
+        const Json::Value &creator_heart_json = comment_action_buttons_renderer_json["creatorHeart"];
+        if(!creator_heart_json.isObject())
+            return "";
+
+        const Json::Value &creator_heart_renderer_json = creator_heart_json["creatorHeartRenderer"];
+        if(!creator_heart_renderer_json.isObject())
+            return "";
+
+        const Json::Value &hearted_tooltip_json = creator_heart_renderer_json["heartedTooltip"];
+        if(!hearted_tooltip_json.isString())
+            return "";
+
+        return hearted_tooltip_json.asString();
+    }
+
+    static std::shared_ptr<BodyItem> comment_renderer_to_body_item(const Json::Value &comment_renderer_json) {
+        if(!comment_renderer_json.isObject())
+            return nullptr;
+
+        std::optional<std::string> author_text = yt_json_get_text(comment_renderer_json, "authorText");
+        if(!author_text)
+            return nullptr;
+
+        std::string title = author_text.value();
+        std::optional<std::string> published_time_text = yt_json_get_text(comment_renderer_json, "publishedTimeText");
+        if(published_time_text)
+            title += " - " + published_time_text.value();
+
+        auto body_item = BodyItem::create(std::move(title));
+        std::string description;
+
+        const Json::Value &author_is_channel_owner_json = comment_renderer_json["authorIsChannelOwner"];
+        if(author_is_channel_owner_json.isBool() && author_is_channel_owner_json.asBool())
+            body_item->set_title_color(sf::Color(150, 255, 150));
+
+        std::optional<std::string> comment = yt_json_get_text(comment_renderer_json, "contentText");
+        if(comment)
+            description = comment.value();
+        
+        std::optional<Thumbnail> thumbnail = yt_json_get_thumbnail(comment_renderer_json["authorThumbnail"], ThumbnailSize::SMALLEST);
+        if(thumbnail) {
+            body_item->thumbnail_url = thumbnail->url;
+            body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE;
+            body_item->thumbnail_size.x = thumbnail->width;
+            body_item->thumbnail_size.y = thumbnail->height;
+            body_item->thumbnail_size = body_item->thumbnail_size;
+        }
+
+        const Json::Value &like_count_json = comment_renderer_json["likeCount"];
+        if(like_count_json.isInt64()) {
+            if(!description.empty())
+                description += '\n';
+            description += "👍 " + std::to_string(like_count_json.asInt64());
+        }
+
+        const Json::Value &reply_count_json = comment_renderer_json["replyCount"];
+        if(reply_count_json.isInt64()) {
+            if(!description.empty())
+                description += '\n';
+            description += std::to_string(reply_count_json.asInt64()) + " replies";
+        }
+
+        std::string hearted_tooltip = comment_renderer_get_hearted_tooltip(comment_renderer_json);
+        if(!hearted_tooltip.empty()) {
+            if(!description.empty())
+                description += ' ';
+            description += std::move(hearted_tooltip);
+        }
+
+        body_item->set_description(std::move(description));
+        return body_item;
+    }
+
+    PluginResult YoutubeCommentsPage::lazy_fetch(BodyItems &result_items) {
+        std::string next_url = "https://www.youtube.com/comment_service_ajax?action_get_comments=1&pbj=1&ctoken=";
+        next_url += url_param_encode(continuation_token);
+        //next_url += "&continuation=";
+        //next_url += url_param_encode(comments_continuation_token);
+        next_url += "&type=next";
+
+        std::vector<CommandArg> additional_args = {
+            { "-H", "x-youtube-client-name: 1" },
+            { "-H", "x-youtube-client-version: 2.20210308.08.00" },
+            { "-F", "session_token=" + xsrf_token }
+        };
+
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+
+        Json::Value json_root;
+        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
+        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
+
+        if(!json_root.isObject())
+            return PluginResult::ERR;
+
+        const Json::Value &xsrf_token_json = json_root["xsrf_token"];
+        if(xsrf_token_json.isString())
+            xsrf_token = xsrf_token_json.asString();
+
+        const Json::Value &response_json = json_root["response"];
+        if(!response_json.isObject())
+            return PluginResult::ERR;
+
+        const Json::Value &continuation_contents_json = response_json["continuationContents"];
+        if(!continuation_contents_json.isObject())
+            return PluginResult::ERR;
+
+        const Json::Value &item_section_continuation_json = continuation_contents_json["itemSectionContinuation"];
+        if(!item_section_continuation_json.isObject())
+            return PluginResult::ERR;
+
+        const Json::Value &contents_json = item_section_continuation_json["contents"];
+        if(contents_json.isArray()) {
+            for(const Json::Value &json_item : contents_json) {
+                if(!json_item.isObject())
+                    continue;
+
+                const Json::Value &comment_thread_renderer = json_item["commentThreadRenderer"];
+                if(!comment_thread_renderer.isObject())
+                    continue;
+
+                const Json::Value &comment_json = comment_thread_renderer["comment"];
+                if(!comment_json.isObject())
+                    continue;
+
+                auto body_item = comment_renderer_to_body_item(comment_json["commentRenderer"]);
+                if(body_item) {
+                    body_item->url = comment_thread_renderer_get_replies_continuation(comment_thread_renderer);
+                    result_items.push_back(std::move(body_item));
+                }
+            }
+        }
+
+        continuation_token = item_section_renderer_get_continuation(item_section_continuation_json);
+
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubeCommentRepliesPage::get_page(const std::string&, int page, BodyItems &result_items) {
+        while(current_page < page) {
+            PluginResult plugin_result = lazy_fetch(result_items);
+            if(plugin_result != PluginResult::OK) return plugin_result;
+            ++current_page;
+        }
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubeCommentRepliesPage::submit(const std::string&, const std::string&, std::vector<Tab>&) {
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubeCommentRepliesPage::lazy_fetch(BodyItems &result_items) {
+        std::string next_url = "https://www.youtube.com/comment_service_ajax?action_get_comment_replies=1&pbj=1&ctoken=";
+        next_url += url_param_encode(continuation_token);
+        //next_url += "&continuation=";
+        //next_url += url_param_encode(comments_continuation_token);
+        next_url += "&type=next";
+
+        std::vector<CommandArg> additional_args = {
+            { "-H", "x-youtube-client-name: 1" },
+            { "-H", "x-youtube-client-version: 2.20210308.08.00" },
+            { "-F", "session_token=" + xsrf_token }
+        };
+
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+
+        Json::Value json_root;
+        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
+        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
+
+        if(!json_root.isArray())
+            return PluginResult::ERR;
+
+        for(const Json::Value &json_item : json_root) {
+            if(!json_item.isObject())
+                continue;
+
+            const Json::Value &xsrf_token_json = json_item["xsrf_token"];
+            if(xsrf_token_json.isString())
+                xsrf_token = xsrf_token_json.asString();
+
+            const Json::Value &response_json = json_item["response"];
+            if(!response_json.isObject())
+                continue;
+
+            const Json::Value &continuation_contents_json = response_json["continuationContents"];
+            if(!continuation_contents_json.isObject())
+                continue;
+
+            const Json::Value &comment_replies_continuation_json = continuation_contents_json["commentRepliesContinuation"];
+            if(!comment_replies_continuation_json.isObject())
+                continue;
+
+            const Json::Value &contents_json = comment_replies_continuation_json["contents"];
+            if(contents_json.isArray()) {
+                for(const Json::Value &content_item_json : contents_json) {
+                    if(!content_item_json.isObject())
+                        continue;
+
+                    auto body_item = comment_renderer_to_body_item(content_item_json["commentRenderer"]);
+                    if(body_item)
+                        result_items.push_back(std::move(body_item));
+                }
+            }
+
+            // item_section_renderer_get_continuation is compatible with commentRepliesContinuation
+            continuation_token = item_section_renderer_get_continuation(comment_replies_continuation_json);
+            return PluginResult::OK;
+        }
+
+        return PluginResult::ERR;
+    }
+
     static std::string channel_url_extract_id(const std::string &channel_url) {
         size_t index = channel_url.find("channel/");
         if(index == std::string::npos)
@@ -634,8 +963,8 @@ namespace QuickMedia {
             { "--data-raw", Json::writeString(json_builder, request_json) }
         };
 
-        //std::vector<CommandArg> cookies = get_cookies();
-        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         Json::Value json_root;
         DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
@@ -717,8 +1046,8 @@ namespace QuickMedia {
             { "--data-raw", Json::writeString(json_builder, request_json) }
         };
 
-        //std::vector<CommandArg> cookies = get_cookies();
-        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         Json::Value json_root;
         DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
@@ -768,7 +1097,7 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    PluginResult YoutubeChannelPage::submit(const std::string&, const std::string &url, std::vector<Tab> &result_tabs) {
+    PluginResult YoutubeChannelPage::submit(const std::string&, const std::string&, std::vector<Tab> &result_tabs) {
         if(url.empty())
             return PluginResult::OK;
         result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program), nullptr});
@@ -785,8 +1114,8 @@ namespace QuickMedia {
             { "-H", "referer: " + url }
         };
 
-        //std::vector<CommandArg> cookies = get_cookies();
-        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         Json::Value json_root;
         DownloadResult result = download_json(json_root, url + "/videos?pbj=1", std::move(additional_args), true);
@@ -800,8 +1129,32 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    // TODO: Make this faster by using string search instead of parsing html.
-    // TODO: If the result is a play
+    static std::string two_column_watch_next_results_get_comments_continuation_token(const Json::Value &tcwnr_json) {
+        const Json::Value &results_json = tcwnr_json["results"];
+        if(!results_json.isObject())
+            return "";
+
+        const Json::Value &results2_json = results_json["results"];
+        if(!results2_json.isObject())
+            return "";
+
+        const Json::Value &contents_json = results2_json["contents"];
+        if(!contents_json.isArray())
+            return "";
+
+        std::string comments_continuation_token;
+        for(const Json::Value &content_item_json : contents_json) {
+            if(!content_item_json.isObject())
+                continue;
+            
+            comments_continuation_token = item_section_renderer_get_continuation(content_item_json["itemSectionRenderer"]);
+            if(!comments_continuation_token.empty())
+                return comments_continuation_token;
+        }
+
+        return "";
+    }
+
     BodyItems YoutubeVideoPage::get_related_media(const std::string &url, std::string &channel_url) {
         BodyItems result_items;
 
@@ -814,8 +1167,8 @@ namespace QuickMedia {
             { "-H", "referer: " + url }
         };
 
-        //std::vector<CommandArg> cookies = get_cookies();
-        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         Json::Value json_root;
         DownloadResult result = download_json(json_root, modified_url + "&pbj=1", std::move(additional_args), true);
@@ -842,6 +1195,12 @@ namespace QuickMedia {
                 }
             }
 
+            if(xsrf_token.empty()) {
+                const Json::Value &xsrf_token_json = json_item["xsrf_token"];
+                if(xsrf_token_json.isString())
+                    xsrf_token = xsrf_token_json.asString();
+            }
+
             const Json::Value &response_json = json_item["response"];
             if(!response_json.isObject())
                 continue;
@@ -854,6 +1213,9 @@ namespace QuickMedia {
             if(!tcwnr_json.isObject())
                 return result_items;
 
+            if(comments_continuation_token.empty())
+                comments_continuation_token = two_column_watch_next_results_get_comments_continuation_token(tcwnr_json);
+
             const Json::Value &secondary_results_json = tcwnr_json["secondaryResults"];
             if(!secondary_results_json.isObject())
                 return result_items;
@@ -901,6 +1263,10 @@ namespace QuickMedia {
         return std::make_unique<YoutubeSearchPage>(program);
     }
 
+    std::unique_ptr<Page> YoutubeVideoPage::create_comments_page(Program *program) {
+        return std::make_unique<YoutubeCommentsPage>(program, xsrf_token, comments_continuation_token);
+    }
+
     std::unique_ptr<RelatedVideosPage> YoutubeVideoPage::create_related_videos_page(Program *program, const std::string&, const std::string&) {
         return std::make_unique<YoutubeRelatedVideosPage>(program);
     }