Youtube add channel page, fix search pagination (update to correct continuation token)

2 files changed, 361 insertions, 72 deletions

diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp
index 96ce789..198b2fd 100644
--- a/src/plugins/Matrix.cpp
+++ b/src/plugins/Matrix.cpp
@@ -818,6 +818,7 @@ namespace QuickMedia {
             return;
 
         assert(!this->delegate);
+        assert(!access_token.empty()); // Need to be logged in
         this->delegate = delegate;
 
         Path matrix_cache_dir = get_cache_dir().join("matrix");
@@ -846,7 +847,7 @@ namespace QuickMedia {
             }
             sync_is_cache = false;
 
-            // Filter with account data. TODO: Test if this is needed for encrypted chats
+            // Filter with account data
             // {"presence":{"limit":0,"types":[""]},"account_data":{"not_types":["im.vector.setting.breadcrumbs","m.push_rules","im.vector.setting.allowed_widgets","io.element.recent_emoji"]},"room":{"state":{"limit":1,"not_types":["m.room.related_groups","m.room.power_levels","m.room.join_rules","m.room.history_visibility"],"lazy_load_members":true},"timeline":{"limit":3,"lazy_load_members":true},"ephemeral":{"limit":0,"types":[""],"lazy_load_members":true},"account_data":{"limit":1,"types":["m.fully_read"],"lazy_load_members":true}}}
             // Filter without account data
             const char *filter = "{\"presence\":{\"limit\":0,\"types\":[\"\"]},\"account_data\":{\"limit\":0,\"types\":[\"\"]},\"room\":{\"state\":{\"not_types\":[\"m.room.related_groups\",\"m.room.power_levels\",\"m.room.join_rules\",\"m.room.history_visibility\"],\"lazy_load_members\":true},\"timeline\":{\"limit\":3,\"lazy_load_members\":true},\"ephemeral\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true},\"account_data\":{\"limit\":1,\"types\":[\"m.fully_read\",\"m.tag\"],\"lazy_load_members\":true}}}";
@@ -861,11 +862,6 @@ namespace QuickMedia {
             PluginResult result;
             bool initial_sync = true;
             while(sync_running) {
-                std::vector<CommandArg> additional_args = {
-                    { "-H", "Authorization: Bearer " + access_token },
-                    { "-m", "35" }
-                };
-
                 char url[1024];
                 if(next_batch.empty())
                     snprintf(url, sizeof(url), "%s/_matrix/client/r0/sync?filter=%s&timeout=0", homeserver.c_str(), filter_encoded.c_str());
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index a157a8c..1ca25a3 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -1,80 +1,62 @@
 #include "../../plugins/Youtube.hpp"
 #include "../../include/Storage.hpp"
 #include "../../include/NetUtils.hpp"
+#include "../../include/Scale.hpp"
 #include <string.h>
-#include <unordered_set>
 
 namespace QuickMedia {
-    static std::shared_ptr<BodyItem> parse_common_video_item(const Json::Value &video_item_json, std::unordered_set<std::string> &added_videos) {
-        const Json::Value &video_id_json = video_item_json["videoId"];
-        if(!video_id_json.isString())
+    // This is a common setup of text in the youtube json
+    static const char* yt_json_get_text(const Json::Value &json, const char *root_name) {
+        if(!json.isObject())
             return nullptr;
 
-        std::string video_id_str = video_id_json.asString();
-        if(added_videos.find(video_id_str) != added_videos.end())
+        const Json::Value &text_json = json[root_name];
+        if(!text_json.isObject())
             return nullptr;
 
-        std::string thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/hqdefault.jpg";
-
-        const char *date = nullptr;
-        const Json::Value &published_time_text_json = video_item_json["publishedTimeText"];
-        if(published_time_text_json.isObject()) {
-            const Json::Value &text_json = published_time_text_json["simpleText"];
-            if(text_json.isString())
-                date = text_json.asCString();
-        }
-
-        const char *length = nullptr;
-        const Json::Value &length_text_json = video_item_json["lengthText"];
-        if(length_text_json.isObject()) {
-            const Json::Value &text_json = length_text_json["simpleText"];
-            if(text_json.isString())
-                length = text_json.asCString();
-        }
-
-        const char *view_count_text = nullptr;
-        const Json::Value &view_count_text_json = video_item_json["viewCountText"];
-        if(view_count_text_json.isObject()) {
-            const Json::Value &text_json = view_count_text_json["simpleText"];
-            if(text_json.isString())
-                view_count_text = text_json.asCString();
-        }
-
-        const char *owner_text = nullptr;
-        const Json::Value &owner_text_json = video_item_json["shortBylineText"];
-        if(owner_text_json.isObject()) {
-            const Json::Value &runs_json = owner_text_json["runs"];
+        const Json::Value &simple_text_json = text_json["simpleText"];
+        if(simple_text_json.isString()) {
+            return simple_text_json.asCString();
+        } else {
+            const Json::Value &runs_json = text_json["runs"];
             if(runs_json.isArray() && !runs_json.empty()) {
                 const Json::Value &first_runs_json = runs_json[0];
                 if(first_runs_json.isObject()) {
                     const Json::Value &text_json = first_runs_json["text"];
                     if(text_json.isString())
-                        owner_text = text_json.asCString();
+                        return text_json.asCString();
                 }
             }
         }
 
-        const char *title = nullptr;
-        const Json::Value &title_json = video_item_json["title"];
-        if(title_json.isObject()) {
-            const Json::Value &simple_text_json = title_json["simpleText"];
-            if(simple_text_json.isString()) {
-                title = simple_text_json.asCString();
-            } else {
-                const Json::Value &runs_json = title_json["runs"];
-                if(runs_json.isArray() && !runs_json.empty()) {
-                    const Json::Value &first_runs_json = runs_json[0];
-                    if(first_runs_json.isObject()) {
-                        const Json::Value &text_json = first_runs_json["text"];
-                        if(text_json.isString())
-                            title = text_json.asCString();
-                    }
-                }
-            }
-        }
+        return nullptr;
+    }
+
+    static std::shared_ptr<BodyItem> parse_common_video_item(const Json::Value &video_item_json, std::unordered_set<std::string> &added_videos) {
+        const Json::Value &video_id_json = video_item_json["videoId"];
+        if(!video_id_json.isString())
+            return nullptr;
+
+        std::string video_id_str = video_id_json.asString();
+        if(added_videos.find(video_id_str) != added_videos.end())
+            return nullptr;
 
+        const char *title = yt_json_get_text(video_item_json, "title");
         if(!title)
             return nullptr;
+
+        const char *date = yt_json_get_text(video_item_json, "publishedTimeText");
+        const char *view_count_text = yt_json_get_text(video_item_json, "viewCountText");
+        const char *owner_text = yt_json_get_text(video_item_json, "shortBylineText");
+        const char *length = yt_json_get_text(video_item_json, "lengthText");
+        if(!length) {
+            const Json::Value &thumbnail_overlays_json = video_item_json["thumbnailOverlays"];
+            if(thumbnail_overlays_json.isArray() && !thumbnail_overlays_json.empty()) {
+                const Json::Value &thumbnail_overlay_json = thumbnail_overlays_json[0];
+                if(thumbnail_overlay_json.isObject())
+                    length = yt_json_get_text(thumbnail_overlay_json["thumbnailOverlayTimeStatusRenderer"], "text");
+            }
+        }
         
         auto body_item = BodyItem::create(title);
         std::string desc;
@@ -97,7 +79,7 @@ namespace QuickMedia {
         }
         body_item->set_description(std::move(desc));
         body_item->url = "https://www.youtube.com/watch?v=" + video_id_str;
-        body_item->thumbnail_url = std::move(thumbnail_url);
+        body_item->thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/hqdefault.jpg";
         body_item->thumbnail_size = sf::Vector2i(175, 131);
         added_videos.insert(video_id_str);
         return body_item;
@@ -114,6 +96,96 @@ namespace QuickMedia {
         return parse_common_video_item(video_renderer_json, added_videos);
     }
 
+    struct Thumbnail {
+        const char *url;
+        int width;
+        int height;
+    };
+
+    static std::optional<Thumbnail> yt_json_get_largest_thumbnail(const Json::Value &thumbnail_json) {
+        if(!thumbnail_json.isObject())
+            return std::nullopt;
+
+        const Json::Value &thumbnails_json = thumbnail_json["thumbnails"];
+        if(!thumbnails_json.isArray())
+            return std::nullopt;
+
+        std::vector<Thumbnail> thumbnails;
+        for(const Json::Value &thumbnail_data_json : thumbnails_json) {
+            if(!thumbnail_data_json.isObject())
+                continue;
+
+            const Json::Value &url_json = thumbnail_data_json["url"];
+            if(!url_json.isString())
+                continue;
+
+            const Json::Value &width_json = thumbnail_data_json["width"];
+            if(!width_json.isInt())
+                continue;
+
+            const Json::Value &height_json = thumbnail_data_json["height"];
+            if(!height_json.isInt())
+                continue;
+
+            thumbnails.push_back({ url_json.asCString(), width_json.asInt(), height_json.asInt() });
+        }
+
+        return *std::max_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) {
+            int size1 = thumbnail1.width * thumbnail1.height;
+            int size2 = thumbnail2.width * thumbnail2.height;
+            return size1 < size2;
+        });
+    }
+
+    static std::shared_ptr<BodyItem> parse_channel_renderer(const Json::Value &channel_renderer_json) {
+        if(!channel_renderer_json.isObject())
+            return nullptr;
+
+        const Json::Value &channel_id_json = channel_renderer_json["channelId"];
+        if(!channel_id_json.isString())
+            return nullptr;
+
+        const char *title = yt_json_get_text(channel_renderer_json, "title");
+        if(!title)
+            return nullptr;
+
+        const char *description = yt_json_get_text(channel_renderer_json, "descriptionSnippet");
+        const char *video_count = yt_json_get_text(channel_renderer_json, "videoCountText");
+        const char *subscribers = yt_json_get_text(channel_renderer_json, "subscriberCountText");
+
+        const Json::Value &thumbnail_json = channel_renderer_json["thumbnail"];
+        std::optional<Thumbnail> thumbnail = yt_json_get_largest_thumbnail(thumbnail_json);
+
+        auto body_item = BodyItem::create(title);
+        std::string desc;
+        if(subscribers)
+            desc += subscribers;
+        if(video_count) {
+            if(!desc.empty())
+                desc += " • ";
+            desc += video_count;
+            if(strcmp(video_count, "1") == 0)
+                desc += " video";
+            else
+                desc += " videos";
+        }
+        if(description) {
+            if(!desc.empty())
+                desc += '\n';
+            desc += description;
+        }
+        body_item->set_description(std::move(desc));
+        body_item->url = "https://www.youtube.com/channel/" + channel_id_json.asString();
+        if(thumbnail) {
+            body_item->thumbnail_url = std::string("https:") + thumbnail->url;
+            body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE;
+            body_item->thumbnail_size.x = thumbnail->width;
+            body_item->thumbnail_size.y = thumbnail->height;
+            body_item->thumbnail_size = clamp_to_size(body_item->thumbnail_size, sf::Vector2i(136, 136));
+        }
+        return body_item;
+    }
+
     // Returns empty string if continuation token can't be found
     static std::string item_section_renderer_get_continuation_token(const Json::Value &item_section_renderer_json) {
         const Json::Value &continuation_item_renderer_json = item_section_renderer_json["continuationItemRenderer"];
@@ -135,6 +207,27 @@ namespace QuickMedia {
         return token_json.asString();
     }
 
+    static std::string grid_renderer_get_continuation_token(const Json::Value &grid_renderer_json) {
+        const Json::Value &continuations_json = grid_renderer_json["continuations"];
+        if(!continuations_json.isArray())
+            return "";
+
+        for(const Json::Value &continuation_json : continuations_json) {
+            if(!continuation_json.isObject())
+                continue;
+
+            const Json::Value &next_continuation_data_json = continuation_json["nextContinuationData"];
+            if(!next_continuation_data_json.isObject())
+                continue;
+
+            const Json::Value &continuation_item_json = next_continuation_data_json["continuation"];
+            if(continuation_item_json.isString())
+                return continuation_item_json.asString();
+        }
+
+        return "";
+    }
+
     static void parse_item_section_renderer(const Json::Value &item_section_renderer_json, std::unordered_set<std::string> &added_videos, BodyItems &result_items) {
         const Json::Value &item_contents_json = item_section_renderer_json["contents"];
         if(!item_contents_json.isArray())
@@ -168,6 +261,10 @@ namespace QuickMedia {
                         if(body_item)
                             result_items.push_back(std::move(body_item));
                     }
+                } else if(key.isString() && strcmp(key.asCString(), "channelRenderer") == 0) {
+                    std::shared_ptr<BodyItem> body_item = parse_channel_renderer(*it);
+                    if(body_item)
+                        result_items.push_back(std::move(body_item));
                 } else {
                     std::shared_ptr<BodyItem> body_item = parse_content_video_renderer(content_item_json, added_videos);
                     if(body_item)
@@ -193,9 +290,106 @@ namespace QuickMedia {
         return parse_common_video_item(compact_video_renderer_json, added_videos);
     }
 
+    static BodyItems parse_channel_videos(const Json::Value &json_root, std::string &continuation_token, std::unordered_set<std::string> &added_videos) {
+        BodyItems body_items;
+        if(!json_root.isArray())
+            return body_items;
+
+        std::string new_continuation_token;
+        for(const Json::Value &json_item : json_root) {
+            if(!json_item.isObject())
+                continue;
+
+            const Json::Value &response_json = json_item["response"];
+            if(!response_json.isObject())
+                continue;
+
+            const Json::Value &contents_json = response_json["contents"];
+            if(!contents_json.isObject())
+                continue;
+
+            const Json::Value &tcbrr_json = contents_json["twoColumnBrowseResultsRenderer"];
+            if(!tcbrr_json.isObject())
+                continue;
+
+            const Json::Value &tabs_json = tcbrr_json["tabs"];
+            if(!tabs_json.isArray())
+                continue;
+
+            for(const Json::Value &tab_json : tabs_json) {
+                if(!tab_json.isObject())
+                    continue;
+
+                const Json::Value &tab_renderer_json = tab_json["tabRenderer"];
+                if(!tab_renderer_json.isObject())
+                    continue;
+                
+                const Json::Value &content_json = tab_renderer_json["content"];
+                if(!content_json.isObject())
+                    continue;
+
+                const Json::Value &section_list_renderer = content_json["sectionListRenderer"];
+                if(!section_list_renderer.isObject())
+                    continue;
+
+                const Json::Value &contents2_json = section_list_renderer["contents"];
+                if(!contents2_json.isArray())
+                    continue;
+                
+                for(const Json::Value &content_item_json : contents2_json) {
+                    if(!content_item_json.isObject())
+                        continue;
+                    
+                    const Json::Value &item_section_renderer_json = content_item_json["itemSectionRenderer"];
+                    if(!item_section_renderer_json.isObject())
+                        continue;
+
+                    const Json::Value &item_contents_json = item_section_renderer_json["contents"];
+                    if(!item_contents_json.isArray())
+                        continue;
+
+                    for(const Json::Value &content_json : item_contents_json) {
+                        if(!content_json.isObject())
+                            continue;
+
+                        const Json::Value &grid_renderer_json = content_json["gridRenderer"];
+                        if(!grid_renderer_json.isObject())
+                            continue;
+
+                        if(new_continuation_token.empty())
+                            new_continuation_token = grid_renderer_get_continuation_token(grid_renderer_json);
+
+                        const Json::Value &items_json = grid_renderer_json["items"];
+                        if(!items_json.isArray())
+                            continue;
+
+                        for(const Json::Value &item_json : items_json) {
+                            if(!item_json.isObject())
+                                continue;
+
+                            const Json::Value &grid_video_renderer = item_json["gridVideoRenderer"];
+                            if(!grid_video_renderer.isObject())
+                                continue;
+
+                            auto body_item = parse_common_video_item(grid_video_renderer, added_videos);
+                            if(body_item)
+                                body_items.push_back(std::move(body_item));
+                        }
+                    }
+                }
+            }
+        }
+
+        if(!new_continuation_token.empty())
+            continuation_token = std::move(new_continuation_token);
+
+        return body_items;
+    }
+
     SearchResult YoutubeSearchPage::search(const std::string &str, BodyItems &result_items) {
         continuation_token.clear();
         current_page = 0;
+        added_videos.clear();
 
         search_url = "https://youtube.com/results?search_query=";
         search_url += url_param_encode(str);
@@ -217,8 +411,6 @@ namespace QuickMedia {
         if(!json_root.isArray())
             return SearchResult::ERR;
 
-        std::unordered_set<std::string> added_videos; /* The input contains duplicates, filter them out! */
-
         for(const Json::Value &json_item : json_root) {
             if(!json_item.isObject())
                 continue;
@@ -265,8 +457,7 @@ namespace QuickMedia {
         return SearchResult::OK;
     }
 
-    PluginResult YoutubeSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) {
-        (void)str;
+    PluginResult YoutubeSearchPage::get_page(const std::string&, int page, BodyItems &result_items) {
         while(current_page < page) {
             PluginResult plugin_result = search_get_continuation(search_url, continuation_token, result_items);
             if(plugin_result != PluginResult::OK) return plugin_result;
@@ -276,9 +467,32 @@ namespace QuickMedia {
     }
 
     PluginResult YoutubeSearchPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
-        (void)title;
-        (void)url;
-        result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program), nullptr});
+        if(strncmp(url.c_str(), "https://www.youtube.com/channel/", 32) == 0) {
+            std::vector<CommandArg> additional_args = {
+                { "-H", "x-spf-referer: " + url },
+                { "-H", "x-youtube-client-name: 1" },
+                { "-H", "x-spf-previous: " + url },
+                { "-H", "x-youtube-client-version: 2.20200626.03.00" },
+                { "-H", "referer: " + url }
+            };
+
+            //std::vector<CommandArg> cookies = get_cookies();
+            //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+
+            Json::Value json_root;
+            DownloadResult result = download_json(json_root, url + "/videos?pbj=1", std::move(additional_args), true);
+            if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
+
+            auto channel_body = create_body();
+            std::unordered_set<std::string> added_videos;
+            std::string continuation_token;
+            channel_body->items = parse_channel_videos(json_root, continuation_token, added_videos);
+            auto channel_page = std::make_unique<YoutubeChannelPage>(program, url, std::move(continuation_token), title);
+            channel_page->added_videos = std::move(added_videos);
+            result_tabs.push_back(Tab{std::move(channel_body), std::move(channel_page), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+        } else {
+            result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program), nullptr});
+        }
         return PluginResult::OK;
     }
 
@@ -303,9 +517,7 @@ namespace QuickMedia {
         if(!json_root.isArray())
             return PluginResult::ERR;
 
-        std::unordered_set<std::string> added_videos;
         std::string new_continuation_token;
-
         for(const Json::Value &json_item : json_root) {
             if(!json_item.isObject())
                 continue;
@@ -348,12 +560,93 @@ namespace QuickMedia {
             }
         }
 
-        if(new_continuation_token.empty())
+        if(!new_continuation_token.empty())
+            continuation_token = std::move(new_continuation_token);
+
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubeChannelPage::get_page(const std::string&, int page, BodyItems &result_items) {
+        while(current_page < page) {
+            PluginResult plugin_result = search_get_continuation(url, continuation_token, result_items);
+            if(plugin_result != PluginResult::OK) return plugin_result;
+            ++current_page;
+        }
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubeChannelPage::search_get_continuation(const std::string &url, const std::string &current_continuation_token, BodyItems &result_items) {
+        std::string next_url = "https://www.youtube.com/browse_ajax?ctoken=" + current_continuation_token + "&continuation=" + current_continuation_token;
+
+        std::vector<CommandArg> additional_args = {
+            { "-H", "x-spf-referer: " + url },
+            { "-H", "x-youtube-client-name: 1" },
+            { "-H", "x-spf-previous: " + url },
+            { "-H", "x-youtube-client-version: 2.20200626.03.00" },
+            { "-H", "referer: " + url }
+        };
+
+        //std::vector<CommandArg> cookies = get_cookies();
+        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+
+        Json::Value json_root;
+        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
+        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
+
+        if(!json_root.isArray())
+            return PluginResult::ERR;
+
+        std::string new_continuation_token;
+        for(const Json::Value &json_item : json_root) {
+            if(!json_item.isObject())
+                continue;
+
+            const Json::Value &response_json = json_item["response"];
+            if(!response_json.isObject())
+                continue;
+
+            const Json::Value &continuation_contents_json = response_json["continuationContents"];
+            if(!continuation_contents_json.isObject())
+                continue;
+
+            const Json::Value &grid_continuation_json = continuation_contents_json["gridContinuation"];
+            if(!grid_continuation_json.isObject())
+                continue;
+
+            if(new_continuation_token.empty()) {
+                // grid_continuation_json is compatible with grid_renderer
+                new_continuation_token = grid_renderer_get_continuation_token(grid_continuation_json);
+            }
+
+            const Json::Value &items_json = grid_continuation_json["items"];
+            if(!items_json.isArray())
+                continue;
+
+            for(const Json::Value &item_json : items_json) {
+                if(!item_json.isObject())
+                    continue;
+
+                const Json::Value &grid_video_renderer = item_json["gridVideoRenderer"];
+                if(!grid_video_renderer.isObject())
+                    continue;
+
+                auto body_item = parse_common_video_item(grid_video_renderer, added_videos);
+                if(body_item)
+                    result_items.push_back(std::move(body_item));
+            }
+        }
+
+        if(!new_continuation_token.empty())
             continuation_token = std::move(new_continuation_token);
 
         return PluginResult::OK;
     }
 
+    PluginResult YoutubeChannelPage::submit(const std::string&, const std::string&, std::vector<Tab> &result_tabs) {
+        result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program), nullptr});
+        return PluginResult::OK;
+    }
+
     // TODO: Make this faster by using string search instead of parsing html.
     // TODO: If the result is a play
     BodyItems YoutubeVideoPage::get_related_media(const std::string &url) {