Youtube: do proper playlist pagination, show playlist videos views and uploaded date

4 files changed, 154 insertions, 35 deletions

diff --git a/TODO b/TODO
index 0981eb9..e66526c 100644
--- a/TODO
+++ b/TODO
@@ -291,4 +291,5 @@ Improve youtube live streaming performance. It's slow because mpv (ffmpeg) is sl
 Add youtube config for allowing to specify download specific codec preference (or ignore video max height). Might have slow download so cant play high quality video but after downloading it it can be played.
 Fix youtube age restricted videos.
 Direct link to youtube playlist should open the playlist page and select the playlist item.
-Support youtube mix. Doesn't work that nicely because the mix playlist doesn't show all items unless you click on the last item and then it will show more items.
\ No newline at end of file
+Support youtube mix. Doesn't work that nicely because the mix playlist doesn't show all items unless you click on the last item and then it will show more items.
+Youtube community tab.
diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp
index 22cb9e5..77aa86c 100644
--- a/plugins/Youtube.hpp
+++ b/plugins/Youtube.hpp
@@ -157,14 +157,16 @@ namespace QuickMedia {
 
     class YoutubePlaylistPage : public LazyFetchPage {
     public:
-        YoutubePlaylistPage(Program *program, std::string url, std::string title) :
-            LazyFetchPage(program), url(std::move(url)), title(std::move(title)) {}
+        YoutubePlaylistPage(Program *program, const std::string &url, std::string title);
         const char* get_title() const override { return title.c_str(); }
+        PluginResult get_page(const std::string &str, int page, BodyItems &result_items) override;
         PluginResult submit(const SubmitArgs &args, std::vector<Tab> &result_tabs) override;
         PluginResult lazy_fetch(BodyItems &result_items) override;
     private:
-        std::string url;
+        std::string playlist_id;
         std::string title;
+        std::string continuation_token;
+        bool reached_end = false;
     };
 
     class YoutubeVideoPage : public VideoPage {
diff --git a/src/plugins/Page.cpp b/src/plugins/Page.cpp
index 991f651..0e52ca4 100644
--- a/src/plugins/Page.cpp
+++ b/src/plugins/Page.cpp
@@ -19,6 +19,10 @@ namespace QuickMedia {
             return DownloadResult::OK;
         }
 
+        //FILE *f = fopen("data.json", "wb");
+        //fwrite(server_response.data(), 1, server_response.size(), f);
+        //fclose(f);
+
         Json::CharReaderBuilder json_builder;
         std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader());
         std::string json_errors;
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index 2e59a0b..d4211b3 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -192,6 +192,20 @@ namespace QuickMedia {
         return false;
     }
 
+    static bool youtube_url_extract_playlist_id(const std::string &youtube_url, std::string &playlist_id) {
+        size_t start_index = youtube_url.find("&list=");
+        if(start_index == std::string::npos)
+            return false;
+
+        start_index += 6;
+        size_t end_index = youtube_url.find("&", start_index);
+        if(end_index == std::string::npos)
+            end_index = youtube_url.size();
+
+        playlist_id = youtube_url.substr(start_index, end_index - start_index);
+        return true;
+    }
+
     static std::mutex cookies_mutex;
     static std::string cookies_filepath;
     static std::string api_key;
@@ -557,6 +571,7 @@ namespace QuickMedia {
         if(!title)
             title = std::move(headline);
 
+        std::optional<std::string> video_info = yt_json_get_text(video_item_json, "videoInfo");
         std::optional<std::string> date = yt_json_get_text(video_item_json, "publishedTimeText");
         std::optional<std::string> view_count_text = yt_json_get_text(video_item_json, "viewCountText");
         std::optional<std::string> owner_text = yt_json_get_text(video_item_json, "shortBylineText");
@@ -596,33 +611,43 @@ namespace QuickMedia {
         
         auto body_item = BodyItem::create(title.value());
         std::string desc;
-        if(view_count_text)
-            desc += view_count_text.value();
-        if(date) {
-            if(!desc.empty())
-                desc += " • ";
-            desc += date.value();
+        if(video_info) {
+            desc += video_info.value();
+        } else {
+            if(view_count_text)
+                desc += view_count_text.value();
+
+            if(date) {
+                if(!desc.empty())
+                    desc += " • ";
+                desc += date.value();
+            }
         }
+
         if(!scheduled_text.empty()) {
             if(!desc.empty())
                 desc += " • ";
             desc += scheduled_text;
         }
+
         if(length) {
             if(!desc.empty())
                 desc += '\n';
             desc += length.value();
         }
+
         if(video_is_live(video_item_json)) {
             if(!desc.empty())
                 desc += '\n';
             desc += "Live now";
         }
+
         if(owner_text) {
             if(!desc.empty())
                 desc += '\n';
             desc += owner_text.value();
         }
+
         /*if(description_snippet) {
             if(!desc.empty())
                 desc += '\n';
@@ -781,8 +806,15 @@ namespace QuickMedia {
         if(long_byline_text.empty())
             long_byline_text = yt_json_get_text(playlist_renderer_json, "shortBylineText").value_or("");
 
+        std::optional<std::string> published_time_text = yt_json_get_text(playlist_renderer_json, "publishedTimeText");
+
         std::string video_id;
         std::string description = std::move(long_byline_text);
+        if(published_time_text) {
+            if(!description.empty())
+                description += " • ";
+            description += published_time_text.value();
+        }
         if(!description.empty())
             description += '\n';
         description += video_count_text + " video" + (strcmp(video_count_text.c_str(), "1") == 0 ? "" : "s");
@@ -842,7 +874,7 @@ namespace QuickMedia {
             if(body_item)
                 result_items.push_back(std::move(body_item));
 
-            // TODO: Mix
+            // TODO: youtube mix
             //body_item = parse_playlist_renderer(item_json["radioRenderer"]);
             //if(body_item)
             //    result_items.push_back(std::move(body_item));
@@ -870,7 +902,7 @@ namespace QuickMedia {
             if(body_item)
                 result_items.push_back(std::move(body_item));
 
-            // TODO: Mix
+            // TODO: youtube mix
             //body_item = parse_playlist_renderer(content_item_json["radioRenderer"]);
             //if(body_item)
             //    result_items.push_back(std::move(body_item));
@@ -912,6 +944,27 @@ namespace QuickMedia {
         return "";
     }
 
+    static void parse_playlist_video_list(const Json::Value &playlist_video_list_json, const char *list_name, std::string &continuation_token, std::unordered_set<std::string> &added_videos, BodyItems &body_items) {
+        if(!playlist_video_list_json.isObject())
+            return;
+
+        const Json::Value &contents_json = playlist_video_list_json[list_name];
+        if(!contents_json.isArray())
+            return;
+
+        for(const Json::Value &content_json : contents_json) {
+            if(!content_json.isObject())
+                continue;
+
+            if(continuation_token.empty())
+                continuation_token = item_section_renderer_get_continuation_token(content_json);
+
+            auto body_item = parse_common_video_item(content_json["playlistVideoRenderer"], added_videos);
+            if(body_item)
+                body_items.push_back(std::move(body_item));
+        }
+    }
+
     static void parse_channel_videos(const Json::Value &json_root, std::string &continuation_token, std::unordered_set<std::string> &added_videos, std::string &browse_id, BodyItems &body_items) {
         if(!json_root.isObject())
             return;
@@ -1014,6 +1067,8 @@ namespace QuickMedia {
                     if(!content_json.isObject())
                         continue;
 
+                    parse_playlist_video_list(content_json["playlistVideoListRenderer"], "contents", continuation_token, added_videos, body_items);
+
                     const Json::Value &grid_renderer_json = content_json["gridRenderer"];
                     if(!grid_renderer_json.isObject())
                         continue;
@@ -1214,7 +1269,7 @@ namespace QuickMedia {
             // TODO: Make all pages (for all services) lazy fetch in a similar manner!
             YoutubeChannelPage::create_each_type(program, args.url, "", args.title, result_tabs);
         } else if(strstr(args.url.c_str(), "&list=")) {
-            result_tabs.push_back(Tab{create_body(false, true), std::make_unique<YoutubePlaylistPage>(program, args.url, args.title), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+            result_tabs.push_back(Tab{create_body(false, true), std::make_unique<YoutubePlaylistPage>(program, args.url, args.title), create_search_bar("Filter...", SEARCH_DELAY_FILTER)});
         } else {
             result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, args.url), nullptr});
         }
@@ -1998,7 +2053,7 @@ namespace QuickMedia {
             return PluginResult::OK;
 
         if(strstr(args.url.c_str(), "&list=")) {
-            result_tabs.push_back(Tab{create_body(false, true), std::make_unique<YoutubePlaylistPage>(program, args.url, args.title), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+            result_tabs.push_back(Tab{create_body(false, true), std::make_unique<YoutubePlaylistPage>(program, args.url, args.title), create_search_bar("Filter...", SEARCH_DELAY_FILTER)});
         } else {
             result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, args.url), nullptr});
         }
@@ -2257,20 +2312,15 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    PluginResult YoutubePlaylistPage::submit(const SubmitArgs &args, std::vector<Tab> &result_tabs) {
-        result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, args.url, true, true), nullptr});
-        return PluginResult::OK;
-    }
-
     static void playlist_get_items(const Json::Value &json_root, std::unordered_set<std::string> &added_videos, BodyItems &result_items) {
         if(!json_root.isObject())
             return;
 
-        const Json::Value &response_json = json_root["response"];
-        if(!response_json.isObject())
-            return;
+        const Json::Value *response_json = &json_root["response"];
+        if(!response_json->isObject())
+            response_json = &json_root;
 
-        const Json::Value &contents_json = response_json["contents"];
+        const Json::Value &contents_json = (*response_json)["contents"];
         if(!contents_json.isObject())
             return;
 
@@ -2301,35 +2351,97 @@ namespace QuickMedia {
         }
     }
 
-    PluginResult YoutubePlaylistPage::lazy_fetch(BodyItems &result_items) {
+    YoutubePlaylistPage::YoutubePlaylistPage(Program *program, const std::string &url, std::string title) :
+        LazyFetchPage(program), title(std::move(title))
+    {
+        if(!youtube_url_extract_playlist_id(url, playlist_id))
+            fprintf(stderr, "Error: failed to extract playlist id from url: %s\n", url.c_str());
+    }
+
+    PluginResult YoutubePlaylistPage::get_page(const std::string&, int, BodyItems &result_items) {
+        if(reached_end)
+            return PluginResult::OK;
+
+        std::vector<CommandArg> cookies = get_cookies();
+        std::string next_url = "https://www.youtube.com/youtubei/v1/browse?key=" + url_param_encode(api_key) + "&gl=US&hl=en&prettyPrint=false";
+
+        Json::Value client_json(Json::objectValue);
+        client_json["hl"] = "en";
+        client_json["gl"] = "US";
+        client_json["deviceMake"] = "";
+        client_json["deviceModel"] = "";
+        client_json["userAgent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36";
+        client_json["clientName"] = "WEB";
+        client_json["clientVersion"] = "2.20210622.10.00";
+        client_json["osName"] = "X11";
+        client_json["osVersion"] = "";
+        client_json["originalUrl"] = "https://www.youtube.com/playlist?list=" + playlist_id;
+
+        Json::Value context_json(Json::objectValue);
+        context_json["client"] = std::move(client_json);
+
+        Json::Value request_json(Json::objectValue);
+        request_json["context"] = std::move(context_json);
+        if(continuation_token.empty())
+            request_json["browseId"] = "VL" + playlist_id;
+        else
+            request_json["continuation"] = continuation_token;
+
+        Json::StreamWriterBuilder json_builder;
+        json_builder["commentStyle"] = "None";
+        json_builder["indentation"] = "";
+
         std::vector<CommandArg> additional_args = {
+            { "-H", "content-type: application/json" },
             { "-H", "x-youtube-client-name: 1" },
             { "-H", youtube_client_version },
+            { "--data-raw", Json::writeString(json_builder, request_json) }
         };
 
-        std::vector<CommandArg> cookies = get_cookies();
         additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         Json::Value json_root;
-        DownloadResult download_result = download_json(json_root, url + "&pbj=1&gl=US&hl=en", additional_args, true);
-        if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result);
+        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
+        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
+
+        if(!json_root.isObject())
+            return PluginResult::ERR;
 
         std::unordered_set<std::string> added_videos;
-        if(json_root.isObject()) {
-            playlist_get_items(json_root, added_videos, result_items);
-            return PluginResult::OK;
-        }
+        std::string browse_id;
+        continuation_token.clear();
 
-        if(!json_root.isArray())
-            return PluginResult::OK;
+        const Json::Value &on_response_received_actions_json = json_root["onResponseReceivedActions"];
+        if(on_response_received_actions_json.isArray()) {
+            for(const Json::Value &json_item : on_response_received_actions_json) {
+                if(!json_item.isObject())
+                    continue;
 
-        for(const Json::Value &json_item : json_root) {
-            playlist_get_items(json_item, added_videos, result_items);
+                const Json::Value &append_continuation_items_action_json = json_item["appendContinuationItemsAction"];
+                if(!append_continuation_items_action_json.isObject())
+                    continue;
+                
+                parse_playlist_video_list(append_continuation_items_action_json, "continuationItems", continuation_token, added_videos, result_items);
+            }
+        } else {
+            parse_channel_videos(json_root, continuation_token, added_videos, browse_id, result_items);
         }
 
+        if(continuation_token.empty())
+            reached_end = true;
+
         return PluginResult::OK;
     }
 
+    PluginResult YoutubePlaylistPage::submit(const SubmitArgs &args, std::vector<Tab> &result_tabs) {
+        result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, args.url, true, true), nullptr});
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubePlaylistPage::lazy_fetch(BodyItems &result_items) {
+        return get_page("", 0, result_items);
+    }
+
     static std::string two_column_watch_next_results_get_comments_continuation_token(const Json::Value &tcwnr_json) {
         const Json::Value &results_json = tcwnr_json["results"];
         if(!results_json.isObject())