Youtube: do proper playlist pagination, show playlist videos views and uploaded date

2 files changed, 147 insertions, 31 deletions

diff --git a/src/plugins/Page.cpp b/src/plugins/Page.cpp
index 991f651..0e52ca4 100644
--- a/src/plugins/Page.cpp
+++ b/src/plugins/Page.cpp
@@ -19,6 +19,10 @@ namespace QuickMedia {
             return DownloadResult::OK;
         }
 
+        //FILE *f = fopen("data.json", "wb");
+        //fwrite(server_response.data(), 1, server_response.size(), f);
+        //fclose(f);
+
         Json::CharReaderBuilder json_builder;
         std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader());
         std::string json_errors;
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index 2e59a0b..d4211b3 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -192,6 +192,20 @@ namespace QuickMedia {
         return false;
     }
 
+    static bool youtube_url_extract_playlist_id(const std::string &youtube_url, std::string &playlist_id) {
+        size_t start_index = youtube_url.find("&list=");
+        if(start_index == std::string::npos)
+            return false;
+
+        start_index += 6;
+        size_t end_index = youtube_url.find("&", start_index);
+        if(end_index == std::string::npos)
+            end_index = youtube_url.size();
+
+        playlist_id = youtube_url.substr(start_index, end_index - start_index);
+        return true;
+    }
+
     static std::mutex cookies_mutex;
     static std::string cookies_filepath;
     static std::string api_key;
@@ -557,6 +571,7 @@ namespace QuickMedia {
         if(!title)
             title = std::move(headline);
 
+        std::optional<std::string> video_info = yt_json_get_text(video_item_json, "videoInfo");
         std::optional<std::string> date = yt_json_get_text(video_item_json, "publishedTimeText");
         std::optional<std::string> view_count_text = yt_json_get_text(video_item_json, "viewCountText");
         std::optional<std::string> owner_text = yt_json_get_text(video_item_json, "shortBylineText");
@@ -596,33 +611,43 @@ namespace QuickMedia {
         
         auto body_item = BodyItem::create(title.value());
         std::string desc;
-        if(view_count_text)
-            desc += view_count_text.value();
-        if(date) {
-            if(!desc.empty())
-                desc += " • ";
-            desc += date.value();
+        if(video_info) {
+            desc += video_info.value();
+        } else {
+            if(view_count_text)
+                desc += view_count_text.value();
+
+            if(date) {
+                if(!desc.empty())
+                    desc += " • ";
+                desc += date.value();
+            }
         }
+
         if(!scheduled_text.empty()) {
             if(!desc.empty())
                 desc += " • ";
             desc += scheduled_text;
         }
+
         if(length) {
             if(!desc.empty())
                 desc += '\n';
             desc += length.value();
         }
+
         if(video_is_live(video_item_json)) {
             if(!desc.empty())
                 desc += '\n';
             desc += "Live now";
         }
+
         if(owner_text) {
             if(!desc.empty())
                 desc += '\n';
             desc += owner_text.value();
         }
+
         /*if(description_snippet) {
             if(!desc.empty())
                 desc += '\n';
@@ -781,8 +806,15 @@ namespace QuickMedia {
         if(long_byline_text.empty())
             long_byline_text = yt_json_get_text(playlist_renderer_json, "shortBylineText").value_or("");
 
+        std::optional<std::string> published_time_text = yt_json_get_text(playlist_renderer_json, "publishedTimeText");
+
         std::string video_id;
         std::string description = std::move(long_byline_text);
+        if(published_time_text) {
+            if(!description.empty())
+                description += " • ";
+            description += published_time_text.value();
+        }
         if(!description.empty())
             description += '\n';
         description += video_count_text + " video" + (strcmp(video_count_text.c_str(), "1") == 0 ? "" : "s");
@@ -842,7 +874,7 @@ namespace QuickMedia {
             if(body_item)
                 result_items.push_back(std::move(body_item));
 
-            // TODO: Mix
+            // TODO: youtube mix
             //body_item = parse_playlist_renderer(item_json["radioRenderer"]);
             //if(body_item)
             //    result_items.push_back(std::move(body_item));
@@ -870,7 +902,7 @@ namespace QuickMedia {
             if(body_item)
                 result_items.push_back(std::move(body_item));
 
-            // TODO: Mix
+            // TODO: youtube mix
             //body_item = parse_playlist_renderer(content_item_json["radioRenderer"]);
             //if(body_item)
             //    result_items.push_back(std::move(body_item));
@@ -912,6 +944,27 @@ namespace QuickMedia {
         return "";
     }
 
+    static void parse_playlist_video_list(const Json::Value &playlist_video_list_json, const char *list_name, std::string &continuation_token, std::unordered_set<std::string> &added_videos, BodyItems &body_items) {
+        if(!playlist_video_list_json.isObject())
+            return;
+
+        const Json::Value &contents_json = playlist_video_list_json[list_name];
+        if(!contents_json.isArray())
+            return;
+
+        for(const Json::Value &content_json : contents_json) {
+            if(!content_json.isObject())
+                continue;
+
+            if(continuation_token.empty())
+                continuation_token = item_section_renderer_get_continuation_token(content_json);
+
+            auto body_item = parse_common_video_item(content_json["playlistVideoRenderer"], added_videos);
+            if(body_item)
+                body_items.push_back(std::move(body_item));
+        }
+    }
+
     static void parse_channel_videos(const Json::Value &json_root, std::string &continuation_token, std::unordered_set<std::string> &added_videos, std::string &browse_id, BodyItems &body_items) {
         if(!json_root.isObject())
             return;
@@ -1014,6 +1067,8 @@ namespace QuickMedia {
                     if(!content_json.isObject())
                         continue;
 
+                    parse_playlist_video_list(content_json["playlistVideoListRenderer"], "contents", continuation_token, added_videos, body_items);
+
                     const Json::Value &grid_renderer_json = content_json["gridRenderer"];
                     if(!grid_renderer_json.isObject())
                         continue;
@@ -1214,7 +1269,7 @@ namespace QuickMedia {
             // TODO: Make all pages (for all services) lazy fetch in a similar manner!
             YoutubeChannelPage::create_each_type(program, args.url, "", args.title, result_tabs);
         } else if(strstr(args.url.c_str(), "&list=")) {
-            result_tabs.push_back(Tab{create_body(false, true), std::make_unique<YoutubePlaylistPage>(program, args.url, args.title), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+            result_tabs.push_back(Tab{create_body(false, true), std::make_unique<YoutubePlaylistPage>(program, args.url, args.title), create_search_bar("Filter...", SEARCH_DELAY_FILTER)});
         } else {
             result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, args.url), nullptr});
         }
@@ -1998,7 +2053,7 @@ namespace QuickMedia {
             return PluginResult::OK;
 
         if(strstr(args.url.c_str(), "&list=")) {
-            result_tabs.push_back(Tab{create_body(false, true), std::make_unique<YoutubePlaylistPage>(program, args.url, args.title), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+            result_tabs.push_back(Tab{create_body(false, true), std::make_unique<YoutubePlaylistPage>(program, args.url, args.title), create_search_bar("Filter...", SEARCH_DELAY_FILTER)});
         } else {
             result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, args.url), nullptr});
         }
@@ -2257,20 +2312,15 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    PluginResult YoutubePlaylistPage::submit(const SubmitArgs &args, std::vector<Tab> &result_tabs) {
-        result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, args.url, true, true), nullptr});
-        return PluginResult::OK;
-    }
-
     static void playlist_get_items(const Json::Value &json_root, std::unordered_set<std::string> &added_videos, BodyItems &result_items) {
         if(!json_root.isObject())
             return;
 
-        const Json::Value &response_json = json_root["response"];
-        if(!response_json.isObject())
-            return;
+        const Json::Value *response_json = &json_root["response"];
+        if(!response_json->isObject())
+            response_json = &json_root;
 
-        const Json::Value &contents_json = response_json["contents"];
+        const Json::Value &contents_json = (*response_json)["contents"];
         if(!contents_json.isObject())
             return;
 
@@ -2301,35 +2351,97 @@ namespace QuickMedia {
         }
     }
 
-    PluginResult YoutubePlaylistPage::lazy_fetch(BodyItems &result_items) {
+    YoutubePlaylistPage::YoutubePlaylistPage(Program *program, const std::string &url, std::string title) :
+        LazyFetchPage(program), title(std::move(title))
+    {
+        if(!youtube_url_extract_playlist_id(url, playlist_id))
+            fprintf(stderr, "Error: failed to extract playlist id from url: %s\n", url.c_str());
+    }
+
+    PluginResult YoutubePlaylistPage::get_page(const std::string&, int, BodyItems &result_items) {
+        if(reached_end)
+            return PluginResult::OK;
+
+        std::vector<CommandArg> cookies = get_cookies();
+        std::string next_url = "https://www.youtube.com/youtubei/v1/browse?key=" + url_param_encode(api_key) + "&gl=US&hl=en&prettyPrint=false";
+
+        Json::Value client_json(Json::objectValue);
+        client_json["hl"] = "en";
+        client_json["gl"] = "US";
+        client_json["deviceMake"] = "";
+        client_json["deviceModel"] = "";
+        client_json["userAgent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36";
+        client_json["clientName"] = "WEB";
+        client_json["clientVersion"] = "2.20210622.10.00";
+        client_json["osName"] = "X11";
+        client_json["osVersion"] = "";
+        client_json["originalUrl"] = "https://www.youtube.com/playlist?list=" + playlist_id;
+
+        Json::Value context_json(Json::objectValue);
+        context_json["client"] = std::move(client_json);
+
+        Json::Value request_json(Json::objectValue);
+        request_json["context"] = std::move(context_json);
+        if(continuation_token.empty())
+            request_json["browseId"] = "VL" + playlist_id;
+        else
+            request_json["continuation"] = continuation_token;
+
+        Json::StreamWriterBuilder json_builder;
+        json_builder["commentStyle"] = "None";
+        json_builder["indentation"] = "";
+
         std::vector<CommandArg> additional_args = {
+            { "-H", "content-type: application/json" },
             { "-H", "x-youtube-client-name: 1" },
             { "-H", youtube_client_version },
+            { "--data-raw", Json::writeString(json_builder, request_json) }
         };
 
-        std::vector<CommandArg> cookies = get_cookies();
         additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         Json::Value json_root;
-        DownloadResult download_result = download_json(json_root, url + "&pbj=1&gl=US&hl=en", additional_args, true);
-        if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result);
+        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
+        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
+
+        if(!json_root.isObject())
+            return PluginResult::ERR;
 
         std::unordered_set<std::string> added_videos;
-        if(json_root.isObject()) {
-            playlist_get_items(json_root, added_videos, result_items);
-            return PluginResult::OK;
-        }
+        std::string browse_id;
+        continuation_token.clear();
 
-        if(!json_root.isArray())
-            return PluginResult::OK;
+        const Json::Value &on_response_received_actions_json = json_root["onResponseReceivedActions"];
+        if(on_response_received_actions_json.isArray()) {
+            for(const Json::Value &json_item : on_response_received_actions_json) {
+                if(!json_item.isObject())
+                    continue;
 
-        for(const Json::Value &json_item : json_root) {
-            playlist_get_items(json_item, added_videos, result_items);
+                const Json::Value &append_continuation_items_action_json = json_item["appendContinuationItemsAction"];
+                if(!append_continuation_items_action_json.isObject())
+                    continue;
+                
+                parse_playlist_video_list(append_continuation_items_action_json, "continuationItems", continuation_token, added_videos, result_items);
+            }
+        } else {
+            parse_channel_videos(json_root, continuation_token, added_videos, browse_id, result_items);
         }
 
+        if(continuation_token.empty())
+            reached_end = true;
+
         return PluginResult::OK;
     }
 
+    PluginResult YoutubePlaylistPage::submit(const SubmitArgs &args, std::vector<Tab> &result_tabs) {
+        result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, args.url, true, true), nullptr});
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubePlaylistPage::lazy_fetch(BodyItems &result_items) {
+        return get_page("", 0, result_items);
+    }
+
     static std::string two_column_watch_next_results_get_comments_continuation_token(const Json::Value &tcwnr_json) {
         const Json::Value &results_json = tcwnr_json["results"];
         if(!results_json.isObject())