Show youtube recommendations instead of local recommendations from related videos

1 files changed, 232 insertions, 18 deletions

diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index 09568f6..0f5e807 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -339,12 +339,6 @@ namespace QuickMedia {
         return (c >= 8 && c <= 13) || c == ' ';
     }
 
-    static void remove_cookies_file_at_exit() {
-        std::lock_guard<std::mutex> lock(cookies_mutex);
-        if(!cookies_filepath.empty())
-            remove(cookies_filepath.c_str());
-    }
-
     // TODO: Cache this and redownload it when a network request fails with this api key?
     static std::string youtube_page_find_api_key() {
         size_t api_key_index;
@@ -387,14 +381,13 @@ namespace QuickMedia {
     static std::vector<CommandArg> get_cookies() {
         std::lock_guard<std::mutex> lock(cookies_mutex);
         if(cookies_filepath.empty()) {
-            char filename[] = "/tmp/quickmedia.youtube.cookie.XXXXXX";
-            int fd = mkstemp(filename);
-            if(fd == -1)
-                return {};
-            close(fd);
+            Path cookies_filepath_p;
+            if(get_cookies_filepath(cookies_filepath_p, "youtube") != 0) {
+                show_notification("QuickMedia", "Failed to create youtube cookies file", Urgency::CRITICAL);
+                abort();
+            }
 
-            cookies_filepath = filename;
-            atexit(remove_cookies_file_at_exit);
+            cookies_filepath = cookies_filepath_p.data;
 
             // TODO: Re-enable this if the api key ever changes in the future
             #if 0
@@ -403,10 +396,14 @@ namespace QuickMedia {
             api_key = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
             #endif
 
-            // TODO: Is there any way to bypass this? this is needed to set VISITOR_INFO1_LIVE which is required to read comments
-            const char *args[] = { "curl", "-I", "-s", "-b", cookies_filepath.c_str(), "-c", cookies_filepath.c_str(), "https://www.youtube.com/subscription_manager?disable_polymer=1", nullptr };
-            if(exec_program(args, nullptr, nullptr) != 0)
-                fprintf(stderr, "Failed to fetch cookies to view youtube comments\n");
+            if(get_file_type(cookies_filepath_p) != FileType::REGULAR) {
+                // TODO: Is there any way to bypass this? this is needed to set VISITOR_INFO1_LIVE which is required to read comments
+                const char *args[] = { "curl", "-I", "-s", "-b", cookies_filepath.c_str(), "-c", cookies_filepath.c_str(), "https://www.youtube.com/subscription_manager?disable_polymer=1", nullptr };
+                if(exec_program(args, nullptr, nullptr) != 0) {
+                    show_notification("QuickMedia", "Failed to fetch cookies to view youtube comments", Urgency::CRITICAL);
+                    abort();
+                }
+            }
         }
 
         return {
@@ -1451,6 +1448,191 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
+    PluginResult YoutubeRecommendedPage::get_page(const std::string&, int page, BodyItems &result_items) {
+        while(current_page < page) {
+            PluginResult plugin_result = search_get_continuation(continuation_token, result_items);
+            if(plugin_result != PluginResult::OK) return plugin_result;
+            ++current_page;
+        }
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubeRecommendedPage::search_get_continuation(const std::string &current_continuation_token, BodyItems &result_items) {
+        std::string next_url = "https://www.youtube.com/?pbj=1&ctoken=" + current_continuation_token;
+
+        std::vector<CommandArg> additional_args = {
+            { "-H", "x-spf-referer: https://www.youtube.com/" },
+            { "-H", "x-youtube-client-name: 1" },
+            { "-H", "x-spf-previous: https://www.youtube.com/" },
+            { "-H", "x-youtube-client-version: 2.20200626.03.00" },
+            { "-H", "referer: https://www.youtube.com/" }
+        };
+
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+
+        Json::Value json_root;
+        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
+        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
+
+        if(!json_root.isArray())
+            return PluginResult::ERR;
+
+        std::string new_continuation_token;
+        for(const Json::Value &json_item : json_root) {
+            if(!json_item.isObject())
+                continue;
+
+            const Json::Value &response_json = json_item["response"];
+            if(!response_json.isObject())
+                continue;
+
+            const Json::Value &on_response_received_actions_json = response_json["onResponseReceivedActions"];
+            if(!on_response_received_actions_json.isArray())
+                continue;
+
+            for(const Json::Value &response_received_command : on_response_received_actions_json) {
+                if(!response_received_command.isObject())
+                    continue;
+
+                const Json::Value &append_continuation_items_action_json = response_received_command["appendContinuationItemsAction"];
+                if(!append_continuation_items_action_json.isObject())
+                    continue;
+
+                const Json::Value &continuation_items_json = append_continuation_items_action_json["continuationItems"];
+                if(!continuation_items_json.isArray())
+                    continue;
+
+                for(const Json::Value &content_item_json : continuation_items_json) {
+                    if(!content_item_json.isObject())
+                        continue;
+
+                    if(new_continuation_token.empty()) 
+                        new_continuation_token = item_section_renderer_get_continuation_token(content_item_json);
+                    
+                    const Json::Value &rich_item_renderer_json = content_item_json["richItemRenderer"];
+                    if(!rich_item_renderer_json.isObject())
+                        continue;
+
+                    const Json::Value &item_content_json = rich_item_renderer_json["content"];
+                    if(!item_content_json.isObject())
+                        continue;
+
+                    const Json::Value &video_renderer_json = item_content_json["videoRenderer"];
+                    if(!video_renderer_json.isObject())
+                        continue;
+
+                    auto body_item = parse_common_video_item(video_renderer_json, added_videos);
+                    if(body_item)
+                        result_items.push_back(std::move(body_item));
+                }
+            }
+        }
+
+        if(!new_continuation_token.empty())
+            continuation_token = std::move(new_continuation_token);
+
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubeRecommendedPage::submit(const std::string&, const std::string &url, std::vector<Tab> &result_tabs) {
+        result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, url), nullptr});
+        return PluginResult::OK;
+    }
+
+    PluginResult YoutubeRecommendedPage::lazy_fetch(BodyItems &result_items) {
+        current_page = 0;
+        continuation_token.clear();
+        added_videos.clear();
+
+        std::vector<CommandArg> additional_args = {
+            { "-H", "x-youtube-client-name: 1" },
+            { "-H", "x-youtube-client-version: 2.20200626.03.00" }
+        };
+
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+
+        Json::Value json_root;
+        DownloadResult result = download_json(json_root, "https://www.youtube.com/?pbj=1", std::move(additional_args), true);
+        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
+
+        if(!json_root.isArray())
+            return PluginResult::ERR;
+
+        std::string new_continuation_token;
+        for(const Json::Value &json_item : json_root) {
+            if(!json_item.isObject())
+                continue;
+
+            const Json::Value &response_json = json_item["response"];
+            if(!response_json.isObject())
+                continue;
+
+            const Json::Value &contents_json = response_json["contents"];
+            if(!contents_json.isObject())
+                continue;
+
+            const Json::Value &tcbrr_json = contents_json["twoColumnBrowseResultsRenderer"];
+            if(!tcbrr_json.isObject())
+                continue;
+
+            const Json::Value &tabs_json = tcbrr_json["tabs"];
+            if(!tabs_json.isArray())
+                continue;
+
+            for(const Json::Value &tab_json : tabs_json) {
+                if(!tab_json.isObject())
+                    continue;
+
+                const Json::Value &tab_renderer_json = tab_json["tabRenderer"];
+                if(!tab_renderer_json.isObject())
+                    continue;
+                
+                const Json::Value &content_json = tab_renderer_json["content"];
+                if(!content_json.isObject())
+                    continue;
+
+                const Json::Value &rich_grid_renderer_json = content_json["richGridRenderer"];
+                if(!rich_grid_renderer_json.isObject())
+                    continue;
+
+                const Json::Value &contents2_json = rich_grid_renderer_json["contents"];
+                if(!contents2_json.isArray())
+                    continue;
+                
+                for(const Json::Value &content_item_json : contents2_json) {
+                    if(!content_item_json.isObject())
+                        continue;
+
+                    if(new_continuation_token.empty()) 
+                        new_continuation_token = item_section_renderer_get_continuation_token(content_item_json);
+                    
+                    const Json::Value &rich_item_renderer_json = content_item_json["richItemRenderer"];
+                    if(!rich_item_renderer_json.isObject())
+                        continue;
+
+                    const Json::Value &item_content_json = rich_item_renderer_json["content"];
+                    if(!item_content_json.isObject())
+                        continue;
+
+                    const Json::Value &video_renderer_json = item_content_json["videoRenderer"];
+                    if(!video_renderer_json.isObject())
+                        continue;
+
+                    auto body_item = parse_common_video_item(video_renderer_json, added_videos);
+                    if(body_item)
+                        result_items.push_back(std::move(body_item));
+                }
+            }
+        }
+
+        if(!new_continuation_token.empty())
+            continuation_token = std::move(new_continuation_token);
+
+        return PluginResult::OK;
+    }
+
     PluginResult YoutubeRelatedVideosPage::submit(const std::string&, const std::string &url, std::vector<Tab> &result_tabs) {
         result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, url), nullptr});
         return PluginResult::OK;
@@ -1512,8 +1694,8 @@ namespace QuickMedia {
             if(!json_item.isObject())
                 continue;
 
+            const Json::Value &player_response_json = json_item["playerResponse"];
             if(channel_url.empty()) {
-                const Json::Value &player_response_json = json_item["playerResponse"];
                 if(player_response_json.isObject()) {
                     const Json::Value &video_details_json = player_response_json["videoDetails"];
                     if(video_details_json.isObject()) {
@@ -1530,6 +1712,38 @@ namespace QuickMedia {
                     xsrf_token = xsrf_token_json.asString();
             }
 
+            if(player_response_json.isObject()) {
+                const Json::Value &playback_tracing_json = player_response_json["playbackTracking"];
+                if(playback_tracing_json.isObject()) {
+                    if(playback_url.empty()) {
+                        const Json::Value &video_stats_playback_url_json = playback_tracing_json["videostatsPlaybackUrl"];
+                        if(video_stats_playback_url_json.isObject()) {
+                            const Json::Value &base_url_json = video_stats_playback_url_json["baseUrl"];
+                            if(base_url_json.isString())
+                                playback_url = base_url_json.asString();
+                        }
+                    }
+
+                    if(watchtime_url.empty()) {
+                        const Json::Value &video_stats_watchtime_url_json = playback_tracing_json["videostatsWatchtimeUrl"];
+                        if(video_stats_watchtime_url_json.isObject()) {
+                            const Json::Value &base_url_json = video_stats_watchtime_url_json["baseUrl"];
+                            if(base_url_json.isString())
+                                watchtime_url = base_url_json.asString();
+                        }
+                    }
+
+                    if(tracking_url.empty()) {
+                        const Json::Value &p_tracking_url_json = playback_tracing_json["ptrackingUrl"];
+                        if(p_tracking_url_json.isObject()) {
+                            const Json::Value &base_url_json = p_tracking_url_json["baseUrl"];
+                            if(base_url_json.isString())
+                                tracking_url = base_url_json.asString();
+                        }
+                    }
+                }
+            }
+
             const Json::Value &response_json = json_item["response"];
             if(!response_json.isObject())
                 continue;