Revert back to recommending youtube videos based on related videos

2 files changed, 247 insertions, 206 deletions

diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 5a94326..affdb93 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -198,6 +198,9 @@ static std::string base64_url_decode(const std::string &data) {
 }
 
 namespace QuickMedia {
+    static Json::Value load_recommended_json(const char *plugin_name);
+    static void fill_recommended_items_from_json(const char *plugin_name, const Json::Value &recommended_json, BodyItems &body_items);
+
     enum class HistoryType {
         YOUTUBE,
         MANGA
@@ -207,10 +210,13 @@ namespace QuickMedia {
     public:
         HistoryPage(Program *program, Page *search_page, HistoryType history_type, bool local_thumbnail = false) :
             LazyFetchPage(program), search_page(search_page), history_type(history_type), local_thumbnail(local_thumbnail) {}
+
         const char* get_title() const override { return "History"; }
+
         PluginResult submit(const SubmitArgs &args, std::vector<Tab> &result_tabs) override {
             return search_page->submit(args, result_tabs);
         }
+
         PluginResult lazy_fetch(BodyItems &result_items) override {
             switch(history_type) {
                 case HistoryType::YOUTUBE:
@@ -222,6 +228,7 @@ namespace QuickMedia {
             }
             return PluginResult::OK;
         }
+
         bool reload_on_page_change() override { return true; }
         const char* get_bookmark_name() const override { return search_page->get_bookmark_name(); }
     private:
@@ -230,6 +237,28 @@ namespace QuickMedia {
         bool local_thumbnail;
     };
 
+    class RecommendedPage : public LazyFetchPage {
+    public:
+        RecommendedPage(Program *program, Page *search_page, const char *plugin_name) :
+            LazyFetchPage(program), search_page(search_page), plugin_name(plugin_name) {}
+
+        const char* get_title() const override { return "Recommended"; }
+
+        PluginResult submit(const SubmitArgs &args, std::vector<Tab> &result_tabs) override {
+            return search_page->submit(args, result_tabs);
+        }
+
+        PluginResult lazy_fetch(BodyItems &result_items) override {
+            fill_recommended_items_from_json(plugin_name, load_recommended_json(plugin_name), result_items);
+            return PluginResult::OK;
+        }
+
+        bool reload_on_page_change() override { return true; }
+    private:
+        Page *search_page;
+        const char *plugin_name;
+    };
+
     using OptionsPageHandler = std::function<void()>;
 
     class OptionsPage : public Page {
@@ -1182,7 +1211,7 @@ namespace QuickMedia {
                 tabs.push_back(Tab{create_body(false, true), std::make_unique<YoutubeSubscriptionsPage>(this), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
                 tabs.push_back(Tab{create_body(false, false), std::make_unique<YoutubeSearchPage>(this), create_search_bar("Search...", 100)});
 
-                auto recommended_page = std::make_unique<YoutubeRecommendedPage>(this);
+                auto recommended_page = std::make_unique<RecommendedPage>(this, tabs.back().page.get(), plugin_name);
                 tabs.push_back(Tab{create_body(false, true), std::move(recommended_page), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
 
                 auto history_body = create_body(false, true);
@@ -1296,13 +1325,6 @@ namespace QuickMedia {
         }
     }
 
-    enum class SearchSuggestionTab {
-        ALL,
-        HISTORY,
-        RECOMMENDED,
-        LOGIN
-    };
-
     static void fill_youtube_history_items_from_json(const Json::Value &history_json, BodyItems &history_items) {
         assert(history_json.isArray());
 
@@ -1377,6 +1399,186 @@ namespace QuickMedia {
         return json_result;
     }
 
+    static Path get_recommended_filepath(const char *plugin_name) {
+        Path video_history_dir = get_storage_dir().join("recommended");
+        if(create_directory_recursive(video_history_dir) != 0) {
+            std::string err_msg = "Failed to create recommended directory ";
+            err_msg += video_history_dir.data;
+            show_notification("QuickMedia", err_msg.c_str(), Urgency::CRITICAL);
+            exit(1);
+        }
+
+        Path video_history_filepath = video_history_dir;
+        return video_history_filepath.join(plugin_name).append(".json");
+    }
+
+    Json::Value load_recommended_json(const char *plugin_name) {
+        Path recommended_filepath = get_recommended_filepath(plugin_name);
+        Json::Value json_result;
+        if(!read_file_as_json(recommended_filepath, json_result) || !json_result.isObject())
+            json_result = Json::Value(Json::objectValue);
+        return json_result;
+    }
+
+    void fill_recommended_items_from_json(const char *plugin_name, const Json::Value &recommended_json, BodyItems &body_items) {
+        assert(recommended_json.isObject());
+
+        const int64_t recommendations_autodelete_period = 60*60*24*20; // 20 days
+        time_t time_now = time(NULL);
+        int num_items_deleted = 0;
+
+        std::vector<std::pair<std::string, Json::Value>> recommended_items(recommended_json.size());
+        /* TODO: Optimize member access */
+        for(auto &member_name : recommended_json.getMemberNames()) {
+            const Json::Value &recommended_item = recommended_json[member_name];
+            if(recommended_item.isObject()) {
+                const Json::Value &recommended_timestamp_json = recommended_item["recommended_timestamp"];
+                const Json::Value &watched_timestamp_json = recommended_item["watched_timestamp"];
+                if(watched_timestamp_json.isNumeric() && time_now - watched_timestamp_json.asInt64() >= recommendations_autodelete_period) {
+                    ++num_items_deleted;
+                } else if(recommended_timestamp_json.isNumeric() && time_now - recommended_timestamp_json.asInt64() >= recommendations_autodelete_period) {
+                    ++num_items_deleted;
+                } else if(recommended_timestamp_json.isNull() && watched_timestamp_json.isNull()) {
+                    ++num_items_deleted;
+                } else {
+                    recommended_items.push_back(std::make_pair(member_name, std::move(recommended_item)));
+                }
+            }
+        }
+
+        if(num_items_deleted > 0) {
+            // TODO: Is there a better way?
+            Json::Value new_recommendations(Json::objectValue);
+            for(auto &recommended : recommended_items) {
+                new_recommendations[recommended.first] = recommended.second;
+            }
+            fprintf(stderr, "Number of old recommendations to delete: %d\n", num_items_deleted);
+            save_json_to_file_atomic(get_recommended_filepath(plugin_name), new_recommendations);
+        }
+
+        /* TODO: Better algorithm for recommendations */
+        std::sort(recommended_items.begin(), recommended_items.end(), [](std::pair<std::string, Json::Value> &a, std::pair<std::string, Json::Value> &b) {
+            const Json::Value &a_timestamp_json = a.second["recommended_timestamp"];
+            const Json::Value &b_timestamp_json = b.second["recommended_timestamp"];
+            int64_t a_timestamp = 0;
+            int64_t b_timestamp = 0;
+            if(a_timestamp_json.isNumeric())
+                a_timestamp = a_timestamp_json.asInt64();
+            if(b_timestamp_json.isNumeric())
+                b_timestamp = b_timestamp_json.asInt64();
+
+            const Json::Value &a_recommended_count_json = a.second["recommended_count"];
+            const Json::Value &b_recommended_count_json = b.second["recommended_count"];
+            int64_t a_recommended_count = 0;
+            int64_t b_recommended_count = 0;
+            if(a_recommended_count_json.isNumeric())
+                a_recommended_count = a_recommended_count_json.asInt64();
+            if(b_recommended_count_json.isNumeric())
+                b_recommended_count = b_recommended_count_json.asInt64();
+
+            /* Put frequently recommended videos on top of recommendations. Each recommendation count is worth 5 minutes */
+            a_timestamp += (300 * a_recommended_count);
+            b_timestamp += (300 * b_recommended_count);
+
+            return a_timestamp > b_timestamp;
+        });
+
+        for(auto it = recommended_items.begin(); it != recommended_items.end(); ++it) {
+            const std::string &recommended_item_id = it->first;
+            const Json::Value &recommended_item = it->second;
+
+            int64_t watched_count = 0;
+            const Json::Value &watched_count_json = recommended_item["watched_count"];
+            if(watched_count_json.isNumeric())
+                watched_count = watched_count_json.asInt64();
+
+            /* TODO: Improve recommendations with some kind of algorithm. Videos we have seen should be recommended in some cases */
+            if(watched_count != 0)
+                continue;
+
+            const Json::Value &recommended_title_json = recommended_item["title"];
+            if(!recommended_title_json.isString())
+                continue;
+
+            auto body_item = BodyItem::create(recommended_title_json.asString());
+            body_item->url = "https://www.youtube.com/watch?v=" + recommended_item_id;
+            body_item->thumbnail_url = "https://img.youtube.com/vi/" + recommended_item_id + "/mqdefault.jpg";
+            body_item->thumbnail_size = mgl::vec2i(192, 108);
+            body_items.push_back(std::move(body_item));
+
+            // We dont want more than 150 recommendations
+            if(body_items.size() == 150)
+                break;
+        }
+
+        std::random_shuffle(body_items.begin(), body_items.end());
+    }
+
+    static void save_recommendations_from_related_videos(const char *plugin_name, const std::string &video_url, const std::string &video_title, const BodyItems &related_media_body_items) {
+        std::string video_id;
+        if(!youtube_url_extract_id(video_url, video_id)) {
+            std::string err_msg = "Failed to extract id of youtube url ";
+            err_msg += video_url;
+            err_msg + ", video wont be saved in recommendations";
+            show_notification("QuickMedia", err_msg.c_str(), Urgency::LOW);
+            return;
+        }
+
+        Json::Value recommended_json = load_recommended_json(plugin_name);
+        time_t time_now = time(NULL);
+
+        Json::Value &existing_recommended_json = recommended_json[video_id];
+        if(existing_recommended_json.isObject()) {
+            int64_t watched_count = 0;
+            Json::Value &watched_count_json = existing_recommended_json["watched_count"];
+            if(watched_count_json.isNumeric())
+                watched_count = watched_count_json.asInt64();
+            existing_recommended_json["watched_count"] = watched_count + 1;
+            existing_recommended_json["watched_timestamp"] = time_now;
+
+            save_json_to_file_atomic(get_recommended_filepath(plugin_name), recommended_json);
+            return;
+        } else {
+            Json::Value new_content_object(Json::objectValue);
+            new_content_object["title"] = video_title;
+            new_content_object["recommended_timestamp"] = time_now;
+            new_content_object["recommended_count"] = 1;
+            new_content_object["watched_count"] = 1;
+            new_content_object["watched_timestamp"] = time_now;
+            recommended_json[video_id] = std::move(new_content_object);
+        }
+
+        int saved_recommendation_count = 0;
+        for(const auto &body_item : related_media_body_items) {
+            std::string recommended_video_id;
+            if(youtube_url_extract_id(body_item->url, recommended_video_id)) {
+                Json::Value &existing_recommendation = recommended_json[recommended_video_id];
+                if(existing_recommendation.isObject()) {
+                    int64_t recommended_count = 0;
+                    Json::Value &count_json = existing_recommendation["recommended_count"];
+                    if(count_json.isNumeric())
+                        recommended_count = count_json.asInt64();
+                    existing_recommendation["recommended_count"] = recommended_count + 1;
+                    existing_recommendation["recommended_timestamp"] = time_now;
+                } else {
+                    Json::Value new_content_object(Json::objectValue);
+                    new_content_object["title"] = body_item->get_title();
+                    new_content_object["recommended_timestamp"] = time_now;
+                    new_content_object["recommended_count"] = 1;
+                    recommended_json[recommended_video_id] = std::move(new_content_object);
+                    saved_recommendation_count++;
+                    /* TODO: Save more than the first 3 video that hasn't been watched yet? */
+                    if(saved_recommendation_count == 3)
+                        break;
+                }
+            } else {
+                fprintf(stderr, "Failed to extract id of youtube url %s, video wont be saved in recommendations\n", video_url.c_str());
+            }
+        }
+
+        save_json_to_file_atomic(get_recommended_filepath(plugin_name), recommended_json);
+    }
+
     void Program::set_clipboard(const std::string &str) {
         window.set_clipboard(str);
     }
@@ -2709,6 +2911,10 @@ namespace QuickMedia {
         std::string youtube_video_id_dummy;
         const bool is_youtube = youtube_url_extract_id(video_page->get_url(), youtube_video_id_dummy);
         const bool is_matrix = strcmp(plugin_name, "matrix") == 0;
+        const bool is_youtube_plugin = strcmp(plugin_name, "youtube") == 0;
+
+        bool added_recommendations = false;
+        mgl::Clock time_watched_timer;
 
         idle_active_handler();
         video_player.reset();
@@ -2742,7 +2948,7 @@ namespace QuickMedia {
         int64_t youtube_audio_content_length = 0;
 
         std::string channel_url;
-        AsyncTask<void> video_tasks;
+        AsyncTask<void> related_videos_task;
         std::function<void(const char*)> video_event_callback;
         bool go_to_previous_page = false;
 
@@ -2945,17 +3151,16 @@ namespace QuickMedia {
                 if(video_page->autoplay_next_item())
                     return;
 
-                if(!is_resume_go_back) {
-                    std::string url = video_page->get_url();
-                    video_tasks = AsyncTask<void>([video_page, url]() {
-                        video_page->mark_watched();
-                    });
-                }
-
                 // TODO: Make this also work for other video plugins
                 if(strcmp(plugin_name, "youtube") != 0 || is_resume_go_back)
                     return;
 
+                std::string url = video_page->get_url();
+                related_videos_task = AsyncTask<void>([&related_videos, url, video_page]() {
+                    video_page->mark_watched();
+                    related_videos = video_page->get_related_media(url);
+                });
+
                 std::string video_id;
                 if(!youtube_url_extract_id(video_page->get_url(), video_id)) {
                     std::string err_msg = "Failed to extract id of youtube url ";
@@ -2992,6 +3197,8 @@ namespace QuickMedia {
             } else if(strcmp(event_name, "playback-restart") == 0) {
                 //video_player->set_paused(false);
             } else if(strcmp(event_name, "start-file") == 0) {
+                added_recommendations = false;
+                time_watched_timer.restart();
                 video_loaded = true;
                 if(video_page->is_local())
                     update_time_pos = true;
@@ -3082,13 +3289,19 @@ namespace QuickMedia {
                     load_video_error_check(std::to_string((int)resume_start_time));
                 } else if(pressed_keysym == XK_r && pressing_ctrl && !video_page->is_local()) {
                     bool cancelled = false;
-                    if(video_tasks.valid()) {
+                    if(related_videos_task.valid()) {
                         XUnmapWindow(disp, video_player_window);
                         XSync(disp, False);
                         XFlush(disp);
 
-                        TaskResult task_result = run_task_with_loading_screen([video_page, &related_videos]() {
-                            related_videos = video_page->get_related_media(video_page->get_url());
+                        TaskResult task_result = run_task_with_loading_screen([&]() {
+                            while(!program_is_dead_in_current_thread()) {
+                                if(related_videos_task.ready()) {
+                                    related_videos_task.get();
+                                    return true;
+                                }
+                                std::this_thread::sleep_for(std::chrono::milliseconds(10));
+                            }
                             return true;
                         });
 
@@ -3164,6 +3377,12 @@ namespace QuickMedia {
                 cursor_visible = true;
             }
 
+            /* Only save recommendations for the video if we have been watching it for 15 seconds */
+            if(is_youtube_plugin && video_loaded && !added_recommendations && time_watched_timer.get_elapsed_time_seconds() >= 15.0) {
+                added_recommendations = true;
+                save_recommendations_from_related_videos(plugin_name, video_page->get_url(), video_title, related_videos);
+            }
+
             VideoPlayer::Error update_err = video_player ? video_player->update() : VideoPlayer::Error::OK;
             if(update_err == VideoPlayer::Error::FAIL_TO_CONNECT_TIMEOUT) {
                 show_notification("QuickMedia", "Failed to connect to mpv ipc after 10 seconds", Urgency::CRITICAL);
@@ -3173,9 +3392,15 @@ namespace QuickMedia {
             } else if(update_err == VideoPlayer::Error::EXITED && video_player->exit_status == 0 && (!is_matrix || is_youtube)) {
                 std::string new_video_url;
 
-                if(video_tasks.valid()) {
-                    TaskResult task_result = run_task_with_loading_screen([video_page, &related_videos]() {
-                        related_videos = video_page->get_related_media(video_page->get_url());
+                if(related_videos_task.valid()) {
+                    TaskResult task_result = run_task_with_loading_screen([&]() {
+                        while(!program_is_dead_in_current_thread()) {
+                            if(related_videos_task.ready()) {
+                                related_videos_task.get();
+                                return true;
+                            }
+                            std::this_thread::sleep_for(std::chrono::milliseconds(10));
+                        }
                         return true;
                     });
 
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index a4212ae..e0d79c3 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -1775,190 +1775,6 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    PluginResult YoutubeRecommendedPage::get_page(const std::string&, int page, BodyItems &result_items) {
-        while(current_page < page) {
-            PluginResult plugin_result = search_get_continuation(continuation_token, result_items);
-            if(plugin_result != PluginResult::OK) return plugin_result;
-            ++current_page;
-        }
-        return PluginResult::OK;
-    }
-
-    PluginResult YoutubeRecommendedPage::search_get_continuation(const std::string &current_continuation_token, BodyItems &result_items) {
-        if(current_continuation_token.empty())
-            return PluginResult::OK;
-
-        std::string next_url = "https://www.youtube.com/?pbj=1&gl=US&hl=en&ctoken=" + current_continuation_token;
-
-        std::vector<CommandArg> additional_args = {
-            { "-H", "x-spf-referer: https://www.youtube.com/" },
-            { "-H", "x-youtube-client-name: 1" },
-            { "-H", "x-spf-previous: https://www.youtube.com/" },
-            { "-H", youtube_client_version },
-            { "-H", "referer: https://www.youtube.com/" }
-        };
-
-        std::vector<CommandArg> cookies = get_cookies();
-        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
-
-        Json::Value json_root;
-        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
-        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
-
-        if(!json_root.isArray())
-            return PluginResult::ERR;
-
-        std::string new_continuation_token;
-        for(const Json::Value &json_item : json_root) {
-            if(!json_item.isObject())
-                continue;
-
-            const Json::Value &response_json = json_item["response"];
-            if(!response_json.isObject())
-                continue;
-
-            const Json::Value &on_response_received_actions_json = response_json["onResponseReceivedActions"];
-            if(!on_response_received_actions_json.isArray())
-                continue;
-
-            for(const Json::Value &response_received_command : on_response_received_actions_json) {
-                if(!response_received_command.isObject())
-                    continue;
-
-                const Json::Value &append_continuation_items_action_json = response_received_command["appendContinuationItemsAction"];
-                if(!append_continuation_items_action_json.isObject())
-                    continue;
-
-                const Json::Value &continuation_items_json = append_continuation_items_action_json["continuationItems"];
-                if(!continuation_items_json.isArray())
-                    continue;
-
-                for(const Json::Value &content_item_json : continuation_items_json) {
-                    if(!content_item_json.isObject())
-                        continue;
-
-                    if(new_continuation_token.empty()) 
-                        new_continuation_token = item_section_renderer_get_continuation_token(content_item_json);
-                    
-                    const Json::Value &rich_item_renderer_json = content_item_json["richItemRenderer"];
-                    if(!rich_item_renderer_json.isObject())
-                        continue;
-
-                    const Json::Value &item_content_json = rich_item_renderer_json["content"];
-                    if(!item_content_json.isObject())
-                        continue;
-
-                    const Json::Value &video_renderer_json = item_content_json["videoRenderer"];
-                    if(!video_renderer_json.isObject())
-                        continue;
-
-                    auto body_item = parse_common_video_item(video_renderer_json, added_videos);
-                    if(body_item)
-                        result_items.push_back(std::move(body_item));
-                }
-            }
-        }
-
-        continuation_token = std::move(new_continuation_token);
-        return PluginResult::OK;
-    }
-
-    PluginResult YoutubeRecommendedPage::submit(const SubmitArgs &args, std::vector<Tab> &result_tabs) {
-        result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, args.url), nullptr});
-        return PluginResult::OK;
-    }
-
-    PluginResult YoutubeRecommendedPage::lazy_fetch(BodyItems &result_items) {
-        current_page = 0;
-        continuation_token.clear();
-        added_videos.clear();
-
-        std::vector<CommandArg> additional_args = {
-            { "-H", "x-youtube-client-name: 1" },
-            { "-H", youtube_client_version }
-        };
-
-        std::vector<CommandArg> cookies = get_cookies();
-        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
-
-        Json::Value json_root;
-        DownloadResult result = download_json(json_root, "https://www.youtube.com/?pbj=1&gl=US&hl=en", std::move(additional_args), true);
-        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
-
-        if(!json_root.isArray())
-            return PluginResult::ERR;
-
-        std::string new_continuation_token;
-        for(const Json::Value &json_item : json_root) {
-            if(!json_item.isObject())
-                continue;
-
-            const Json::Value &response_json = json_item["response"];
-            if(!response_json.isObject())
-                continue;
-
-            const Json::Value &contents_json = response_json["contents"];
-            if(!contents_json.isObject())
-                continue;
-
-            const Json::Value &tcbrr_json = contents_json["twoColumnBrowseResultsRenderer"];
-            if(!tcbrr_json.isObject())
-                continue;
-
-            const Json::Value &tabs_json = tcbrr_json["tabs"];
-            if(!tabs_json.isArray())
-                continue;
-
-            for(const Json::Value &tab_json : tabs_json) {
-                if(!tab_json.isObject())
-                    continue;
-
-                const Json::Value &tab_renderer_json = tab_json["tabRenderer"];
-                if(!tab_renderer_json.isObject())
-                    continue;
-                
-                const Json::Value &content_json = tab_renderer_json["content"];
-                if(!content_json.isObject())
-                    continue;
-
-                const Json::Value &rich_grid_renderer_json = content_json["richGridRenderer"];
-                if(!rich_grid_renderer_json.isObject())
-                    continue;
-
-                const Json::Value &contents2_json = rich_grid_renderer_json["contents"];
-                if(!contents2_json.isArray())
-                    continue;
-                
-                for(const Json::Value &content_item_json : contents2_json) {
-                    if(!content_item_json.isObject())
-                        continue;
-
-                    if(new_continuation_token.empty()) 
-                        new_continuation_token = item_section_renderer_get_continuation_token(content_item_json);
-                    
-                    const Json::Value &rich_item_renderer_json = content_item_json["richItemRenderer"];
-                    if(!rich_item_renderer_json.isObject())
-                        continue;
-
-                    const Json::Value &item_content_json = rich_item_renderer_json["content"];
-                    if(!item_content_json.isObject())
-                        continue;
-
-                    const Json::Value &video_renderer_json = item_content_json["videoRenderer"];
-                    if(!video_renderer_json.isObject())
-                        continue;
-
-                    auto body_item = parse_common_video_item(video_renderer_json, added_videos);
-                    if(body_item)
-                        result_items.push_back(std::move(body_item));
-                }
-            }
-        }
-
-        continuation_token = std::move(new_continuation_token);
-        return PluginResult::OK;
-    }
-
     PluginResult YoutubeRelatedVideosPage::submit(const SubmitArgs &args, std::vector<Tab> &result_tabs) {
         result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program, args.url), nullptr});
         return PluginResult::OK;