Make get_related_videos async, readd mark-watched

8 files changed, 208 insertions, 119 deletions

diff --git a/TODO b/TODO
index c397294..f5a46e1 100644
--- a/TODO
+++ b/TODO
@@ -158,7 +158,6 @@ Do not set fps to monitor hz if no key is pressed and cursor is not moving (for
 Ctrl+arrow key to move to previous/next video.
 Add keybinding to view file-manager images in fullscreen to preview them. Also add keybinding to create/delete directories.
 Completely remove youtube-dl dependency (or at least for downloading videos/music).
-Readd mark-watched.
 Add loading of english subtitles for youtube.
 Cancel search when new search happens.
 Set mpv title to the youtube video title (that we got from GET watch?v when fetching url).
diff --git a/plugins/MediaGeneric.hpp b/plugins/MediaGeneric.hpp
index 102e315..a9d1e2a 100644
--- a/plugins/MediaGeneric.hpp
+++ b/plugins/MediaGeneric.hpp
@@ -82,7 +82,7 @@ namespace QuickMedia {
     public:
         MediaGenericVideoPage(Program *program, MediaGenericSearchPage *search_page, const std::string &url) : VideoPage(program, url), search_page(search_page) {}
         const char* get_title() const override { return ""; }
-        BodyItems get_related_media(const std::string &url, std::string &channel_url) override;
+        BodyItems get_related_media(const std::string &url) override;
         std::unique_ptr<Page> create_search_page(Program *program, int &search_delay) override;
         std::unique_ptr<RelatedVideosPage> create_related_videos_page(Program *program) override;
         std::unique_ptr<Page> create_channels_page(Program*, const std::string&) override {
diff --git a/plugins/Page.hpp b/plugins/Page.hpp
index 0c5b093..0654fa7 100644
--- a/plugins/Page.hpp
+++ b/plugins/Page.hpp
@@ -105,7 +105,7 @@ namespace QuickMedia {
         VideoPage(Program *program, std::string url) : Page(program), url(std::move(url)) {}
         virtual PageTypez get_type() const override { return PageTypez::VIDEO; }
         virtual bool autoplay_next_item() { return false; }
-        virtual BodyItems get_related_media(const std::string &url, std::string &channel_url) { (void)url; (void)channel_url; return {}; }
+        virtual BodyItems get_related_media(const std::string &url) { (void)url; return {}; }
         virtual std::unique_ptr<Page> create_search_page(Program *program, int &search_delay) { (void)program; (void)search_delay; return nullptr; }
         virtual std::unique_ptr<Page> create_comments_page(Program *program) { (void)program; return nullptr; }
         // Return nullptr if the service doesn't support related videos page
@@ -124,7 +124,8 @@ namespace QuickMedia {
         virtual std::string get_audio_url() { return ""; }
         virtual std::string url_get_playable_url(const std::string &url) { return url; }
         virtual bool video_should_be_skipped(const std::string &url) { (void)url; return false; }
-        virtual PluginResult load() { return PluginResult::OK; }
+        virtual PluginResult load(std::string &channel_url) { (void)channel_url; return PluginResult::OK; }
+        virtual void mark_watched() {};
     protected:
         std::string url;
     };
diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp
index dea6cc2..20c04fc 100644
--- a/plugins/Youtube.hpp
+++ b/plugins/Youtube.hpp
@@ -133,14 +133,15 @@ namespace QuickMedia {
     public:
         YoutubeVideoPage(Program *program, std::string url) : VideoPage(program, std::move(url)) {}
         const char* get_title() const override { return ""; }
-        BodyItems get_related_media(const std::string &url, std::string &channel_url) override;
+        BodyItems get_related_media(const std::string &url) override;
         std::unique_ptr<Page> create_search_page(Program *program, int &search_delay) override;
         std::unique_ptr<Page> create_comments_page(Program *program) override;
         std::unique_ptr<RelatedVideosPage> create_related_videos_page(Program *program) override;
         std::unique_ptr<Page> create_channels_page(Program *program, const std::string &channel_url) override;
         std::string get_video_url(int max_height, bool &has_embedded_audio) override;
         std::string get_audio_url() override;
-        PluginResult load() override;
+        PluginResult load(std::string &channel_url) override;
+        void mark_watched() override;
     private:
         void parse_format(const Json::Value &format_json, bool is_adaptive);
         void parse_formats(const Json::Value &streaming_data_json);
diff --git a/plugins/youtube/Signature.hpp b/plugins/youtube/Signature.hpp
index 7456615..8c782f9 100644
--- a/plugins/youtube/Signature.hpp
+++ b/plugins/youtube/Signature.hpp
@@ -33,6 +33,8 @@ namespace QuickMedia {
         bool js_code_to_operations(const std::string &function_body_str, const std::string &var_body_str, std::vector<DecryptFuncCall> &new_func_calls, std::map<std::string, DecryptFunction> &new_func_decls);
         int update_decrypt_function();
     private:
+        // TODO: Remove this task and instead add the network download task to a queue and add a poll function to check if it has finished downloading
+        // or if it needs to be updated.
         AsyncTask<void> poll_task;
         std::mutex update_signature_mutex;
         std::string decryption_function;
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 7479299..4b78117 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -2362,8 +2362,6 @@ namespace QuickMedia {
         if(video_page->autoplay_next_item() && play_index + 1 >= 0 && play_index + 1 < (int)next_play_items.size())
             related_videos.insert(related_videos.end(), next_play_items.begin() + play_index + 1, next_play_items.end());
 
-        std::string channel_url;
-
         sf::WindowHandle video_player_window = None;
         auto on_window_create = [this, &video_player_window](sf::WindowHandle _video_player_window) mutable {
             video_player_window = _video_player_window;
@@ -2371,11 +2369,18 @@ namespace QuickMedia {
             XSync(disp, False);
         };
 
+        std::string channel_url;
+        AsyncTask<BodyItems> video_tasks;
         std::function<void(const char*)> video_event_callback;
 
-        auto load_video_error_check = [this, &related_videos, &channel_url, &video_title, &video_player, previous_page, &time_watched_timer, &video_loaded, video_page, &video_event_callback, &on_window_create, &video_player_window, is_youtube, is_matrix, download_if_streaming_fails](bool resume_video) mutable {
-            TaskResult load_result = run_task_with_loading_screen([video_page]() {
-                return video_page->load() == PluginResult::OK;
+        auto load_video_error_check = [this, &video_title, &video_tasks, &video_player, &channel_url, previous_page, &time_watched_timer, &video_loaded, video_page, &video_event_callback, &on_window_create, &video_player_window, is_youtube, is_matrix, download_if_streaming_fails](bool resume_video) mutable {
+            video_player.reset();
+            channel_url.clear();
+            video_loaded = false;
+            video_player_window = None;
+            
+            TaskResult load_result = run_task_with_loading_screen([video_page, &channel_url]() {
+                return video_page->load(channel_url) == PluginResult::OK;
             });
 
             if(load_result == TaskResult::CANCEL) {
@@ -2471,33 +2476,26 @@ namespace QuickMedia {
             }
             
             time_watched_timer.restart();
-            video_loaded = false;
-            video_player_window = None;
-            watched_videos.insert(video_url);
+            watched_videos.insert(video_page->get_url());
 
             video_player = std::make_unique<VideoPlayer>(is_audio_only, use_system_mpv_config, resume_video, is_matrix && !is_youtube, video_event_callback, on_window_create, resources_root, largest_monitor_height);
             VideoPlayer::Error err = video_player->load_video(video_url.c_str(), audio_url.c_str(), window.getSystemHandle(), plugin_name, video_title);
             if(err != VideoPlayer::Error::OK) {
                 std::string err_msg = "Failed to play url: ";
-                err_msg += video_url;
+                err_msg += video_page->get_url();
                 show_notification("QuickMedia", err_msg.c_str(), Urgency::CRITICAL);
                 current_page = previous_page;
             } else {
                 if(video_page->autoplay_next_item())
                     return;
 
-                channel_url.clear();
-                TaskResult load_related_media_result = run_task_with_loading_screen([video_page, &related_videos, &channel_url]{
-                    // TODO: Do async
-                    related_videos = video_page->get_related_media(video_page->get_url(), channel_url);
-                    return true;
+                std::string url = video_page->get_url();
+                video_tasks = AsyncTask<BodyItems>([video_page, url]() {
+                    BodyItems related_videos = video_page->get_related_media(url);
+                    video_page->mark_watched();
+                    return related_videos;
                 });
 
-                if(load_related_media_result == TaskResult::CANCEL) {
-                    current_page = previous_page;
-                    return;
-                }
-
                 // TODO: Make this also work for other video plugins
                 if(strcmp(plugin_name, "youtube") != 0 || resume_video)
                     return;
@@ -2622,62 +2620,89 @@ namespace QuickMedia {
                 } else if(pressed_keysym == XK_s && pressing_ctrl) {
                     video_page_download_video(video_page->get_url(), !is_matrix || is_youtube, video_player_window);
                 } else if(pressed_keysym == XK_r && pressing_ctrl) {
-                    if(!cursor_visible)
-                        window.setMouseCursorVisible(true);
-                    cursor_visible = true;
-
-                    int search_delay = 0;
-                    auto search_page = video_page->create_search_page(this, search_delay);
-                    auto comments_page = video_page->create_comments_page(this);
-                    auto related_videos_page = video_page->create_related_videos_page(this);
-                    auto channels_page = video_page->create_channels_page(this, channel_url);
-                    if(search_page || related_videos_page || channels_page) {
+                    bool cancelled = false;
+                    if(video_tasks.valid()) {
                         XUnmapWindow(disp, video_player_window);
                         XSync(disp, False);
 
-                        std::vector<Tab> tabs;
-                        if(search_page) {
-                            tabs.push_back(Tab{create_body(false, true), std::move(search_page), create_search_bar("Search...", search_delay)});
-                        }
-                        if(comments_page) {
-                            tabs.push_back(Tab{create_body(), std::move(comments_page), nullptr});
-                        }
-                        if(related_videos_page) {
-                            auto related_videos_body = create_body(false, true);
-                            related_videos_body->items = related_videos;
-                            tabs.push_back(Tab{std::move(related_videos_body), std::move(related_videos_page), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
-                        }
-                        if(channels_page) {
-                            tabs.push_back(Tab{create_body(false, true), std::move(channels_page), create_search_bar("Search...", 350)});
-                        }
+                        TaskResult task_result = run_task_with_loading_screen([&video_tasks, &related_videos]() {
+                            while(true) {
+                                if(program_is_dead_in_current_thread())
+                                    return false;
 
-                        bool page_changed = false;
-                        page_loop(tabs, 1, [this, &video_player, &page_changed](const std::vector<Tab> &new_tabs) {
-                            if(!page_changed && new_tabs.size() == 1 && new_tabs[0].page->get_type() == PageTypez::VIDEO) {
-                                window.setMouseCursorVisible(true);
-                                if(video_player) {
-                                    video_player->quit_and_save_watch_later();
-                                    while(true) {
-                                        VideoPlayer::Error update_err = video_player->update();
-                                        if(update_err != VideoPlayer::Error::OK || !window.isOpen() || current_page == PageType::EXIT)
-                                            break;
-                                        std::this_thread::sleep_for(std::chrono::milliseconds(20));
-                                    }
-                                    video_player.reset();
+                                if(video_tasks.ready()) {
+                                    related_videos = video_tasks.get();
+                                    return true;
                                 }
-                                page_changed = true;
+                                std::this_thread::sleep_for(std::chrono::milliseconds(50));
                             }
                         });
 
-                        if(!window.isOpen() || current_page == PageType::EXIT)
-                            return;
+                        XMapWindow(disp, video_player_window);
+                        XSync(disp, False);
 
-                        if(page_changed) {
-                            current_page = PageType::VIDEO_CONTENT;
-                            load_video_error_check(true);
-                        } else {
-                            XMapWindow(disp, video_player_window);
+                        if(task_result == TaskResult::CANCEL || task_result == TaskResult::FALSE)
+                            cancelled = true;
+                    }
+
+                    if(!cancelled) {
+                        if(!cursor_visible)
+                            window.setMouseCursorVisible(true);
+                        cursor_visible = true;
+
+                        int search_delay = 0;
+                        auto search_page = video_page->create_search_page(this, search_delay);
+                        auto comments_page = video_page->create_comments_page(this);
+                        auto related_videos_page = video_page->create_related_videos_page(this);
+                        auto channels_page = video_page->create_channels_page(this, channel_url);
+                        if(search_page || related_videos_page || channels_page) {
+                            XUnmapWindow(disp, video_player_window);
                             XSync(disp, False);
+
+                            std::vector<Tab> tabs;
+                            if(search_page) {
+                                tabs.push_back(Tab{create_body(false, true), std::move(search_page), create_search_bar("Search...", search_delay)});
+                            }
+                            if(comments_page) {
+                                tabs.push_back(Tab{create_body(), std::move(comments_page), nullptr});
+                            }
+                            if(related_videos_page) {
+                                auto related_videos_body = create_body(false, true);
+                                related_videos_body->items = related_videos;
+                                tabs.push_back(Tab{std::move(related_videos_body), std::move(related_videos_page), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+                            }
+                            if(channels_page) {
+                                tabs.push_back(Tab{create_body(false, true), std::move(channels_page), create_search_bar("Search...", 350)});
+                            }
+
+                            bool page_changed = false;
+                            page_loop(tabs, 1, [this, &video_player, &page_changed](const std::vector<Tab> &new_tabs) {
+                                if(!page_changed && new_tabs.size() == 1 && new_tabs[0].page->get_type() == PageTypez::VIDEO) {
+                                    window.setMouseCursorVisible(true);
+                                    if(video_player) {
+                                        video_player->quit_and_save_watch_later();
+                                        while(true) {
+                                            VideoPlayer::Error update_err = video_player->update();
+                                            if(update_err != VideoPlayer::Error::OK || !window.isOpen() || current_page == PageType::EXIT)
+                                                break;
+                                            std::this_thread::sleep_for(std::chrono::milliseconds(20));
+                                        }
+                                        video_player.reset();
+                                    }
+                                    page_changed = true;
+                                }
+                            });
+
+                            if(!window.isOpen() || current_page == PageType::EXIT)
+                                return;
+
+                            if(page_changed) {
+                                current_page = PageType::VIDEO_CONTENT;
+                                load_video_error_check(true);
+                            } else {
+                                XMapWindow(disp, video_player_window);
+                                XSync(disp, False);
+                            }
                         }
                     }
                 } else if(pressed_keysym == XK_c && pressing_ctrl) {
@@ -2702,6 +2727,26 @@ namespace QuickMedia {
                 std::string new_video_url;
                 std::string new_video_title;
 
+                if(video_tasks.valid()) {
+                    TaskResult task_result = run_task_with_loading_screen([&video_tasks, &related_videos]() {
+                        while(true) {
+                            if(program_is_dead_in_current_thread())
+                                return false;
+
+                            if(video_tasks.ready()) {
+                                related_videos = video_tasks.get();
+                                return true;
+                            }
+                            std::this_thread::sleep_for(std::chrono::milliseconds(50));
+                        }
+                    });
+
+                    if(task_result == TaskResult::CANCEL || task_result == TaskResult::FALSE) {
+                        current_page = previous_page;
+                        return;
+                    }
+                }
+
                 // Find video that hasn't been played before in this video session
                 auto find_next_video = [this, &related_videos, &video_page, &new_video_url, &new_video_title]() {
                     for(auto it = related_videos.begin(), end = related_videos.end(); it != end; ++it) {
diff --git a/src/plugins/MediaGeneric.cpp b/src/plugins/MediaGeneric.cpp
index c3a8d8e..11b0957 100644
--- a/src/plugins/MediaGeneric.cpp
+++ b/src/plugins/MediaGeneric.cpp
@@ -180,7 +180,7 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    BodyItems MediaGenericVideoPage::get_related_media(const std::string &url, std::string&) {
+    BodyItems MediaGenericVideoPage::get_related_media(const std::string &url) {
         BodyItems result_items;
         search_page->get_related_media(url, result_items);
         return result_items;
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index 9cec69c..04c29e0 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -13,6 +13,7 @@ extern "C" {
 #include <json/writer.h>
 #include <string.h>
 #include <unistd.h>
+#include <fcntl.h>
 
 namespace QuickMedia {
     bool youtube_url_extract_id(const std::string &youtube_url, std::string &youtube_video_id) {
@@ -1691,7 +1692,7 @@ namespace QuickMedia {
         return "";
     }
 
-    BodyItems YoutubeVideoPage::get_related_media(const std::string &url, std::string &channel_url) {
+    BodyItems YoutubeVideoPage::get_related_media(const std::string &url) {
         BodyItems result_items;
 
         std::string modified_url = remove_index_from_playlist_url(url);
@@ -1721,56 +1722,12 @@ namespace QuickMedia {
             if(!json_item.isObject())
                 continue;
 
-            const Json::Value &player_response_json = json_item["playerResponse"];
-            if(channel_url.empty()) {
-                if(player_response_json.isObject()) {
-                    const Json::Value &video_details_json = player_response_json["videoDetails"];
-                    if(video_details_json.isObject()) {
-                        const Json::Value &channel_id_json = video_details_json["channelId"];
-                        if(channel_id_json.isString())
-                            channel_url = "https://www.youtube.com/channel/" + channel_id_json.asString();
-                    }
-                }
-            }
-
             if(xsrf_token.empty()) {
                 const Json::Value &xsrf_token_json = json_item["xsrf_token"];
                 if(xsrf_token_json.isString())
                     xsrf_token = xsrf_token_json.asString();
             }
 
-            if(player_response_json.isObject()) {
-                const Json::Value &playback_tracing_json = player_response_json["playbackTracking"];
-                if(playback_tracing_json.isObject()) {
-                    if(playback_url.empty()) {
-                        const Json::Value &video_stats_playback_url_json = playback_tracing_json["videostatsPlaybackUrl"];
-                        if(video_stats_playback_url_json.isObject()) {
-                            const Json::Value &base_url_json = video_stats_playback_url_json["baseUrl"];
-                            if(base_url_json.isString())
-                                playback_url = base_url_json.asString();
-                        }
-                    }
-
-                    if(watchtime_url.empty()) {
-                        const Json::Value &video_stats_watchtime_url_json = playback_tracing_json["videostatsWatchtimeUrl"];
-                        if(video_stats_watchtime_url_json.isObject()) {
-                            const Json::Value &base_url_json = video_stats_watchtime_url_json["baseUrl"];
-                            if(base_url_json.isString())
-                                watchtime_url = base_url_json.asString();
-                        }
-                    }
-
-                    if(tracking_url.empty()) {
-                        const Json::Value &p_tracking_url_json = playback_tracing_json["ptrackingUrl"];
-                        if(p_tracking_url_json.isObject()) {
-                            const Json::Value &base_url_json = p_tracking_url_json["baseUrl"];
-                            if(base_url_json.isString())
-                                tracking_url = base_url_json.asString();
-                        }
-                    }
-                }
-            }
-
             const Json::Value &response_json = json_item["response"];
             if(!response_json.isObject())
                 continue;
@@ -1900,7 +1857,7 @@ namespace QuickMedia {
         return audio_formats.front().base.url;
     }
 
-    PluginResult YoutubeVideoPage::load() {
+    PluginResult YoutubeVideoPage::load(std::string &channel_url) {
         video_formats.clear();
         audio_formats.clear();
 
@@ -1919,7 +1876,7 @@ namespace QuickMedia {
         additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
 
         std::string response;
-        DownloadResult download_result = download_to_string("https://www.youtube.com/get_video_info?html5=1&video_id=" + video_id + "&eurl=https://www.youtube.googleapis.com/v/" + video_id, response, std::move(additional_args), true); // TODO: true?
+        DownloadResult download_result = download_to_string("https://www.youtube.com/get_video_info?html5=1&video_id=" + video_id + "&eurl=https://www.youtube.googleapis.com/v/" + video_id, response, std::move(additional_args), true);
         if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result);
 
         std::string player_response_param = url_extract_param(response, "player_response");
@@ -1946,6 +1903,43 @@ namespace QuickMedia {
         if(video_formats.empty() && audio_formats.empty())
             return PluginResult::ERR;
 
+        const Json::Value &video_details_json = json_root["videoDetails"];
+        if(video_details_json.isObject()) {
+            const Json::Value &channel_id_json = video_details_json["channelId"];
+            if(channel_id_json.isString())
+                channel_url = "https://www.youtube.com/channel/" + channel_id_json.asString();
+        }
+
+        const Json::Value &playback_tracing_json = json_root["playbackTracking"];
+        if(playback_tracing_json.isObject()) {
+            if(playback_url.empty()) {
+                const Json::Value &video_stats_playback_url_json = playback_tracing_json["videostatsPlaybackUrl"];
+                if(video_stats_playback_url_json.isObject()) {
+                    const Json::Value &base_url_json = video_stats_playback_url_json["baseUrl"];
+                    if(base_url_json.isString())
+                        playback_url = base_url_json.asString();
+                }
+            }
+
+            if(watchtime_url.empty()) {
+                const Json::Value &video_stats_watchtime_url_json = playback_tracing_json["videostatsWatchtimeUrl"];
+                if(video_stats_watchtime_url_json.isObject()) {
+                    const Json::Value &base_url_json = video_stats_watchtime_url_json["baseUrl"];
+                    if(base_url_json.isString())
+                        watchtime_url = base_url_json.asString();
+                }
+            }
+
+            if(tracking_url.empty()) {
+                const Json::Value &p_tracking_url_json = playback_tracing_json["ptrackingUrl"];
+                if(p_tracking_url_json.isObject()) {
+                    const Json::Value &base_url_json = p_tracking_url_json["baseUrl"];
+                    if(base_url_json.isString())
+                        tracking_url = base_url_json.asString();
+                }
+            }
+        }
+
         std::sort(video_formats.begin(), video_formats.end(), [](const YoutubeVideoFormat &format1, const YoutubeVideoFormat &format2) {
             return format1.base.bitrate > format2.base.bitrate;
         });
@@ -1957,6 +1951,53 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
+    static bool generate_random_characters(char *buffer, int buffer_size, const char *alphabet, size_t alphabet_size) {
+        int fd = open("/dev/urandom", O_RDONLY);
+        if(fd == -1) {
+            perror("/dev/urandom");
+            return false;
+        }
+
+        if(read(fd, buffer, buffer_size) < buffer_size) {
+            fprintf(stderr, "Failed to read %d bytes from /dev/urandom\n", buffer_size);
+            close(fd);
+            return false;
+        }
+
+        for(int i = 0; i < buffer_size; ++i) {
+            unsigned char c = *(unsigned char*)&buffer[i];
+            buffer[i] = alphabet[c % alphabet_size];
+        }
+        close(fd);
+        return true;
+    }
+
+    void YoutubeVideoPage::mark_watched() {
+        if(playback_url.empty()) {
+            fprintf(stderr, "Failed to mark video as watched because playback_url is empty\n");
+            return;
+        }
+
+        std::vector<CommandArg> additional_args = {
+            { "-H", "x-youtube-client-name: 1" },
+            { "-H", "x-youtube-client-version: 2.20200626.03.00" }
+        };
+
+        std::vector<CommandArg> cookies = get_cookies();
+        additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());
+
+        std::string cpn;
+        cpn.resize(16);
+        generate_random_characters(cpn.data(), cpn.size(), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_", 64);
+
+        std::string response;
+        DownloadResult download_result = download_to_string(playback_url + "&ver=2&cpn=" + cpn, response, std::move(additional_args), true);
+        if(download_result != DownloadResult::OK) {
+            fprintf(stderr, "Failed to mark video as watched because http request failed\n");
+            return;
+        }
+    }
+
     static bool parse_cipher_format(const Json::Value &format, YoutubeFormat &youtube_format) {
         std::map<std::string, std::string> cipher_params;
         const Json::Value &cipher_json = format["cipher"];