Add recommended tab for youtube

author: dec05eba <dec05eba@protonmail.com> 2020-07-13 12:04:12 +0200
committer: dec05eba <dec05eba@protonmail.com> 2020-07-13 12:04:12 +0200
commit: b96dce2d670e085067d7b62b43fceac7658c2d5a (patch)
tree: 21af560d73232d985273cede3dcdd07996b5cd94
parent: f38d25cc18720203ff74ca3f8452119c71a82453 (diff)
6 files changed, 256 insertions, 47 deletions
diff --git a/README.md b/README.md
index 958ebfe..fc5ba12 100644
--- a/README.md
+++ b/README.md
@@ -56,7 +56,8 @@ See project.conf \[dependencies].
 `mpv` is required for playing videos. This is not required if you dont plan on playing videos.\
 `youtube-dl` needs to be installed to play videos from youtube.\
 `notify-send` needs to be installed to show notifications (on Linux and other systems that uses d-bus notification system).\
-`torsocks` needs to be installed when using the `--tor` option.
+`torsocks` needs to be installed when using the `--tor` option.\
+`automedia` needs to be installed when tracking manga with `Ctrl + T`.
 # TODO
 If a search returns no results, then "No results found for ..." should be shown and navigation should go back to searching with suggestions.\
 Give user the option to start where they left off or from the start or from the start.\
diff --git a/include/Body.hpp b/include/Body.hpp
index 3cb735c..4c7044d 100644
--- a/include/Body.hpp
+++ b/include/Body.hpp
@@ -62,6 +62,7 @@ namespace QuickMedia {
         static bool string_find_case_insensitive(const std::string &str, const std::string &substr);
 
         // TODO: Make this actually fuzzy... Right now it's just a case insensitive string find.
+        // This would require reordering the body.
         // TODO: Highlight the part of the text that matches the search.
         // TODO: Ignore dot, whitespace and special characters
         void filter_search_fuzzy(const std::string &text);
diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp
index 5a88970..8d57cc3 100644
--- a/plugins/Youtube.hpp
+++ b/plugins/Youtube.hpp
@@ -5,7 +5,8 @@
 namespace QuickMedia {
     class Youtube : public Plugin {
     public:
-        Youtube() : Plugin("youtube") {}
+        Youtube();
+        PluginResult get_front_page(BodyItems &result_items) override;
         SuggestionResult update_search_suggestions(const std::string &text, BodyItems &result_items) override;
         BodyItems get_related_media(const std::string &url) override;
         bool search_suggestions_has_thumbnails() const override { return true; }
@@ -16,7 +17,10 @@ namespace QuickMedia {
         Page get_page_after_search() const override { return Page::VIDEO_CONTENT; }
     private:
         void search_suggestions_get_continuation(const std::string &url, const std::string &continuation_token, BodyItems &result_items);
+        std::optional<CommandArg> get_cookies() const;
     private:
         std::string last_autocomplete_result;
+        std::string visitor_info_live;
+        std::string ysc;
     };
 }
 \ No newline at end of file
diff --git a/project.conf b/project.conf
index 04ed905..ffe0fac 100644
--- a/project.conf
+++ b/project.conf
@@ -12,3 +12,4 @@ sfml-graphics = "2"
 x11 = "1"
 jsoncpp = "1"
 cppcodec-1 = "0"
+uuid = "2"
+\ No newline at end of file
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index db244c8..ed94700 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -355,7 +355,8 @@ namespace QuickMedia {
 
     enum class SearchSuggestionTab {
         ALL,
-        HISTORY
+        HISTORY,
+        RECOMMENDED
     };
 
     static void fill_history_items_from_json(const Json::Value &history_json, BodyItems &history_items) {
@@ -477,10 +478,17 @@ namespace QuickMedia {
         bool autocomplete_running = false;
 
         Body history_body(this, font, bold_font);
+        std::unique_ptr<Body> recommended_body;
         const float tab_text_size = 18.0f;
         const float tab_height = tab_text_size + 10.0f;
         sf::Text all_tab_text("All", font, tab_text_size);
         sf::Text history_tab_text("History", font, tab_text_size);
+        sf::Text recommended_tab_text("Recommended", font, tab_text_size);
+
+        if(current_plugin->name == "youtube") {
+            recommended_body = std::make_unique<Body>(this, font, bold_font);
+            recommended_body->draw_thumbnails = true;
+        }
 
         struct Tab {
             Body *body;
@@ -488,7 +496,10 @@ namespace QuickMedia {
             sf::Text *text;
         };
 
-        std::array<Tab, 2> tabs = { Tab{body, SearchSuggestionTab::ALL, &all_tab_text}, Tab{&history_body, SearchSuggestionTab::HISTORY, &history_tab_text} };
+        std::vector<Tab> tabs = { Tab{body, SearchSuggestionTab::ALL, &all_tab_text}, Tab{&history_body, SearchSuggestionTab::HISTORY, &history_tab_text} };
+        if(recommended_body)
+            tabs.push_back(Tab{recommended_body.get(), SearchSuggestionTab::RECOMMENDED, &recommended_tab_text});
+
         int selected_tab = 0;
 
         plugin_get_watch_history(current_plugin, history_body.items);
@@ -505,13 +516,17 @@ namespace QuickMedia {
                 autocomplete_text = text;
         };
 
-        search_bar->onTextUpdateCallback = [&update_search_text, this, &tabs, &selected_tab, &typing](const std::string &text) {
+        std::string recommended_filter;
+
+        search_bar->onTextUpdateCallback = [&update_search_text, this, &tabs, &selected_tab, &typing, &recommended_body, &recommended_filter](const std::string &text) {
             if(tabs[selected_tab].body == body && !current_plugin->search_is_filter())
                 update_search_text = text;
             else {
                 tabs[selected_tab].body->filter_search_fuzzy(text);
                 tabs[selected_tab].body->clamp_selection();
             }
+            if(tabs[selected_tab].body == recommended_body.get())
+                recommended_filter = text;
             typing = false;
         };
 
@@ -567,8 +582,17 @@ namespace QuickMedia {
             return true;
         };
 
-        PluginResult front_page_result = current_plugin->get_front_page(body->items);
-        body->clamp_selection();
+        std::future<BodyItems> recommended_future;
+        if(recommended_body) {
+            recommended_future = std::async(std::launch::async, [this]() {
+                BodyItems body_items;
+                PluginResult front_page_result = current_plugin->get_front_page(body_items);
+                return body_items;
+            });
+        } else {
+            PluginResult front_page_result = current_plugin->get_front_page(body->items);
+            body->clamp_selection();
+        }
 
         sf::Vector2f body_pos;
         sf::Vector2f body_size;
@@ -663,6 +687,12 @@ namespace QuickMedia {
                 autocomplete_running = false;
             }
 
+            if(recommended_future.valid() && recommended_future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) {
+                recommended_body->items = recommended_future.get();
+                recommended_body->filter_search_fuzzy(recommended_filter);
+                recommended_body->clamp_selection();
+            }
+
             window.clear(back_color);
             {
                 tab_spacing_rect.setPosition(0.0f, search_bar->getBottomWithoutShadow());
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index 3ab405c..95ee8e3 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -1,8 +1,21 @@
 #include "../../plugins/Youtube.hpp"
+#include "../../include/Storage.hpp"
 #include <json/reader.h>
+#include <json/writer.h>
+#include <uuid.h>
 #include <string.h>
 
 namespace QuickMedia {
+    const int VISITOR_ID_SIZE = 12;
+    const char visitor_id_chars[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQSTUVWXYZ0123456789-_";
+    static void generate_visitor_id(char visitor_id[VISITOR_ID_SIZE]) {
+        uuid_t uuid_num;
+        uuid_generate_random(uuid_num);
+        for(int i = 0; i < VISITOR_ID_SIZE; ++i) {
+            visitor_id[i] = visitor_id_chars[uuid_num[i] % (sizeof(visitor_id_chars) - 1)];
+        }
+    }
+
     static void iterate_suggestion_result(const Json::Value &value, std::vector<std::string> &result_items, int &iterate_count) {
         ++iterate_count;
         if(value.isArray()) {
@@ -14,6 +27,183 @@ namespace QuickMedia {
         }
     }
 
+    static bool json_read_cookies(const Json::Value &json_cookies, std::string &visitor_info_live, std::string &ysc) {
+        if(!json_cookies.isObject())
+            return false;
+
+        const Json::Value &visitor_info_live_json = json_cookies["visitor_info_live"];
+        if(!visitor_info_live_json.isString())
+            return false;
+
+        const Json::Value &ysc_json = json_cookies["ysc"];
+        if(!ysc_json.isString())
+            return false;
+
+        visitor_info_live = visitor_info_live_json.asString();
+        ysc = ysc_json.asString();
+        return true;
+    }
+
+    static std::unique_ptr<BodyItem> parse_content_video_renderer(const Json::Value &content_item_json) {
+        if(!content_item_json.isObject())
+            return nullptr;
+        
+        const Json::Value &video_renderer_json = content_item_json["videoRenderer"];
+        if(!video_renderer_json.isObject())
+            return nullptr;
+        
+        const Json::Value &video_id_json = video_renderer_json["videoId"];
+        if(!video_id_json.isString())
+            return nullptr;
+
+        std::string video_id_str = video_id_json.asString();
+        std::string thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/hqdefault.jpg";
+
+        const char *title = nullptr;
+        const Json::Value &title_json = video_renderer_json["title"];
+        if(title_json.isObject()) {
+            const Json::Value &runs_json = title_json["runs"];
+            if(runs_json.isArray() && !runs_json.empty()) {
+                const Json::Value &first_runs_json = runs_json[0];
+                if(first_runs_json.isObject()) {
+                    const Json::Value &text_json = first_runs_json["text"];
+                    if(text_json.isString())
+                        title = text_json.asCString();
+                }
+            }
+        }
+
+        if(!title)
+            return nullptr;
+        
+        auto body_item = std::make_unique<BodyItem>(title);
+        body_item->url = "https://www.youtube.com/watch?v=" + video_id_str;
+        body_item->thumbnail_url = std::move(thumbnail_url);
+        return body_item;
+    }
+
+    Youtube::Youtube() : Plugin("youtube") {
+        Path youtube_config_dir = get_storage_dir().join("youtube");
+        if(create_directory_recursive(youtube_config_dir) == 0) {
+            Path cookie_path = youtube_config_dir;
+            cookie_path.join("cookies.json");
+
+            bool cookies_read_from_file = false;
+            std::string cookie_file_content;
+            if(file_get_content(cookie_path, cookie_file_content) == 0) {
+                Json::Value json_root;
+                Json::CharReaderBuilder json_builder;
+                std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader());
+                std::string json_errors;
+                if(json_reader->parse(&cookie_file_content[0], &cookie_file_content[cookie_file_content.size()], &json_root, &json_errors) && json_read_cookies(json_root, visitor_info_live, ysc)) {
+                    cookies_read_from_file = true;
+                } else {
+                    fprintf(stderr, "Youtube failed to read cookies: %s\n", json_errors.c_str());
+                }
+            }
+
+            if(!cookies_read_from_file) {
+                visitor_info_live.resize(VISITOR_ID_SIZE);
+                ysc.resize(VISITOR_ID_SIZE);
+                generate_visitor_id(&visitor_info_live[0]);
+                generate_visitor_id(&ysc[0]);
+
+                Json::Value json_cookies(Json::objectValue);
+                json_cookies["visitor_info_live"] = visitor_info_live;
+                json_cookies["ysc"] = ysc;
+
+                Json::StreamWriterBuilder json_builder;
+                file_overwrite(cookie_path, Json::writeString(json_builder, json_cookies));
+            }
+        }
+    }
+
+    PluginResult Youtube::get_front_page(BodyItems &result_items) {
+        std::string url = "https://youtube.com/";
+
+        std::vector<CommandArg> additional_args = {
+            { "-H", "x-spf-referer: " + url },
+            { "-H", "x-youtube-client-name: 1" },
+            { "-H", "x-youtube-client-version: 2.20200626.03.00" },
+            { "-H", "referer: " + url }
+        };
+
+        std::optional<CommandArg> cookies = get_cookies();
+        if(cookies)
+            additional_args.push_back(cookies.value());
+
+        std::string website_data;
+        if(download_to_string(url + "?pbj=1", website_data, additional_args, use_tor, true) != DownloadResult::OK)
+            return PluginResult::NET_ERR;
+
+        Json::Value json_root;
+        Json::CharReaderBuilder json_builder;
+        std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader());
+        std::string json_errors;
+        if(!json_reader->parse(&website_data[0], &website_data[website_data.size()], &json_root, &json_errors)) {
+            fprintf(stderr, "Youtube get front page error: %s\n", json_errors.c_str());
+            return PluginResult::ERR;
+        }
+
+        if(!json_root.isArray())
+            return PluginResult::ERR;
+
+        for(const Json::Value &json_item : json_root) {
+            if(!json_item.isObject())
+                continue;
+
+            const Json::Value &response_json = json_item["response"];
+            if(!response_json.isObject())
+                continue;
+
+            const Json::Value &contents_json = response_json["contents"];
+            if(!contents_json.isObject())
+                continue;
+
+            const Json::Value &tcbrr_json = contents_json["twoColumnBrowseResultsRenderer"];
+            if(!tcbrr_json.isObject())
+                continue;
+
+            const Json::Value &tabs_json = tcbrr_json["tabs"];
+            if(!tabs_json.isArray())
+                continue;
+
+            for(const Json::Value &tab_item_json : tabs_json) {
+                if(!tab_item_json.isObject())
+                    continue;
+                
+                const Json::Value &tab_renderer_json = tab_item_json["tabRenderer"];
+                if(!tab_renderer_json.isObject())
+                    continue;
+                
+                const Json::Value &content_json = tab_renderer_json["content"];
+                if(!content_json.isObject())
+                    continue;
+
+                const Json::Value &rich_grid_renderer = content_json["richGridRenderer"];
+                if(!rich_grid_renderer.isObject())
+                    continue;
+
+                const Json::Value &contents2_json = rich_grid_renderer["contents"];
+                if(!contents2_json.isArray())
+                    continue;
+
+                for(const Json::Value &contents_item : contents2_json) {
+                    const Json::Value &rich_item_renderer_json = contents_item["richItemRenderer"];
+                    if(!rich_item_renderer_json.isObject())
+                        continue;
+                    
+                    const Json::Value &rich_item_contents = rich_item_renderer_json["content"];
+                    std::unique_ptr<BodyItem> body_item = parse_content_video_renderer(rich_item_contents);
+                    if(body_item)
+                        result_items.push_back(std::move(body_item));
+                }
+            }
+        }
+
+        return PluginResult::OK;
+    }
+
     std::string Youtube::autocomplete_search(const std::string &query) {
         // Return the last result if the query is a substring of the autocomplete result
         if(last_autocomplete_result.size() >= query.size() && memcmp(query.data(), last_autocomplete_result.data(), query.size()) == 0)
@@ -118,41 +308,9 @@ namespace QuickMedia {
             return;
         
         for(const Json::Value &content_item_json : item_contents_json) {
-            if(!content_item_json.isObject())
-                continue;
-            
-            const Json::Value &video_renderer_json = content_item_json["videoRenderer"];
-            if(!video_renderer_json.isObject())
-                continue;
-            
-            const Json::Value &video_id_json = video_renderer_json["videoId"];
-            if(!video_id_json.isString())
-                continue;
-
-            std::string video_id_str = video_id_json.asString();
-            std::string thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/hqdefault.jpg";
-
-            const char *title = nullptr;
-            const Json::Value &title_json = video_renderer_json["title"];
-            if(title_json.isObject()) {
-                const Json::Value &runs_json = title_json["runs"];
-                if(runs_json.isArray() && !runs_json.empty()) {
-                    const Json::Value &first_runs_json = runs_json[0];
-                    if(first_runs_json.isObject()) {
-                        const Json::Value &text_json = first_runs_json["text"];
-                        if(text_json.isString())
-                            title = text_json.asCString();
-                    }
-                }
-            }
-
-            if(!title)
-                continue;
-            
-            auto body_item = std::make_unique<BodyItem>(title);
-            body_item->url = "https://www.youtube.com/watch?v=" + video_id_str;
-            body_item->thumbnail_url = std::move(thumbnail_url);
-            result_items.push_back(std::move(body_item));
+            std::unique_ptr<BodyItem> body_item = parse_content_video_renderer(content_item_json);
+            if(body_item)
+                result_items.push_back(std::move(body_item));
         }
     }
 
@@ -167,6 +325,10 @@ namespace QuickMedia {
             { "-H", "referer: " + url }
         };
 
+        std::optional<CommandArg> cookies = get_cookies();
+        if(cookies)
+            additional_args.push_back(cookies.value());
+
         std::string website_data;
         if(download_to_string(url + "&pbj=1", website_data, additional_args, use_tor, true) != DownloadResult::OK)
             return SuggestionResult::NET_ERR;
@@ -195,23 +357,23 @@ namespace QuickMedia {
 
             const Json::Value &contents_json = response_json["contents"];
             if(!contents_json.isObject())
-                return SuggestionResult::ERR;
+                continue;
 
             const Json::Value &tcsrr_json = contents_json["twoColumnSearchResultsRenderer"];
             if(!tcsrr_json.isObject())
-                return SuggestionResult::ERR;
+                continue;
 
             const Json::Value &primary_contents_json = tcsrr_json["primaryContents"];
             if(!primary_contents_json.isObject())
-                return SuggestionResult::ERR;
+                continue;
 
             const Json::Value &section_list_renderer_json = primary_contents_json["sectionListRenderer"];
             if(!section_list_renderer_json.isObject())
-                return SuggestionResult::ERR;
+                continue;
             
             const Json::Value &contents2_json = section_list_renderer_json["contents"];
             if(!contents2_json.isArray())
-                return SuggestionResult::ERR;
+                continue;
             
             for(const Json::Value &item_json : contents2_json) {
                 if(!item_json.isObject())
@@ -243,6 +405,10 @@ namespace QuickMedia {
             { "-H", "referer: " + url }
         };
 
+        std::optional<CommandArg> cookies = get_cookies();
+        if(cookies)
+            additional_args.push_back(cookies.value());
+
         std::string website_data;
         if(download_to_string(next_url, website_data, additional_args, use_tor, true) != DownloadResult::OK)
             return;
@@ -282,6 +448,12 @@ namespace QuickMedia {
         }
     }
 
+    std::optional<CommandArg> Youtube::get_cookies() const {
+        if(!visitor_info_live.empty() && !ysc.empty())
+            return CommandArg { "-H", "cookie: VISITOR_INFO1_LIVE=" + visitor_info_live + "; YSC=" + ysc + "; wide=1; PREF=; GPS=0" };
+        return std::nullopt;
+    }
+
     static std::string remove_index_from_playlist_url(const std::string &url) {
         std::string result = url;
         size_t index = result.rfind("&index=");
author	dec05eba <dec05eba@protonmail.com>	2020-07-13 12:04:12 +0200
committer	dec05eba <dec05eba@protonmail.com>	2020-07-13 12:04:12 +0200
commit	b96dce2d670e085067d7b62b43fceac7658c2d5a (patch)
tree	21af560d73232d985273cede3dcdd07996b5cd94
parent	f38d25cc18720203ff74ca3f8452119c71a82453 (diff)