From b96dce2d670e085067d7b62b43fceac7658c2d5a Mon Sep 17 00:00:00 2001 From: dec05eba Date: Mon, 13 Jul 2020 12:04:12 +0200 Subject: Add recommended tab for youtube --- README.md | 3 +- include/Body.hpp | 1 + plugins/Youtube.hpp | 6 +- project.conf | 1 + src/QuickMedia.cpp | 40 +++++++- src/plugins/Youtube.cpp | 252 ++++++++++++++++++++++++++++++++++++++++-------- 6 files changed, 256 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 958ebfe..fc5ba12 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,8 @@ See project.conf \[dependencies]. `mpv` is required for playing videos. This is not required if you dont plan on playing videos.\ `youtube-dl` needs to be installed to play videos from youtube.\ `notify-send` needs to be installed to show notifications (on Linux and other systems that uses d-bus notification system).\ -`torsocks` needs to be installed when using the `--tor` option. +`torsocks` needs to be installed when using the `--tor` option.\ +`automedia` needs to be installed when tracking manga with `Ctrl + T`. # TODO If a search returns no results, then "No results found for ..." should be shown and navigation should go back to searching with suggestions.\ Give user the option to start where they left off or from the start or from the start.\ diff --git a/include/Body.hpp b/include/Body.hpp index 3cb735c..4c7044d 100644 --- a/include/Body.hpp +++ b/include/Body.hpp @@ -62,6 +62,7 @@ namespace QuickMedia { static bool string_find_case_insensitive(const std::string &str, const std::string &substr); // TODO: Make this actually fuzzy... Right now it's just a case insensitive string find. + // This would require reordering the body. // TODO: Highlight the part of the text that matches the search. // TODO: Ignore dot, whitespace and special characters void filter_search_fuzzy(const std::string &text); diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp index 5a88970..8d57cc3 100644 --- a/plugins/Youtube.hpp +++ b/plugins/Youtube.hpp @@ -5,7 +5,8 @@ namespace QuickMedia { class Youtube : public Plugin { public: - Youtube() : Plugin("youtube") {} + Youtube(); + PluginResult get_front_page(BodyItems &result_items) override; SuggestionResult update_search_suggestions(const std::string &text, BodyItems &result_items) override; BodyItems get_related_media(const std::string &url) override; bool search_suggestions_has_thumbnails() const override { return true; } @@ -16,7 +17,10 @@ namespace QuickMedia { Page get_page_after_search() const override { return Page::VIDEO_CONTENT; } private: void search_suggestions_get_continuation(const std::string &url, const std::string &continuation_token, BodyItems &result_items); + std::optional get_cookies() const; private: std::string last_autocomplete_result; + std::string visitor_info_live; + std::string ysc; }; } \ No newline at end of file diff --git a/project.conf b/project.conf index 04ed905..ffe0fac 100644 --- a/project.conf +++ b/project.conf @@ -12,3 +12,4 @@ sfml-graphics = "2" x11 = "1" jsoncpp = "1" cppcodec-1 = "0" +uuid = "2" \ No newline at end of file diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index db244c8..ed94700 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -355,7 +355,8 @@ namespace QuickMedia { enum class SearchSuggestionTab { ALL, - HISTORY + HISTORY, + RECOMMENDED }; static void fill_history_items_from_json(const Json::Value &history_json, BodyItems &history_items) { @@ -477,10 +478,17 @@ namespace QuickMedia { bool autocomplete_running = false; Body history_body(this, font, bold_font); + std::unique_ptr recommended_body; const float tab_text_size = 18.0f; const float tab_height = tab_text_size + 10.0f; sf::Text all_tab_text("All", font, tab_text_size); sf::Text history_tab_text("History", font, tab_text_size); + sf::Text recommended_tab_text("Recommended", font, tab_text_size); + + if(current_plugin->name == "youtube") { + recommended_body = std::make_unique(this, font, bold_font); + recommended_body->draw_thumbnails = true; + } struct Tab { Body *body; @@ -488,7 +496,10 @@ namespace QuickMedia { sf::Text *text; }; - std::array tabs = { Tab{body, SearchSuggestionTab::ALL, &all_tab_text}, Tab{&history_body, SearchSuggestionTab::HISTORY, &history_tab_text} }; + std::vector tabs = { Tab{body, SearchSuggestionTab::ALL, &all_tab_text}, Tab{&history_body, SearchSuggestionTab::HISTORY, &history_tab_text} }; + if(recommended_body) + tabs.push_back(Tab{recommended_body.get(), SearchSuggestionTab::RECOMMENDED, &recommended_tab_text}); + int selected_tab = 0; plugin_get_watch_history(current_plugin, history_body.items); @@ -505,13 +516,17 @@ namespace QuickMedia { autocomplete_text = text; }; - search_bar->onTextUpdateCallback = [&update_search_text, this, &tabs, &selected_tab, &typing](const std::string &text) { + std::string recommended_filter; + + search_bar->onTextUpdateCallback = [&update_search_text, this, &tabs, &selected_tab, &typing, &recommended_body, &recommended_filter](const std::string &text) { if(tabs[selected_tab].body == body && !current_plugin->search_is_filter()) update_search_text = text; else { tabs[selected_tab].body->filter_search_fuzzy(text); tabs[selected_tab].body->clamp_selection(); } + if(tabs[selected_tab].body == recommended_body.get()) + recommended_filter = text; typing = false; }; @@ -567,8 +582,17 @@ namespace QuickMedia { return true; }; - PluginResult front_page_result = current_plugin->get_front_page(body->items); - body->clamp_selection(); + std::future recommended_future; + if(recommended_body) { + recommended_future = std::async(std::launch::async, [this]() { + BodyItems body_items; + PluginResult front_page_result = current_plugin->get_front_page(body_items); + return body_items; + }); + } else { + PluginResult front_page_result = current_plugin->get_front_page(body->items); + body->clamp_selection(); + } sf::Vector2f body_pos; sf::Vector2f body_size; @@ -663,6 +687,12 @@ namespace QuickMedia { autocomplete_running = false; } + if(recommended_future.valid() && recommended_future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) { + recommended_body->items = recommended_future.get(); + recommended_body->filter_search_fuzzy(recommended_filter); + recommended_body->clamp_selection(); + } + window.clear(back_color); { tab_spacing_rect.setPosition(0.0f, search_bar->getBottomWithoutShadow()); diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index 3ab405c..95ee8e3 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -1,8 +1,21 @@ #include "../../plugins/Youtube.hpp" +#include "../../include/Storage.hpp" #include +#include +#include #include namespace QuickMedia { + const int VISITOR_ID_SIZE = 12; + const char visitor_id_chars[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQSTUVWXYZ0123456789-_"; + static void generate_visitor_id(char visitor_id[VISITOR_ID_SIZE]) { + uuid_t uuid_num; + uuid_generate_random(uuid_num); + for(int i = 0; i < VISITOR_ID_SIZE; ++i) { + visitor_id[i] = visitor_id_chars[uuid_num[i] % (sizeof(visitor_id_chars) - 1)]; + } + } + static void iterate_suggestion_result(const Json::Value &value, std::vector &result_items, int &iterate_count) { ++iterate_count; if(value.isArray()) { @@ -14,6 +27,183 @@ namespace QuickMedia { } } + static bool json_read_cookies(const Json::Value &json_cookies, std::string &visitor_info_live, std::string &ysc) { + if(!json_cookies.isObject()) + return false; + + const Json::Value &visitor_info_live_json = json_cookies["visitor_info_live"]; + if(!visitor_info_live_json.isString()) + return false; + + const Json::Value &ysc_json = json_cookies["ysc"]; + if(!ysc_json.isString()) + return false; + + visitor_info_live = visitor_info_live_json.asString(); + ysc = ysc_json.asString(); + return true; + } + + static std::unique_ptr parse_content_video_renderer(const Json::Value &content_item_json) { + if(!content_item_json.isObject()) + return nullptr; + + const Json::Value &video_renderer_json = content_item_json["videoRenderer"]; + if(!video_renderer_json.isObject()) + return nullptr; + + const Json::Value &video_id_json = video_renderer_json["videoId"]; + if(!video_id_json.isString()) + return nullptr; + + std::string video_id_str = video_id_json.asString(); + std::string thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/hqdefault.jpg"; + + const char *title = nullptr; + const Json::Value &title_json = video_renderer_json["title"]; + if(title_json.isObject()) { + const Json::Value &runs_json = title_json["runs"]; + if(runs_json.isArray() && !runs_json.empty()) { + const Json::Value &first_runs_json = runs_json[0]; + if(first_runs_json.isObject()) { + const Json::Value &text_json = first_runs_json["text"]; + if(text_json.isString()) + title = text_json.asCString(); + } + } + } + + if(!title) + return nullptr; + + auto body_item = std::make_unique(title); + body_item->url = "https://www.youtube.com/watch?v=" + video_id_str; + body_item->thumbnail_url = std::move(thumbnail_url); + return body_item; + } + + Youtube::Youtube() : Plugin("youtube") { + Path youtube_config_dir = get_storage_dir().join("youtube"); + if(create_directory_recursive(youtube_config_dir) == 0) { + Path cookie_path = youtube_config_dir; + cookie_path.join("cookies.json"); + + bool cookies_read_from_file = false; + std::string cookie_file_content; + if(file_get_content(cookie_path, cookie_file_content) == 0) { + Json::Value json_root; + Json::CharReaderBuilder json_builder; + std::unique_ptr json_reader(json_builder.newCharReader()); + std::string json_errors; + if(json_reader->parse(&cookie_file_content[0], &cookie_file_content[cookie_file_content.size()], &json_root, &json_errors) && json_read_cookies(json_root, visitor_info_live, ysc)) { + cookies_read_from_file = true; + } else { + fprintf(stderr, "Youtube failed to read cookies: %s\n", json_errors.c_str()); + } + } + + if(!cookies_read_from_file) { + visitor_info_live.resize(VISITOR_ID_SIZE); + ysc.resize(VISITOR_ID_SIZE); + generate_visitor_id(&visitor_info_live[0]); + generate_visitor_id(&ysc[0]); + + Json::Value json_cookies(Json::objectValue); + json_cookies["visitor_info_live"] = visitor_info_live; + json_cookies["ysc"] = ysc; + + Json::StreamWriterBuilder json_builder; + file_overwrite(cookie_path, Json::writeString(json_builder, json_cookies)); + } + } + } + + PluginResult Youtube::get_front_page(BodyItems &result_items) { + std::string url = "https://youtube.com/"; + + std::vector additional_args = { + { "-H", "x-spf-referer: " + url }, + { "-H", "x-youtube-client-name: 1" }, + { "-H", "x-youtube-client-version: 2.20200626.03.00" }, + { "-H", "referer: " + url } + }; + + std::optional cookies = get_cookies(); + if(cookies) + additional_args.push_back(cookies.value()); + + std::string website_data; + if(download_to_string(url + "?pbj=1", website_data, additional_args, use_tor, true) != DownloadResult::OK) + return PluginResult::NET_ERR; + + Json::Value json_root; + Json::CharReaderBuilder json_builder; + std::unique_ptr json_reader(json_builder.newCharReader()); + std::string json_errors; + if(!json_reader->parse(&website_data[0], &website_data[website_data.size()], &json_root, &json_errors)) { + fprintf(stderr, "Youtube get front page error: %s\n", json_errors.c_str()); + return PluginResult::ERR; + } + + if(!json_root.isArray()) + return PluginResult::ERR; + + for(const Json::Value &json_item : json_root) { + if(!json_item.isObject()) + continue; + + const Json::Value &response_json = json_item["response"]; + if(!response_json.isObject()) + continue; + + const Json::Value &contents_json = response_json["contents"]; + if(!contents_json.isObject()) + continue; + + const Json::Value &tcbrr_json = contents_json["twoColumnBrowseResultsRenderer"]; + if(!tcbrr_json.isObject()) + continue; + + const Json::Value &tabs_json = tcbrr_json["tabs"]; + if(!tabs_json.isArray()) + continue; + + for(const Json::Value &tab_item_json : tabs_json) { + if(!tab_item_json.isObject()) + continue; + + const Json::Value &tab_renderer_json = tab_item_json["tabRenderer"]; + if(!tab_renderer_json.isObject()) + continue; + + const Json::Value &content_json = tab_renderer_json["content"]; + if(!content_json.isObject()) + continue; + + const Json::Value &rich_grid_renderer = content_json["richGridRenderer"]; + if(!rich_grid_renderer.isObject()) + continue; + + const Json::Value &contents2_json = rich_grid_renderer["contents"]; + if(!contents2_json.isArray()) + continue; + + for(const Json::Value &contents_item : contents2_json) { + const Json::Value &rich_item_renderer_json = contents_item["richItemRenderer"]; + if(!rich_item_renderer_json.isObject()) + continue; + + const Json::Value &rich_item_contents = rich_item_renderer_json["content"]; + std::unique_ptr body_item = parse_content_video_renderer(rich_item_contents); + if(body_item) + result_items.push_back(std::move(body_item)); + } + } + } + + return PluginResult::OK; + } + std::string Youtube::autocomplete_search(const std::string &query) { // Return the last result if the query is a substring of the autocomplete result if(last_autocomplete_result.size() >= query.size() && memcmp(query.data(), last_autocomplete_result.data(), query.size()) == 0) @@ -118,41 +308,9 @@ namespace QuickMedia { return; for(const Json::Value &content_item_json : item_contents_json) { - if(!content_item_json.isObject()) - continue; - - const Json::Value &video_renderer_json = content_item_json["videoRenderer"]; - if(!video_renderer_json.isObject()) - continue; - - const Json::Value &video_id_json = video_renderer_json["videoId"]; - if(!video_id_json.isString()) - continue; - - std::string video_id_str = video_id_json.asString(); - std::string thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/hqdefault.jpg"; - - const char *title = nullptr; - const Json::Value &title_json = video_renderer_json["title"]; - if(title_json.isObject()) { - const Json::Value &runs_json = title_json["runs"]; - if(runs_json.isArray() && !runs_json.empty()) { - const Json::Value &first_runs_json = runs_json[0]; - if(first_runs_json.isObject()) { - const Json::Value &text_json = first_runs_json["text"]; - if(text_json.isString()) - title = text_json.asCString(); - } - } - } - - if(!title) - continue; - - auto body_item = std::make_unique(title); - body_item->url = "https://www.youtube.com/watch?v=" + video_id_str; - body_item->thumbnail_url = std::move(thumbnail_url); - result_items.push_back(std::move(body_item)); + std::unique_ptr body_item = parse_content_video_renderer(content_item_json); + if(body_item) + result_items.push_back(std::move(body_item)); } } @@ -167,6 +325,10 @@ namespace QuickMedia { { "-H", "referer: " + url } }; + std::optional cookies = get_cookies(); + if(cookies) + additional_args.push_back(cookies.value()); + std::string website_data; if(download_to_string(url + "&pbj=1", website_data, additional_args, use_tor, true) != DownloadResult::OK) return SuggestionResult::NET_ERR; @@ -195,23 +357,23 @@ namespace QuickMedia { const Json::Value &contents_json = response_json["contents"]; if(!contents_json.isObject()) - return SuggestionResult::ERR; + continue; const Json::Value &tcsrr_json = contents_json["twoColumnSearchResultsRenderer"]; if(!tcsrr_json.isObject()) - return SuggestionResult::ERR; + continue; const Json::Value &primary_contents_json = tcsrr_json["primaryContents"]; if(!primary_contents_json.isObject()) - return SuggestionResult::ERR; + continue; const Json::Value §ion_list_renderer_json = primary_contents_json["sectionListRenderer"]; if(!section_list_renderer_json.isObject()) - return SuggestionResult::ERR; + continue; const Json::Value &contents2_json = section_list_renderer_json["contents"]; if(!contents2_json.isArray()) - return SuggestionResult::ERR; + continue; for(const Json::Value &item_json : contents2_json) { if(!item_json.isObject()) @@ -243,6 +405,10 @@ namespace QuickMedia { { "-H", "referer: " + url } }; + std::optional cookies = get_cookies(); + if(cookies) + additional_args.push_back(cookies.value()); + std::string website_data; if(download_to_string(next_url, website_data, additional_args, use_tor, true) != DownloadResult::OK) return; @@ -282,6 +448,12 @@ namespace QuickMedia { } } + std::optional Youtube::get_cookies() const { + if(!visitor_info_live.empty() && !ysc.empty()) + return CommandArg { "-H", "cookie: VISITOR_INFO1_LIVE=" + visitor_info_live + "; YSC=" + ysc + "; wide=1; PREF=; GPS=0" }; + return std::nullopt; + } + static std::string remove_index_from_playlist_url(const std::string &url) { std::string result = url; size_t index = result.rfind("&index="); -- cgit v1.2.3