From 117eb25e36ac3b1e1ba18cc9f1e177016c076f34 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 3 Aug 2019 01:52:27 +0200 Subject: Add search suggestions for youtube & manganelo --- plugins/Manganelo.hpp | 1 + plugins/Plugin.hpp | 15 +++++++++++- plugins/Youtube.hpp | 1 + project.conf | 3 ++- src/Manganelo.cpp | 63 +++++++++++++++++++++++++++++++++++++++++++++++++-- src/Plugin.cpp | 36 ++++++++++++++++++++++++++--- src/Youtube.cpp | 58 +++++++++++++++++++++++++++++++++++++++++++---- src/main.cpp | 26 +++++++++++++++++++-- 8 files changed, 190 insertions(+), 13 deletions(-) diff --git a/plugins/Manganelo.hpp b/plugins/Manganelo.hpp index fd56b85..4ed976f 100644 --- a/plugins/Manganelo.hpp +++ b/plugins/Manganelo.hpp @@ -6,5 +6,6 @@ namespace QuickMedia { class Manganelo : public Plugin { public: SearchResult search(const std::string &text, std::vector> &result_items) override; + SuggestionResult update_search_suggestions(const std::string &text, std::vector> &result_items) override; }; } \ No newline at end of file diff --git a/plugins/Plugin.hpp b/plugins/Plugin.hpp index bc518a8..9d62356 100644 --- a/plugins/Plugin.hpp +++ b/plugins/Plugin.hpp @@ -21,18 +21,31 @@ namespace QuickMedia { NET_ERR }; + enum class SuggestionResult { + OK, + ERR, + NET_ERR + }; + enum class DownloadResult { OK, ERR, NET_ERR }; + struct CommandArg { + std::string option; + std::string value; + }; + class Plugin { public: virtual ~Plugin() = default; virtual SearchResult search(const std::string &text, std::vector> &result_items) = 0; + virtual SuggestionResult update_search_suggestions(const std::string &text, std::vector> &result_items); protected: - DownloadResult download_to_string(const std::string &url, std::string &result); + DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector &additional_args = {}); + std::string url_param_encode(const std::string ¶m) const; }; } \ No newline at end of file diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp index ea2918d..073cb0c 100644 --- a/plugins/Youtube.hpp +++ b/plugins/Youtube.hpp @@ -6,5 +6,6 @@ namespace QuickMedia { class Youtube : public Plugin { public: SearchResult search(const std::string &text, std::vector> &result_items) override; + SuggestionResult update_search_suggestions(const std::string &text, std::vector> &result_items) override; }; } \ No newline at end of file diff --git a/project.conf b/project.conf index c0fdf68..c24ca64 100644 --- a/project.conf +++ b/project.conf @@ -5,4 +5,5 @@ version = "0.1.0" platforms = ["posix"] [dependencies] -sfml-graphics = "2" \ No newline at end of file +sfml-graphics = "2" +jsoncpp = "1.5" \ No newline at end of file diff --git a/src/Manganelo.cpp b/src/Manganelo.cpp index 56a6d50..85a3802 100644 --- a/src/Manganelo.cpp +++ b/src/Manganelo.cpp @@ -1,11 +1,11 @@ #include "../plugins/Manganelo.hpp" #include +#include namespace QuickMedia { SearchResult Manganelo::search(const std::string &text, std::vector> &result_items) { std::string url = "https://manganelo.com/search/"; - // TODO: Escape @text. Need to replace space with underscore for example. - url += text; + url += url_param_encode(text); std::string website_data; if(download_to_string(url, website_data) != DownloadResult::OK) @@ -48,4 +48,63 @@ namespace QuickMedia { quickmedia_html_search_deinit(&html_search); return result == 0 ? SearchResult::OK : SearchResult::ERR; } + + static void remove_html_span(std::string &str) { + size_t open_tag_start = str.find("', open_tag_start + 5); + if(open_tag_end == std::string::npos) + return; + + str.erase(open_tag_start, open_tag_end - open_tag_start + 1); + + size_t close_tag = str.find(""); + if(close_tag == std::string::npos) + return; + + str.erase(close_tag, 7); + } + + SuggestionResult Manganelo::update_search_suggestions(const std::string &text, std::vector> &result_items) { + if(text.empty()) + return SuggestionResult::OK; + + std::string url = "https://manganelo.com/home_json_search"; + std::string search_term = "searchword="; + search_term += url_param_encode(text); + CommandArg data_arg = { "--data", std::move(search_term) }; + + std::string server_response; + if(download_to_string(url, server_response, {data_arg}) != DownloadResult::OK) + return SuggestionResult::NET_ERR; + + if(server_response.empty()) + return SuggestionResult::OK; + + Json::Value json_root; + Json::CharReaderBuilder json_builder; + std::unique_ptr json_reader(json_builder.newCharReader()); + std::string json_errors; + if(json_reader->parse(&server_response.front(), &server_response.back(), &json_root, &json_errors)) { + fprintf(stderr, "Manganelo suggestions json error: %s\n", json_errors.c_str()); + return SuggestionResult::ERR; + } + + if(json_root.isArray()) { + for(const Json::Value &child : json_root) { + if(child.isObject()) { + Json::Value name = child.get("name", ""); + if(name.isString() && name.asCString()[0] != '\0') { + std::string name_str = name.asString(); + remove_html_span(name_str); + auto item = std::make_unique(name_str); + result_items.push_back(std::move(item)); + } + } + } + } + return SuggestionResult::OK; + } } \ No newline at end of file diff --git a/src/Plugin.cpp b/src/Plugin.cpp index d2fe925..382ebbf 100644 --- a/src/Plugin.cpp +++ b/src/Plugin.cpp @@ -1,5 +1,7 @@ #include "../plugins/Plugin.hpp" #include "../include/Program.h" +#include +#include static int accumulate_string(char *data, int size, void *userdata) { std::string *str = (std::string*)userdata; @@ -8,10 +10,38 @@ static int accumulate_string(char *data, int size, void *userdata) { } namespace QuickMedia { - DownloadResult Plugin::download_to_string(const std::string &url, std::string &result) { - const char *args[] = { "curl", "-H", "Accept-Language: en-US,en;q=0.5", "--compressed", "-s", "-L", url.c_str(), nullptr }; - if(exec_program(args, accumulate_string, &result) != 0) + SuggestionResult Plugin::update_search_suggestions(const std::string &text, std::vector> &result_items) { + (void)text; + (void)result_items; + return SuggestionResult::OK; + } + + DownloadResult Plugin::download_to_string(const std::string &url, std::string &result, const std::vector &additional_args) { + std::vector args = { "curl", "-H", "Accept-Language: en-US,en;q=0.5", "--compressed", "-s", "-L", url.c_str() }; + for(const CommandArg &arg : additional_args) { + args.push_back(arg.option.c_str()); + args.push_back(arg.value.c_str()); + } + args.push_back(nullptr); + if(exec_program(args.data(), accumulate_string, &result) != 0) return DownloadResult::NET_ERR; return DownloadResult::OK; } + + std::string Plugin::url_param_encode(const std::string ¶m) const { + std::ostringstream result; + result.fill('0'); + result << std::hex; + + for(char c : param) { + if(isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') { + result << c; + } else { + result << std::uppercase; + result << "%" << std::setw(2) << (int)(unsigned char)(c); + } + } + + return result.str(); + } } \ No newline at end of file diff --git a/src/Youtube.cpp b/src/Youtube.cpp index d8b046f..9f9c4c2 100644 --- a/src/Youtube.cpp +++ b/src/Youtube.cpp @@ -1,11 +1,11 @@ #include "../plugins/Youtube.hpp" #include +#include namespace QuickMedia { SearchResult Youtube::search(const std::string &text, std::vector> &result_items) { std::string url = "https://youtube.com/results?search_query="; - // TODO: Escape @text - url += text; + url += url_param_encode(text); std::string website_data; if(download_to_string(url, website_data) != DownloadResult::OK) @@ -21,8 +21,6 @@ namespace QuickMedia { auto *result_items = (std::vector>*)userdata; const char *href = quickmedia_html_node_get_attribute_value(node, "href"); const char *title = quickmedia_html_node_get_attribute_value(node, "title"); - printf("a href: %s, title: %s\n", href, title); - auto item = std::make_unique(title); result_items->push_back(std::move(item)); }, &result_items); @@ -31,4 +29,56 @@ namespace QuickMedia { quickmedia_html_search_deinit(&html_search); return result == 0 ? SearchResult::OK : SearchResult::ERR; } + + static void iterate_suggestion_result(const Json::Value &value, std::vector> &result_items, int &ignore_count) { + if(value.isArray()) { + for(const Json::Value &child : value) { + iterate_suggestion_result(child, result_items, ignore_count); + } + } else if(value.isString()) { + if(ignore_count > 1) { + auto item = std::make_unique(value.asString()); + result_items.push_back(std::move(item)); + } + ++ignore_count; + } + } + + SuggestionResult Youtube::update_search_suggestions(const std::string &text, std::vector> &result_items) { + if(text.empty()) + return SuggestionResult::OK; + + std::string url = "https://clients1.google.com/complete/search?client=youtube&hl=en&gl=us&q="; + url += url_param_encode(text); + + std::string server_response; + if(download_to_string(url, server_response) != DownloadResult::OK) + return SuggestionResult::NET_ERR; + + size_t json_start = server_response.find_first_of('('); + if(json_start == std::string::npos) + return SuggestionResult::ERR; + ++json_start; + + size_t json_end = server_response.find_last_of(')'); + if(json_end == std::string::npos) + return SuggestionResult::ERR; + + if(json_end == 0 || json_start >= json_end) + return SuggestionResult::ERR; + --json_end; + + Json::Value json_root; + Json::CharReaderBuilder json_builder; + std::unique_ptr json_reader(json_builder.newCharReader()); + std::string json_errors; + if(json_reader->parse(&server_response[json_start], &server_response[json_end], &json_root, &json_errors)) { + fprintf(stderr, "Youtube suggestions json error: %s\n", json_errors.c_str()); + return SuggestionResult::ERR; + } + + int ignore_count = 0; + iterate_suggestion_result(json_root, result_items, ignore_count); + return SuggestionResult::OK; + } } \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index 4689bc5..c3c91b7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -81,12 +81,18 @@ namespace QuickMedia { }; } -static void search(sf::String text, QuickMedia::Body *body, QuickMedia::Plugin *plugin) { +static void search(const sf::String &text, QuickMedia::Body *body, QuickMedia::Plugin *plugin) { body->clear_items(); QuickMedia::SearchResult search_result = plugin->search(text, body->items); fprintf(stderr, "Search result: %d\n", search_result); } +static void update_search_suggestions(const sf::String &text, QuickMedia::Body *body, QuickMedia::Plugin *plugin) { + body->clear_items(); + QuickMedia::SuggestionResult suggestion_result = plugin->update_search_suggestions(text, body->items); + fprintf(stderr, "Suggestion result: %d\n", suggestion_result); +} + int main() { const float padding_horizontal = 10.0f; const float padding_vertical = 10.0f; @@ -120,10 +126,14 @@ int main() { QuickMedia::Body body(font); QuickMedia::Manganelo manganelo_plugin; QuickMedia::Youtube youtube_plugin; - QuickMedia::Plugin *plugin = &youtube_plugin; + QuickMedia::Plugin *plugin = &manganelo_plugin; + + sf::Clock time_since_search_update; + bool updated_search = false; while (window.isOpen()) { sf::Event event; + while (window.pollEvent(event)) { if (event.type == sf::Event::Closed) window.close(); @@ -150,6 +160,8 @@ int main() { search_text.setString("Search..."); search_text.setFillColor(text_placeholder_color); } + updated_search = true; + time_since_search_update.restart(); } } else if(event.text.unicode == 13 && !show_placeholder) { // Return body.reset_selected(); @@ -166,10 +178,20 @@ int main() { sf::String str = search_text.getString(); str += event.text.unicode; search_text.setString(str); + updated_search = true; + time_since_search_update.restart(); } } } + if(updated_search && time_since_search_update.getElapsedTime().asMilliseconds() >= 90) { + updated_search = false; + sf::String str = search_text.getString(); + if(show_placeholder) + str.clear(); + update_search_suggestions(str, &body, plugin); + } + if(resized) { resized = false; -- cgit v1.2.3