Add support for mangatown

8 files changed, 255 insertions, 17 deletions

diff --git a/README.md b/README.md
index 4713433..3f46c6b 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # QuickMedia
 Native clients of websites with fast access to what you want to see, **with TOR support**. See [old video demo with manga](https://lbry.tv/quickmedia_manga-2019-08-05_21.20.46/7).\
-Currently supported websites: `youtube`, `manganelo`, `mangadex`, `4chan` and _others_.\
+Currently supported websites: `youtube`, `manganelo`, `mangatown`, `mangadex`, `4chan` and _others_.\
 **Note:** Manganelo doesn't work when used with TOR.\
 **Note:** Posting comments on 4chan doesn't work when used with TOR. However browing works.\
 **Note:** TOR system service needs to be running (`systemctl start tor.service`).\
@@ -12,7 +12,7 @@ Cache is stored under `$HOME/.cache/quickmedia`.
 ```
 usage: QuickMedia <plugin> [--tor]
 OPTIONS:
-plugin    The plugin to use. Should be either 4chan, manganelo, mangadex, youtube or dmenu
+plugin    The plugin to use. Should be either 4chan, manganelo, mangatown, mangadex, youtube or dmenu
 --tor     Use tor. Disabled by default
 EXAMPLES:
 QuickMedia manganelo
diff --git a/images/mangatown_logo.png b/images/mangatown_logo.png
new file mode 100644
index 0000000..1f85281
--- /dev/null
+++ b/images/mangatown_logo.png
diff --git a/include/DownloadUtils.hpp b/include/DownloadUtils.hpp
index 622f5fe..6af53ac 100644
--- a/include/DownloadUtils.hpp
+++ b/include/DownloadUtils.hpp
@@ -20,7 +20,7 @@ namespace QuickMedia {
         std::string value;
     };
 
-    DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor);
-    DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor);
+    DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent = false);
+    DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent = false);
     std::vector<CommandArg> create_command_args_from_form_data(const std::vector<FormData> &form_data);
 }
\ No newline at end of file
diff --git a/plugins/Mangatown.hpp b/plugins/Mangatown.hpp
new file mode 100644
index 0000000..bfb1f6a
--- /dev/null
+++ b/plugins/Mangatown.hpp
@@ -0,0 +1,27 @@
+#pragma once
+
+#include "Manga.hpp"
+#include <functional>
+#include <mutex>
+
+namespace QuickMedia {
+    class Mangatown : public Manga {
+    public:
+        Mangatown() : Manga("mangatown") {}
+        SearchResult search(const std::string &url, BodyItems &result_items) override;
+        SuggestionResult update_search_suggestions(const std::string &text, BodyItems &result_items) override;
+        ImageResult get_number_of_images(const std::string &url, int &num_images) override;
+        bool search_suggestions_has_thumbnails() const override { return false; }
+        bool search_results_has_thumbnails() const override { return false; }
+        int get_search_delay() const override { return 150; }
+        Page get_page_after_search() const override { return Page::EPISODE_LIST; }
+
+        ImageResult for_each_page_in_chapter(const std::string &chapter_url, PageCallback callback) override;
+
+        bool extract_id_from_url(const std::string &url, std::string &manga_id) override;
+    private:
+        std::string last_chapter_url_num_images;
+        int last_num_pages = 0;
+        std::mutex image_urls_mutex;
+    };
+}
\ No newline at end of file
diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp
index 6f31c2c..129234f 100644
--- a/src/DownloadUtils.cpp
+++ b/src/DownloadUtils.cpp
@@ -10,9 +10,11 @@ static int accumulate_string(char *data, int size, void *userdata) {
     return 0;
 }
 
+static const char *useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36";
+
 namespace QuickMedia {
     // TODO: Add timeout
-    DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor) {
+    DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent) {
         sf::Clock timer;
         std::vector<const char*> args;
         if(use_tor)
@@ -22,6 +24,10 @@ namespace QuickMedia {
             args.push_back(arg.option.c_str());
             args.push_back(arg.value.c_str());
         }
+        if(use_browser_useragent) {
+            args.push_back("-H");
+            args.push_back(useragent_str);
+        }
         args.push_back("--");
         args.push_back(url.c_str());
         args.push_back(nullptr);
@@ -31,7 +37,7 @@ namespace QuickMedia {
         return DownloadResult::OK;
     }
 
-    DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor) {
+    DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent) {
         Path media_dir = get_cache_dir().join("media");
         Path media_file_path = Path(media_dir).join(cppcodec::base64_rfc4648::encode(url));
         Path media_finished_path(media_file_path.data + ".finished");
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 644c644..6cd92b7 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -1,5 +1,6 @@
 #include "../include/QuickMedia.hpp"
 #include "../plugins/Manganelo.hpp"
+#include "../plugins/Mangatown.hpp"
 #include "../plugins/Mangadex.hpp"
 #include "../plugins/Youtube.hpp"
 #include "../plugins/Pornhub.hpp"
@@ -99,7 +100,7 @@ namespace QuickMedia {
     static void usage() {
         fprintf(stderr, "usage: QuickMedia <plugin> [--tor]\n");
         fprintf(stderr, "OPTIONS:\n");
-        fprintf(stderr, "plugin    The plugin to use. Should be either 4chan, manganelo, mangadex, pornhub, youtube or dmenu\n");
+        fprintf(stderr, "plugin    The plugin to use. Should be either 4chan, manganelo, mangatown, mangadex, pornhub, youtube or dmenu\n");
         fprintf(stderr, "--tor     Use tor. Disabled by default\n");
         fprintf(stderr, "EXAMPLES:\n");
         fprintf(stderr, "QuickMedia manganelo\n");
@@ -141,6 +142,9 @@ namespace QuickMedia {
                 if(strcmp(argv[i], "manganelo") == 0) {
                     current_plugin = new Manganelo();
                     plugin_logo_path = "../../../images/manganelo_logo.png";
+                } else if(strcmp(argv[i], "mangatown") == 0) {
+                    current_plugin = new Mangatown();
+                    plugin_logo_path = "../../../images/mangatown_logo.png";
                 } else if(strcmp(argv[i], "mangadex") == 0) {
                     current_plugin = new Mangadex();
                     plugin_logo_path = "../../../images/mangadex_logo.png";
diff --git a/src/plugins/Mangadex.cpp b/src/plugins/Mangadex.cpp
index 3518d4a..b3c5e36 100644
--- a/src/plugins/Mangadex.cpp
+++ b/src/plugins/Mangadex.cpp
@@ -5,7 +5,6 @@
 #include <json/reader.h>
 
 static const std::string mangadex_url = "https://mangadex.org";
-static const std::string useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36";
 
 namespace QuickMedia {
     static std::string title_url_extract_manga_id(const std::string &url) {
@@ -35,12 +34,10 @@ namespace QuickMedia {
     };
 
     SearchResult Mangadex::search(const std::string &url, BodyItems &result_items) {
-        CommandArg user_agent_arg = { "-H", useragent_str };
-
         std::string manga_id = title_url_extract_manga_id(url);
         std::string request_url = "https://mangadex.org/api/?id=" + manga_id + "&type=manga";
         std::string server_response;
-        if(download_to_string(request_url, server_response, {std::move(user_agent_arg)}, use_tor) != DownloadResult::OK)
+        if(download_to_string(request_url, server_response, {}, use_tor, true) != DownloadResult::OK)
             return SearchResult::NET_ERR;
 
         if(server_response.empty())
@@ -170,10 +167,9 @@ namespace QuickMedia {
         std::string url = "https://mangadex.org/search?title=";
         url += url_param_encode(text);
         CommandArg cookie_arg = { "-H", "cookie: mangadex_rememberme_token=" + rememberme_token };
-        CommandArg user_agent_arg = { "-H", useragent_str };
 
         std::string website_data;
-        if(download_to_string(url, website_data, {std::move(cookie_arg), std::move(user_agent_arg)}, use_tor) != DownloadResult::OK)
+        if(download_to_string(url, website_data, {std::move(cookie_arg)}, use_tor, true) != DownloadResult::OK)
             return SuggestionResult::NET_ERR;
 
         QuickMediaHtmlSearch html_search;
@@ -226,10 +222,9 @@ namespace QuickMedia {
     }
 
     bool Mangadex::save_mangadex_cookies(const std::string &url, const std::string &cookie_filepath) {
-        CommandArg user_agent_arg = { "-H", useragent_str };
         CommandArg cookie_arg = { "-c", std::move(cookie_filepath) };
         std::string server_response;
-        if(download_to_string(url, server_response, {std::move(user_agent_arg), std::move(cookie_arg)}, use_tor) != DownloadResult::OK)
+        if(download_to_string(url, server_response, {std::move(cookie_arg)}, use_tor, true) != DownloadResult::OK)
             return false;
 
         return true;
@@ -248,12 +243,11 @@ namespace QuickMedia {
         if(!save_mangadex_cookies(url, cookie_filepath))
             return ImageResult::ERR;
 
-        CommandArg user_agent_arg = { "-H", useragent_str };
         CommandArg cookie_arg = { "-b", std::move(cookie_filepath) };
         std::string manga_id = chapter_url_extract_manga_id(url);
         std::string request_url = mangadex_url + "/api/?id=" + manga_id + "&server=null&type=chapter";
         std::string server_response;
-        if(download_to_string(request_url, server_response, {std::move(user_agent_arg), std::move(cookie_arg)}, use_tor) != DownloadResult::OK)
+        if(download_to_string(request_url, server_response, {std::move(cookie_arg)}, use_tor, true) != DownloadResult::OK)
             return ImageResult::NET_ERR;
 
         if(server_response.empty())
diff --git a/src/plugins/Mangatown.cpp b/src/plugins/Mangatown.cpp
new file mode 100644
index 0000000..90db349
--- /dev/null
+++ b/src/plugins/Mangatown.cpp
@@ -0,0 +1,207 @@
+#include "../../plugins/Mangatown.hpp"
+#include "../../include/Notification.hpp"
+#include <quickmedia/HtmlSearch.h>
+#include <json/reader.h>
+
+static const std::string mangatown_url = "https://www.mangatown.com";
+
+namespace QuickMedia {
+    SearchResult Mangatown::search(const std::string &url, BodyItems &result_items) {
+        std::string website_data;
+        if(download_to_string(url, website_data, {}, use_tor, true) != DownloadResult::OK)
+            return SearchResult::NET_ERR;
+
+        QuickMediaHtmlSearch html_search;
+        int result = quickmedia_html_search_init(&html_search, website_data.c_str());
+        if(result != 0)
+            goto cleanup;
+
+        result = quickmedia_html_find_nodes_xpath(&html_search, "//ul[class='chapter_list']//a",
+            [](QuickMediaHtmlNode *node, void *userdata) {
+                auto *item_data = (BodyItems*)userdata;
+                const char *href = quickmedia_html_node_get_attribute_value(node, "href");
+                const char *text = quickmedia_html_node_get_text(node);
+                if(href && text && strncmp(href, "/manga/", 7) == 0) {
+                    auto item = std::make_unique<BodyItem>(strip(text));
+                    item->url = mangatown_url + href;
+                    item_data->push_back(std::move(item));
+                }
+            }, &result_items);
+
+        cleanup:
+        quickmedia_html_search_deinit(&html_search);
+        return result == 0 ? SearchResult::OK : SearchResult::ERR;
+    }
+
+    SuggestionResult Mangatown::update_search_suggestions(const std::string &text, BodyItems &result_items) {
+        std::string url = "https://www.mangatown.com/ajax/search/?query=";
+        url += url_param_encode(text);
+
+        std::string server_response;
+        if(download_to_string(url, server_response, {}, use_tor, true) != DownloadResult::OK)
+            return SuggestionResult::NET_ERR;
+
+        if(server_response.empty())
+            return SuggestionResult::OK;
+
+        Json::Value json_root;
+        Json::CharReaderBuilder json_builder;
+        std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader());
+        std::string json_errors;
+        if(!json_reader->parse(&server_response[0], &server_response[server_response.size()], &json_root, &json_errors)) {
+            fprintf(stderr, "Mangatown suggestions json error: %s\n", json_errors.c_str());
+            return SuggestionResult::ERR;
+        }
+
+        if(!json_root.isObject())
+            return SuggestionResult::OK;
+
+        Json::Value &json_data = json_root["data"];
+        Json::Value &json_suggestions = json_root["suggestions"];
+        if(!json_data.isArray() || !json_suggestions.isArray())
+            return SuggestionResult::OK;
+
+        for(const Json::Value &child : json_suggestions) {
+            if(!child.isString()) {
+                result_items.push_back(std::make_unique<BodyItem>(""));
+                continue;
+            }
+            result_items.push_back(std::make_unique<BodyItem>(child.asString()));
+        }
+        
+        size_t index = 0;
+        for(const Json::Value &child : json_data) {
+            BodyItem *body_item = nullptr;
+            if(index < result_items.size()) {
+                body_item = result_items[index].get();
+            } else {
+                result_items.push_back(std::make_unique<BodyItem>(""));
+                body_item = result_items.back().get();
+            }
+
+            ++index;
+
+            if(!child.isString())
+                continue;
+
+            body_item->url = mangatown_url + child.asString();
+        }
+
+        return SuggestionResult::OK;
+    }
+
+    static bool is_number_with_zero_fill(const char *str) {
+        while(*str == '0') { ++str; }
+        return atoi(str) != 0;
+    }
+
+    ImageResult Mangatown::get_number_of_images(const std::string &url, int &num_images) {
+        std::lock_guard<std::mutex> lock(image_urls_mutex);
+
+        num_images = last_num_pages;
+        if(url == last_chapter_url_num_images)
+            return ImageResult::OK;
+
+        last_num_pages = 0;
+
+        std::string website_data;
+        if(download_to_string(url, website_data, {}, use_tor, true) != DownloadResult::OK)
+            return ImageResult::NET_ERR;
+
+        QuickMediaHtmlSearch html_search;
+        int result = quickmedia_html_search_init(&html_search, website_data.c_str());
+        if(result != 0)
+            goto cleanup;
+
+        result = quickmedia_html_find_nodes_xpath(&html_search, "//div[class='page_select']//option",
+            [](QuickMediaHtmlNode *node, void *userdata) {
+                int *last_num_pages = (int*)userdata;
+                const char *value = quickmedia_html_node_get_attribute_value(node, "value");
+                const char *text = quickmedia_html_node_get_text(node);
+                if(value && strncmp(value, "/manga/", 7) == 0) {
+                    if(is_number_with_zero_fill(text)) {
+                        (*last_num_pages)++;
+                    }
+                }
+            }, &last_num_pages);
+
+        last_num_pages /= 2;
+        num_images = last_num_pages;
+
+        cleanup:
+        quickmedia_html_search_deinit(&html_search);
+
+        if(result == 0)
+            last_chapter_url_num_images = url;
+        if(last_num_pages == 0) {
+            last_chapter_url_num_images.clear();
+            return ImageResult::ERR;
+        }
+        return result == 0 ? ImageResult::OK : ImageResult::ERR;
+    }
+
+    ImageResult Mangatown::for_each_page_in_chapter(const std::string &chapter_url, PageCallback callback) {
+        int num_pages;
+        ImageResult image_result = get_number_of_images(chapter_url, num_pages);
+        if(image_result != ImageResult::OK)
+            return image_result;
+
+        int result = 0;
+        int page_index = 1;
+
+        while(true) {
+            std::string image_src;
+            std::string website_data;
+            std::string full_url = chapter_url + std::to_string(page_index++) + ".html";
+            if(download_to_string(full_url, website_data, {}, use_tor, true) != DownloadResult::OK)
+                break;
+
+            QuickMediaHtmlSearch html_search;
+            result = quickmedia_html_search_init(&html_search, website_data.c_str());
+            if(result != 0)
+                goto cleanup;
+
+            result = quickmedia_html_find_nodes_xpath(&html_search, "//div[id='viewer']//img",
+                [](QuickMediaHtmlNode *node, void *userdata) {
+                    std::string *image_src = (std::string*)userdata;
+                    const char *src = quickmedia_html_node_get_attribute_value(node, "src");
+                    if(src && strstr(src, "/store/manga/")) {
+                        if(strncmp(src, "//", 2) == 0)
+                            *image_src = src + 2;
+                        else
+                            *image_src = src;
+                    }
+                }, &image_src);
+
+            cleanup:
+            quickmedia_html_search_deinit(&html_search);
+
+            if(result != 0)
+                return ImageResult::ERR;
+
+            if(image_src.empty())
+                break;
+
+            if(!callback(image_src))
+                break;
+        }
+
+        return ImageResult::OK;
+    }
+
+    bool Mangatown::extract_id_from_url(const std::string &url, std::string &manga_id) {
+        size_t start_index = url.find("/manga/");
+        if(start_index == std::string::npos)
+            return false;
+        
+        start_index += 7;
+        size_t end_index = url.find("/", start_index);
+        if(end_index == std::string::npos) {
+            manga_id = url.substr(start_index);
+            return true;
+        }
+        
+        manga_id = url.substr(start_index, end_index - start_index);
+        return true;
+    }
+}
\ No newline at end of file