diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | images/mangatown_logo.png | bin | 0 -> 7664 bytes | |||
| -rw-r--r-- | include/DownloadUtils.hpp | 4 | ||||
| -rw-r--r-- | plugins/Mangatown.hpp | 27 | ||||
| -rw-r--r-- | src/DownloadUtils.cpp | 10 | ||||
| -rw-r--r-- | src/QuickMedia.cpp | 6 | ||||
| -rw-r--r-- | src/plugins/Mangadex.cpp | 14 | ||||
| -rw-r--r-- | src/plugins/Mangatown.cpp | 207 |
8 files changed, 255 insertions, 17 deletions
@@ -1,6 +1,6 @@ # QuickMedia Native clients of websites with fast access to what you want to see, **with TOR support**. See [old video demo with manga](https://lbry.tv/quickmedia_manga-2019-08-05_21.20.46/7).\ -Currently supported websites: `youtube`, `manganelo`, `mangadex`, `4chan` and _others_.\ +Currently supported websites: `youtube`, `manganelo`, `mangatown`, `mangadex`, `4chan` and _others_.\ **Note:** Manganelo doesn't work when used with TOR.\ **Note:** Posting comments on 4chan doesn't work when used with TOR. However browing works.\ **Note:** TOR system service needs to be running (`systemctl start tor.service`).\ @@ -12,7 +12,7 @@ Cache is stored under `$HOME/.cache/quickmedia`. ``` usage: QuickMedia <plugin> [--tor] OPTIONS: -plugin The plugin to use. Should be either 4chan, manganelo, mangadex, youtube or dmenu +plugin The plugin to use. Should be either 4chan, manganelo, mangatown, mangadex, youtube or dmenu --tor Use tor. Disabled by default EXAMPLES: QuickMedia manganelo diff --git a/images/mangatown_logo.png b/images/mangatown_logo.png Binary files differnew file mode 100644 index 0000000..1f85281 --- /dev/null +++ b/images/mangatown_logo.png diff --git a/include/DownloadUtils.hpp b/include/DownloadUtils.hpp index 622f5fe..6af53ac 100644 --- a/include/DownloadUtils.hpp +++ b/include/DownloadUtils.hpp @@ -20,7 +20,7 @@ namespace QuickMedia { std::string value; }; - DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor); - DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor); + DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent = false); + DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent = false); std::vector<CommandArg> create_command_args_from_form_data(const std::vector<FormData> &form_data); }
\ No newline at end of file diff --git a/plugins/Mangatown.hpp b/plugins/Mangatown.hpp new file mode 100644 index 0000000..bfb1f6a --- /dev/null +++ b/plugins/Mangatown.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include "Manga.hpp" +#include <functional> +#include <mutex> + +namespace QuickMedia { + class Mangatown : public Manga { + public: + Mangatown() : Manga("mangatown") {} + SearchResult search(const std::string &url, BodyItems &result_items) override; + SuggestionResult update_search_suggestions(const std::string &text, BodyItems &result_items) override; + ImageResult get_number_of_images(const std::string &url, int &num_images) override; + bool search_suggestions_has_thumbnails() const override { return false; } + bool search_results_has_thumbnails() const override { return false; } + int get_search_delay() const override { return 150; } + Page get_page_after_search() const override { return Page::EPISODE_LIST; } + + ImageResult for_each_page_in_chapter(const std::string &chapter_url, PageCallback callback) override; + + bool extract_id_from_url(const std::string &url, std::string &manga_id) override; + private: + std::string last_chapter_url_num_images; + int last_num_pages = 0; + std::mutex image_urls_mutex; + }; +}
\ No newline at end of file diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp index 6f31c2c..129234f 100644 --- a/src/DownloadUtils.cpp +++ b/src/DownloadUtils.cpp @@ -10,9 +10,11 @@ static int accumulate_string(char *data, int size, void *userdata) { return 0; } +static const char *useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; + namespace QuickMedia { // TODO: Add timeout - DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor) { + DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent) { sf::Clock timer; std::vector<const char*> args; if(use_tor) @@ -22,6 +24,10 @@ namespace QuickMedia { args.push_back(arg.option.c_str()); args.push_back(arg.value.c_str()); } + if(use_browser_useragent) { + args.push_back("-H"); + args.push_back(useragent_str); + } args.push_back("--"); args.push_back(url.c_str()); args.push_back(nullptr); @@ -31,7 +37,7 @@ namespace QuickMedia { return DownloadResult::OK; } - DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor) { + DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent) { Path media_dir = get_cache_dir().join("media"); Path media_file_path = Path(media_dir).join(cppcodec::base64_rfc4648::encode(url)); Path media_finished_path(media_file_path.data + ".finished"); diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 644c644..6cd92b7 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -1,5 +1,6 @@ #include "../include/QuickMedia.hpp" #include "../plugins/Manganelo.hpp" +#include "../plugins/Mangatown.hpp" #include "../plugins/Mangadex.hpp" #include "../plugins/Youtube.hpp" #include "../plugins/Pornhub.hpp" @@ -99,7 +100,7 @@ namespace QuickMedia { static void usage() { fprintf(stderr, "usage: QuickMedia <plugin> [--tor]\n"); fprintf(stderr, "OPTIONS:\n"); - fprintf(stderr, "plugin The plugin to use. Should be either 4chan, manganelo, mangadex, pornhub, youtube or dmenu\n"); + fprintf(stderr, "plugin The plugin to use. Should be either 4chan, manganelo, mangatown, mangadex, pornhub, youtube or dmenu\n"); fprintf(stderr, "--tor Use tor. Disabled by default\n"); fprintf(stderr, "EXAMPLES:\n"); fprintf(stderr, "QuickMedia manganelo\n"); @@ -141,6 +142,9 @@ namespace QuickMedia { if(strcmp(argv[i], "manganelo") == 0) { current_plugin = new Manganelo(); plugin_logo_path = "../../../images/manganelo_logo.png"; + } else if(strcmp(argv[i], "mangatown") == 0) { + current_plugin = new Mangatown(); + plugin_logo_path = "../../../images/mangatown_logo.png"; } else if(strcmp(argv[i], "mangadex") == 0) { current_plugin = new Mangadex(); plugin_logo_path = "../../../images/mangadex_logo.png"; diff --git a/src/plugins/Mangadex.cpp b/src/plugins/Mangadex.cpp index 3518d4a..b3c5e36 100644 --- a/src/plugins/Mangadex.cpp +++ b/src/plugins/Mangadex.cpp @@ -5,7 +5,6 @@ #include <json/reader.h> static const std::string mangadex_url = "https://mangadex.org"; -static const std::string useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; namespace QuickMedia { static std::string title_url_extract_manga_id(const std::string &url) { @@ -35,12 +34,10 @@ namespace QuickMedia { }; SearchResult Mangadex::search(const std::string &url, BodyItems &result_items) { - CommandArg user_agent_arg = { "-H", useragent_str }; - std::string manga_id = title_url_extract_manga_id(url); std::string request_url = "https://mangadex.org/api/?id=" + manga_id + "&type=manga"; std::string server_response; - if(download_to_string(request_url, server_response, {std::move(user_agent_arg)}, use_tor) != DownloadResult::OK) + if(download_to_string(request_url, server_response, {}, use_tor, true) != DownloadResult::OK) return SearchResult::NET_ERR; if(server_response.empty()) @@ -170,10 +167,9 @@ namespace QuickMedia { std::string url = "https://mangadex.org/search?title="; url += url_param_encode(text); CommandArg cookie_arg = { "-H", "cookie: mangadex_rememberme_token=" + rememberme_token }; - CommandArg user_agent_arg = { "-H", useragent_str }; std::string website_data; - if(download_to_string(url, website_data, {std::move(cookie_arg), std::move(user_agent_arg)}, use_tor) != DownloadResult::OK) + if(download_to_string(url, website_data, {std::move(cookie_arg)}, use_tor, true) != DownloadResult::OK) return SuggestionResult::NET_ERR; QuickMediaHtmlSearch html_search; @@ -226,10 +222,9 @@ namespace QuickMedia { } bool Mangadex::save_mangadex_cookies(const std::string &url, const std::string &cookie_filepath) { - CommandArg user_agent_arg = { "-H", useragent_str }; CommandArg cookie_arg = { "-c", std::move(cookie_filepath) }; std::string server_response; - if(download_to_string(url, server_response, {std::move(user_agent_arg), std::move(cookie_arg)}, use_tor) != DownloadResult::OK) + if(download_to_string(url, server_response, {std::move(cookie_arg)}, use_tor, true) != DownloadResult::OK) return false; return true; @@ -248,12 +243,11 @@ namespace QuickMedia { if(!save_mangadex_cookies(url, cookie_filepath)) return ImageResult::ERR; - CommandArg user_agent_arg = { "-H", useragent_str }; CommandArg cookie_arg = { "-b", std::move(cookie_filepath) }; std::string manga_id = chapter_url_extract_manga_id(url); std::string request_url = mangadex_url + "/api/?id=" + manga_id + "&server=null&type=chapter"; std::string server_response; - if(download_to_string(request_url, server_response, {std::move(user_agent_arg), std::move(cookie_arg)}, use_tor) != DownloadResult::OK) + if(download_to_string(request_url, server_response, {std::move(cookie_arg)}, use_tor, true) != DownloadResult::OK) return ImageResult::NET_ERR; if(server_response.empty()) diff --git a/src/plugins/Mangatown.cpp b/src/plugins/Mangatown.cpp new file mode 100644 index 0000000..90db349 --- /dev/null +++ b/src/plugins/Mangatown.cpp @@ -0,0 +1,207 @@ +#include "../../plugins/Mangatown.hpp" +#include "../../include/Notification.hpp" +#include <quickmedia/HtmlSearch.h> +#include <json/reader.h> + +static const std::string mangatown_url = "https://www.mangatown.com"; + +namespace QuickMedia { + SearchResult Mangatown::search(const std::string &url, BodyItems &result_items) { + std::string website_data; + if(download_to_string(url, website_data, {}, use_tor, true) != DownloadResult::OK) + return SearchResult::NET_ERR; + + QuickMediaHtmlSearch html_search; + int result = quickmedia_html_search_init(&html_search, website_data.c_str()); + if(result != 0) + goto cleanup; + + result = quickmedia_html_find_nodes_xpath(&html_search, "//ul[class='chapter_list']//a", + [](QuickMediaHtmlNode *node, void *userdata) { + auto *item_data = (BodyItems*)userdata; + const char *href = quickmedia_html_node_get_attribute_value(node, "href"); + const char *text = quickmedia_html_node_get_text(node); + if(href && text && strncmp(href, "/manga/", 7) == 0) { + auto item = std::make_unique<BodyItem>(strip(text)); + item->url = mangatown_url + href; + item_data->push_back(std::move(item)); + } + }, &result_items); + + cleanup: + quickmedia_html_search_deinit(&html_search); + return result == 0 ? SearchResult::OK : SearchResult::ERR; + } + + SuggestionResult Mangatown::update_search_suggestions(const std::string &text, BodyItems &result_items) { + std::string url = "https://www.mangatown.com/ajax/search/?query="; + url += url_param_encode(text); + + std::string server_response; + if(download_to_string(url, server_response, {}, use_tor, true) != DownloadResult::OK) + return SuggestionResult::NET_ERR; + + if(server_response.empty()) + return SuggestionResult::OK; + + Json::Value json_root; + Json::CharReaderBuilder json_builder; + std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader()); + std::string json_errors; + if(!json_reader->parse(&server_response[0], &server_response[server_response.size()], &json_root, &json_errors)) { + fprintf(stderr, "Mangatown suggestions json error: %s\n", json_errors.c_str()); + return SuggestionResult::ERR; + } + + if(!json_root.isObject()) + return SuggestionResult::OK; + + Json::Value &json_data = json_root["data"]; + Json::Value &json_suggestions = json_root["suggestions"]; + if(!json_data.isArray() || !json_suggestions.isArray()) + return SuggestionResult::OK; + + for(const Json::Value &child : json_suggestions) { + if(!child.isString()) { + result_items.push_back(std::make_unique<BodyItem>("")); + continue; + } + result_items.push_back(std::make_unique<BodyItem>(child.asString())); + } + + size_t index = 0; + for(const Json::Value &child : json_data) { + BodyItem *body_item = nullptr; + if(index < result_items.size()) { + body_item = result_items[index].get(); + } else { + result_items.push_back(std::make_unique<BodyItem>("")); + body_item = result_items.back().get(); + } + + ++index; + + if(!child.isString()) + continue; + + body_item->url = mangatown_url + child.asString(); + } + + return SuggestionResult::OK; + } + + static bool is_number_with_zero_fill(const char *str) { + while(*str == '0') { ++str; } + return atoi(str) != 0; + } + + ImageResult Mangatown::get_number_of_images(const std::string &url, int &num_images) { + std::lock_guard<std::mutex> lock(image_urls_mutex); + + num_images = last_num_pages; + if(url == last_chapter_url_num_images) + return ImageResult::OK; + + last_num_pages = 0; + + std::string website_data; + if(download_to_string(url, website_data, {}, use_tor, true) != DownloadResult::OK) + return ImageResult::NET_ERR; + + QuickMediaHtmlSearch html_search; + int result = quickmedia_html_search_init(&html_search, website_data.c_str()); + if(result != 0) + goto cleanup; + + result = quickmedia_html_find_nodes_xpath(&html_search, "//div[class='page_select']//option", + [](QuickMediaHtmlNode *node, void *userdata) { + int *last_num_pages = (int*)userdata; + const char *value = quickmedia_html_node_get_attribute_value(node, "value"); + const char *text = quickmedia_html_node_get_text(node); + if(value && strncmp(value, "/manga/", 7) == 0) { + if(is_number_with_zero_fill(text)) { + (*last_num_pages)++; + } + } + }, &last_num_pages); + + last_num_pages /= 2; + num_images = last_num_pages; + + cleanup: + quickmedia_html_search_deinit(&html_search); + + if(result == 0) + last_chapter_url_num_images = url; + if(last_num_pages == 0) { + last_chapter_url_num_images.clear(); + return ImageResult::ERR; + } + return result == 0 ? ImageResult::OK : ImageResult::ERR; + } + + ImageResult Mangatown::for_each_page_in_chapter(const std::string &chapter_url, PageCallback callback) { + int num_pages; + ImageResult image_result = get_number_of_images(chapter_url, num_pages); + if(image_result != ImageResult::OK) + return image_result; + + int result = 0; + int page_index = 1; + + while(true) { + std::string image_src; + std::string website_data; + std::string full_url = chapter_url + std::to_string(page_index++) + ".html"; + if(download_to_string(full_url, website_data, {}, use_tor, true) != DownloadResult::OK) + break; + + QuickMediaHtmlSearch html_search; + result = quickmedia_html_search_init(&html_search, website_data.c_str()); + if(result != 0) + goto cleanup; + + result = quickmedia_html_find_nodes_xpath(&html_search, "//div[id='viewer']//img", + [](QuickMediaHtmlNode *node, void *userdata) { + std::string *image_src = (std::string*)userdata; + const char *src = quickmedia_html_node_get_attribute_value(node, "src"); + if(src && strstr(src, "/store/manga/")) { + if(strncmp(src, "//", 2) == 0) + *image_src = src + 2; + else + *image_src = src; + } + }, &image_src); + + cleanup: + quickmedia_html_search_deinit(&html_search); + + if(result != 0) + return ImageResult::ERR; + + if(image_src.empty()) + break; + + if(!callback(image_src)) + break; + } + + return ImageResult::OK; + } + + bool Mangatown::extract_id_from_url(const std::string &url, std::string &manga_id) { + size_t start_index = url.find("/manga/"); + if(start_index == std::string::npos) + return false; + + start_index += 7; + size_t end_index = url.find("/", start_index); + if(end_index == std::string::npos) { + manga_id = url.substr(start_index); + return true; + } + + manga_id = url.substr(start_index, end_index - start_index); + return true; + } +}
\ No newline at end of file |
