aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordec05eba <dec05eba@protonmail.com>2020-06-07 04:31:30 +0200
committerdec05eba <dec05eba@protonmail.com>2020-06-07 04:31:30 +0200
commita0dbe64272d051671bc120b1845c7485264b097d (patch)
treefdd5daf29cfd541624b9c5f06e9a171f7becd58b
parent5ccd94a771cf1031b837c7bc4d0baeab4b810a38 (diff)
Add support for mangatown
-rw-r--r--README.md4
-rw-r--r--images/mangatown_logo.pngbin0 -> 7664 bytes
-rw-r--r--include/DownloadUtils.hpp4
-rw-r--r--plugins/Mangatown.hpp27
-rw-r--r--src/DownloadUtils.cpp10
-rw-r--r--src/QuickMedia.cpp6
-rw-r--r--src/plugins/Mangadex.cpp14
-rw-r--r--src/plugins/Mangatown.cpp207
8 files changed, 255 insertions, 17 deletions
diff --git a/README.md b/README.md
index 4713433..3f46c6b 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
# QuickMedia
Native clients of websites with fast access to what you want to see, **with TOR support**. See [old video demo with manga](https://lbry.tv/quickmedia_manga-2019-08-05_21.20.46/7).\
-Currently supported websites: `youtube`, `manganelo`, `mangadex`, `4chan` and _others_.\
+Currently supported websites: `youtube`, `manganelo`, `mangatown`, `mangadex`, `4chan` and _others_.\
**Note:** Manganelo doesn't work when used with TOR.\
**Note:** Posting comments on 4chan doesn't work when used with TOR. However browing works.\
**Note:** TOR system service needs to be running (`systemctl start tor.service`).\
@@ -12,7 +12,7 @@ Cache is stored under `$HOME/.cache/quickmedia`.
```
usage: QuickMedia <plugin> [--tor]
OPTIONS:
-plugin The plugin to use. Should be either 4chan, manganelo, mangadex, youtube or dmenu
+plugin The plugin to use. Should be either 4chan, manganelo, mangatown, mangadex, youtube or dmenu
--tor Use tor. Disabled by default
EXAMPLES:
QuickMedia manganelo
diff --git a/images/mangatown_logo.png b/images/mangatown_logo.png
new file mode 100644
index 0000000..1f85281
--- /dev/null
+++ b/images/mangatown_logo.png
Binary files differ
diff --git a/include/DownloadUtils.hpp b/include/DownloadUtils.hpp
index 622f5fe..6af53ac 100644
--- a/include/DownloadUtils.hpp
+++ b/include/DownloadUtils.hpp
@@ -20,7 +20,7 @@ namespace QuickMedia {
std::string value;
};
- DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor);
- DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor);
+ DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent = false);
+ DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent = false);
std::vector<CommandArg> create_command_args_from_form_data(const std::vector<FormData> &form_data);
} \ No newline at end of file
diff --git a/plugins/Mangatown.hpp b/plugins/Mangatown.hpp
new file mode 100644
index 0000000..bfb1f6a
--- /dev/null
+++ b/plugins/Mangatown.hpp
@@ -0,0 +1,27 @@
+#pragma once
+
+#include "Manga.hpp"
+#include <functional>
+#include <mutex>
+
+namespace QuickMedia {
+ class Mangatown : public Manga {
+ public:
+ Mangatown() : Manga("mangatown") {}
+ SearchResult search(const std::string &url, BodyItems &result_items) override;
+ SuggestionResult update_search_suggestions(const std::string &text, BodyItems &result_items) override;
+ ImageResult get_number_of_images(const std::string &url, int &num_images) override;
+ bool search_suggestions_has_thumbnails() const override { return false; }
+ bool search_results_has_thumbnails() const override { return false; }
+ int get_search_delay() const override { return 150; }
+ Page get_page_after_search() const override { return Page::EPISODE_LIST; }
+
+ ImageResult for_each_page_in_chapter(const std::string &chapter_url, PageCallback callback) override;
+
+ bool extract_id_from_url(const std::string &url, std::string &manga_id) override;
+ private:
+ std::string last_chapter_url_num_images;
+ int last_num_pages = 0;
+ std::mutex image_urls_mutex;
+ };
+} \ No newline at end of file
diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp
index 6f31c2c..129234f 100644
--- a/src/DownloadUtils.cpp
+++ b/src/DownloadUtils.cpp
@@ -10,9 +10,11 @@ static int accumulate_string(char *data, int size, void *userdata) {
return 0;
}
+static const char *useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36";
+
namespace QuickMedia {
// TODO: Add timeout
- DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor) {
+ DownloadResult download_to_string(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent) {
sf::Clock timer;
std::vector<const char*> args;
if(use_tor)
@@ -22,6 +24,10 @@ namespace QuickMedia {
args.push_back(arg.option.c_str());
args.push_back(arg.value.c_str());
}
+ if(use_browser_useragent) {
+ args.push_back("-H");
+ args.push_back(useragent_str);
+ }
args.push_back("--");
args.push_back(url.c_str());
args.push_back(nullptr);
@@ -31,7 +37,7 @@ namespace QuickMedia {
return DownloadResult::OK;
}
- DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor) {
+ DownloadResult download_to_string_cache(const std::string &url, std::string &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent) {
Path media_dir = get_cache_dir().join("media");
Path media_file_path = Path(media_dir).join(cppcodec::base64_rfc4648::encode(url));
Path media_finished_path(media_file_path.data + ".finished");
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 644c644..6cd92b7 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -1,5 +1,6 @@
#include "../include/QuickMedia.hpp"
#include "../plugins/Manganelo.hpp"
+#include "../plugins/Mangatown.hpp"
#include "../plugins/Mangadex.hpp"
#include "../plugins/Youtube.hpp"
#include "../plugins/Pornhub.hpp"
@@ -99,7 +100,7 @@ namespace QuickMedia {
static void usage() {
fprintf(stderr, "usage: QuickMedia <plugin> [--tor]\n");
fprintf(stderr, "OPTIONS:\n");
- fprintf(stderr, "plugin The plugin to use. Should be either 4chan, manganelo, mangadex, pornhub, youtube or dmenu\n");
+ fprintf(stderr, "plugin The plugin to use. Should be either 4chan, manganelo, mangatown, mangadex, pornhub, youtube or dmenu\n");
fprintf(stderr, "--tor Use tor. Disabled by default\n");
fprintf(stderr, "EXAMPLES:\n");
fprintf(stderr, "QuickMedia manganelo\n");
@@ -141,6 +142,9 @@ namespace QuickMedia {
if(strcmp(argv[i], "manganelo") == 0) {
current_plugin = new Manganelo();
plugin_logo_path = "../../../images/manganelo_logo.png";
+ } else if(strcmp(argv[i], "mangatown") == 0) {
+ current_plugin = new Mangatown();
+ plugin_logo_path = "../../../images/mangatown_logo.png";
} else if(strcmp(argv[i], "mangadex") == 0) {
current_plugin = new Mangadex();
plugin_logo_path = "../../../images/mangadex_logo.png";
diff --git a/src/plugins/Mangadex.cpp b/src/plugins/Mangadex.cpp
index 3518d4a..b3c5e36 100644
--- a/src/plugins/Mangadex.cpp
+++ b/src/plugins/Mangadex.cpp
@@ -5,7 +5,6 @@
#include <json/reader.h>
static const std::string mangadex_url = "https://mangadex.org";
-static const std::string useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36";
namespace QuickMedia {
static std::string title_url_extract_manga_id(const std::string &url) {
@@ -35,12 +34,10 @@ namespace QuickMedia {
};
SearchResult Mangadex::search(const std::string &url, BodyItems &result_items) {
- CommandArg user_agent_arg = { "-H", useragent_str };
-
std::string manga_id = title_url_extract_manga_id(url);
std::string request_url = "https://mangadex.org/api/?id=" + manga_id + "&type=manga";
std::string server_response;
- if(download_to_string(request_url, server_response, {std::move(user_agent_arg)}, use_tor) != DownloadResult::OK)
+ if(download_to_string(request_url, server_response, {}, use_tor, true) != DownloadResult::OK)
return SearchResult::NET_ERR;
if(server_response.empty())
@@ -170,10 +167,9 @@ namespace QuickMedia {
std::string url = "https://mangadex.org/search?title=";
url += url_param_encode(text);
CommandArg cookie_arg = { "-H", "cookie: mangadex_rememberme_token=" + rememberme_token };
- CommandArg user_agent_arg = { "-H", useragent_str };
std::string website_data;
- if(download_to_string(url, website_data, {std::move(cookie_arg), std::move(user_agent_arg)}, use_tor) != DownloadResult::OK)
+ if(download_to_string(url, website_data, {std::move(cookie_arg)}, use_tor, true) != DownloadResult::OK)
return SuggestionResult::NET_ERR;
QuickMediaHtmlSearch html_search;
@@ -226,10 +222,9 @@ namespace QuickMedia {
}
bool Mangadex::save_mangadex_cookies(const std::string &url, const std::string &cookie_filepath) {
- CommandArg user_agent_arg = { "-H", useragent_str };
CommandArg cookie_arg = { "-c", std::move(cookie_filepath) };
std::string server_response;
- if(download_to_string(url, server_response, {std::move(user_agent_arg), std::move(cookie_arg)}, use_tor) != DownloadResult::OK)
+ if(download_to_string(url, server_response, {std::move(cookie_arg)}, use_tor, true) != DownloadResult::OK)
return false;
return true;
@@ -248,12 +243,11 @@ namespace QuickMedia {
if(!save_mangadex_cookies(url, cookie_filepath))
return ImageResult::ERR;
- CommandArg user_agent_arg = { "-H", useragent_str };
CommandArg cookie_arg = { "-b", std::move(cookie_filepath) };
std::string manga_id = chapter_url_extract_manga_id(url);
std::string request_url = mangadex_url + "/api/?id=" + manga_id + "&server=null&type=chapter";
std::string server_response;
- if(download_to_string(request_url, server_response, {std::move(user_agent_arg), std::move(cookie_arg)}, use_tor) != DownloadResult::OK)
+ if(download_to_string(request_url, server_response, {std::move(cookie_arg)}, use_tor, true) != DownloadResult::OK)
return ImageResult::NET_ERR;
if(server_response.empty())
diff --git a/src/plugins/Mangatown.cpp b/src/plugins/Mangatown.cpp
new file mode 100644
index 0000000..90db349
--- /dev/null
+++ b/src/plugins/Mangatown.cpp
@@ -0,0 +1,207 @@
+#include "../../plugins/Mangatown.hpp"
+#include "../../include/Notification.hpp"
+#include <quickmedia/HtmlSearch.h>
+#include <json/reader.h>
+
+static const std::string mangatown_url = "https://www.mangatown.com";
+
+namespace QuickMedia {
+ SearchResult Mangatown::search(const std::string &url, BodyItems &result_items) {
+ std::string website_data;
+ if(download_to_string(url, website_data, {}, use_tor, true) != DownloadResult::OK)
+ return SearchResult::NET_ERR;
+
+ QuickMediaHtmlSearch html_search;
+ int result = quickmedia_html_search_init(&html_search, website_data.c_str());
+ if(result != 0)
+ goto cleanup;
+
+ result = quickmedia_html_find_nodes_xpath(&html_search, "//ul[class='chapter_list']//a",
+ [](QuickMediaHtmlNode *node, void *userdata) {
+ auto *item_data = (BodyItems*)userdata;
+ const char *href = quickmedia_html_node_get_attribute_value(node, "href");
+ const char *text = quickmedia_html_node_get_text(node);
+ if(href && text && strncmp(href, "/manga/", 7) == 0) {
+ auto item = std::make_unique<BodyItem>(strip(text));
+ item->url = mangatown_url + href;
+ item_data->push_back(std::move(item));
+ }
+ }, &result_items);
+
+ cleanup:
+ quickmedia_html_search_deinit(&html_search);
+ return result == 0 ? SearchResult::OK : SearchResult::ERR;
+ }
+
+ SuggestionResult Mangatown::update_search_suggestions(const std::string &text, BodyItems &result_items) {
+ std::string url = "https://www.mangatown.com/ajax/search/?query=";
+ url += url_param_encode(text);
+
+ std::string server_response;
+ if(download_to_string(url, server_response, {}, use_tor, true) != DownloadResult::OK)
+ return SuggestionResult::NET_ERR;
+
+ if(server_response.empty())
+ return SuggestionResult::OK;
+
+ Json::Value json_root;
+ Json::CharReaderBuilder json_builder;
+ std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader());
+ std::string json_errors;
+ if(!json_reader->parse(&server_response[0], &server_response[server_response.size()], &json_root, &json_errors)) {
+ fprintf(stderr, "Mangatown suggestions json error: %s\n", json_errors.c_str());
+ return SuggestionResult::ERR;
+ }
+
+ if(!json_root.isObject())
+ return SuggestionResult::OK;
+
+ Json::Value &json_data = json_root["data"];
+ Json::Value &json_suggestions = json_root["suggestions"];
+ if(!json_data.isArray() || !json_suggestions.isArray())
+ return SuggestionResult::OK;
+
+ for(const Json::Value &child : json_suggestions) {
+ if(!child.isString()) {
+ result_items.push_back(std::make_unique<BodyItem>(""));
+ continue;
+ }
+ result_items.push_back(std::make_unique<BodyItem>(child.asString()));
+ }
+
+ size_t index = 0;
+ for(const Json::Value &child : json_data) {
+ BodyItem *body_item = nullptr;
+ if(index < result_items.size()) {
+ body_item = result_items[index].get();
+ } else {
+ result_items.push_back(std::make_unique<BodyItem>(""));
+ body_item = result_items.back().get();
+ }
+
+ ++index;
+
+ if(!child.isString())
+ continue;
+
+ body_item->url = mangatown_url + child.asString();
+ }
+
+ return SuggestionResult::OK;
+ }
+
+ static bool is_number_with_zero_fill(const char *str) {
+ while(*str == '0') { ++str; }
+ return atoi(str) != 0;
+ }
+
+ ImageResult Mangatown::get_number_of_images(const std::string &url, int &num_images) {
+ std::lock_guard<std::mutex> lock(image_urls_mutex);
+
+ num_images = last_num_pages;
+ if(url == last_chapter_url_num_images)
+ return ImageResult::OK;
+
+ last_num_pages = 0;
+
+ std::string website_data;
+ if(download_to_string(url, website_data, {}, use_tor, true) != DownloadResult::OK)
+ return ImageResult::NET_ERR;
+
+ QuickMediaHtmlSearch html_search;
+ int result = quickmedia_html_search_init(&html_search, website_data.c_str());
+ if(result != 0)
+ goto cleanup;
+
+ result = quickmedia_html_find_nodes_xpath(&html_search, "//div[class='page_select']//option",
+ [](QuickMediaHtmlNode *node, void *userdata) {
+ int *last_num_pages = (int*)userdata;
+ const char *value = quickmedia_html_node_get_attribute_value(node, "value");
+ const char *text = quickmedia_html_node_get_text(node);
+ if(value && strncmp(value, "/manga/", 7) == 0) {
+ if(is_number_with_zero_fill(text)) {
+ (*last_num_pages)++;
+ }
+ }
+ }, &last_num_pages);
+
+ last_num_pages /= 2;
+ num_images = last_num_pages;
+
+ cleanup:
+ quickmedia_html_search_deinit(&html_search);
+
+ if(result == 0)
+ last_chapter_url_num_images = url;
+ if(last_num_pages == 0) {
+ last_chapter_url_num_images.clear();
+ return ImageResult::ERR;
+ }
+ return result == 0 ? ImageResult::OK : ImageResult::ERR;
+ }
+
+ ImageResult Mangatown::for_each_page_in_chapter(const std::string &chapter_url, PageCallback callback) {
+ int num_pages;
+ ImageResult image_result = get_number_of_images(chapter_url, num_pages);
+ if(image_result != ImageResult::OK)
+ return image_result;
+
+ int result = 0;
+ int page_index = 1;
+
+ while(true) {
+ std::string image_src;
+ std::string website_data;
+ std::string full_url = chapter_url + std::to_string(page_index++) + ".html";
+ if(download_to_string(full_url, website_data, {}, use_tor, true) != DownloadResult::OK)
+ break;
+
+ QuickMediaHtmlSearch html_search;
+ result = quickmedia_html_search_init(&html_search, website_data.c_str());
+ if(result != 0)
+ goto cleanup;
+
+ result = quickmedia_html_find_nodes_xpath(&html_search, "//div[id='viewer']//img",
+ [](QuickMediaHtmlNode *node, void *userdata) {
+ std::string *image_src = (std::string*)userdata;
+ const char *src = quickmedia_html_node_get_attribute_value(node, "src");
+ if(src && strstr(src, "/store/manga/")) {
+ if(strncmp(src, "//", 2) == 0)
+ *image_src = src + 2;
+ else
+ *image_src = src;
+ }
+ }, &image_src);
+
+ cleanup:
+ quickmedia_html_search_deinit(&html_search);
+
+ if(result != 0)
+ return ImageResult::ERR;
+
+ if(image_src.empty())
+ break;
+
+ if(!callback(image_src))
+ break;
+ }
+
+ return ImageResult::OK;
+ }
+
+ bool Mangatown::extract_id_from_url(const std::string &url, std::string &manga_id) {
+ size_t start_index = url.find("/manga/");
+ if(start_index == std::string::npos)
+ return false;
+
+ start_index += 7;
+ size_t end_index = url.find("/", start_index);
+ if(end_index == std::string::npos) {
+ manga_id = url.substr(start_index);
+ return true;
+ }
+
+ manga_id = url.substr(start_index, end_index - start_index);
+ return true;
+ }
+} \ No newline at end of file