diff options
Diffstat (limited to 'src/plugins')
| -rw-r--r-- | src/plugins/Mangadex.cpp | 377 | ||||
| -rw-r--r-- | src/plugins/Manganelo.cpp | 34 |
2 files changed, 411 insertions, 0 deletions
diff --git a/src/plugins/Mangadex.cpp b/src/plugins/Mangadex.cpp new file mode 100644 index 0000000..43368af --- /dev/null +++ b/src/plugins/Mangadex.cpp @@ -0,0 +1,377 @@ +#include "../../plugins/Mangadex.hpp" +#include "../../include/Storage.hpp" +#include <quickmedia/HtmlSearch.h> +#include <json/reader.h> + +static const std::string mangadex_url = "https://mangadex.org"; +// TODO: Allow selecting other languages than english +static const char *language_code = "1"; // english +static const std::string useragent_str = "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; + +namespace QuickMedia { + static std::string title_url_extract_manga_id(const std::string &url) { + size_t find_index = url.find("/title/"); + if(find_index == std::string::npos) + return ""; + + size_t id_start_index = find_index + 7; + size_t end_index = url.find("/", id_start_index); + if(end_index == std::string::npos) + return url.substr(id_start_index); + + return url.substr(id_start_index, end_index - id_start_index); + } + + static std::string chapter_url_extract_manga_id(const std::string &url) { + size_t find_index = url.find("/chapter/"); + if(find_index == std::string::npos) + return ""; + return url.substr(find_index + 9); + } + + struct BodyItemChapterContext { + BodyItems *body_items; + int prev_chapter_number; + bool *is_last_page; + }; + + SearchResult Mangadex::search_page(const std::string &url, BodyItems &result_items, int page, bool *is_last_page) { + *is_last_page = true; + CommandArg user_agent_arg = { "-H", useragent_str }; + + std::string chapter_url = url; + if(chapter_url[0] != '/') + chapter_url += "/"; + chapter_url += "chapters/" + std::to_string(page) + "/"; + std::string website_data; + if(download_to_string(chapter_url, website_data, {std::move(user_agent_arg)}, use_tor) != DownloadResult::OK) + return SearchResult::NET_ERR; + + std::string manga_id = title_url_extract_manga_id(chapter_url); + std::string query = "//div[data-manga-id='" + manga_id + "']"; + + BodyItemChapterContext body_item_chapter_context; + body_item_chapter_context.body_items = &result_items; + body_item_chapter_context.prev_chapter_number = -1; + body_item_chapter_context.is_last_page = is_last_page; + + QuickMediaHtmlSearch html_search; + int result = quickmedia_html_search_init(&html_search, website_data.c_str()); + if(result != 0) + goto cleanup; + + result = quickmedia_html_find_nodes_xpath(&html_search, query.c_str(), + [](QuickMediaHtmlNode *node, void *userdata) { + auto *item_data = (BodyItemChapterContext*)userdata; + const char *data_lang = quickmedia_html_node_get_attribute_value(node, "data-lang"); + if(strcmp(data_lang, language_code) != 0) + return; + + const char *chapter_id = quickmedia_html_node_get_attribute_value(node, "data-id"); + if(!chapter_id) + return; + + const char *chapter_number_str = quickmedia_html_node_get_attribute_value(node, "data-chapter"); + if(!chapter_number_str) + return; + + int chapter_number = atoi(chapter_number_str); + if(chapter_number == 0 || chapter_number == item_data->prev_chapter_number) + return; + + item_data->prev_chapter_number = chapter_number; + + const char *chapter_title = quickmedia_html_node_get_attribute_value(node, "data-title"); + std::string chapter_url = mangadex_url + "/chapter/" + chapter_id; + std::string chapter_name = std::string("Ch. ") + chapter_number_str; + if(chapter_title) + chapter_name += std::string(" - ") + chapter_title; + + auto item = std::make_unique<BodyItem>(std::move(chapter_name)); + item->url = std::move(chapter_url); + item_data->body_items->push_back(std::move(item)); + *item_data->is_last_page = false; + }, &body_item_chapter_context); + + cleanup: + quickmedia_html_search_deinit(&html_search); + return result == 0 ? SearchResult::OK : SearchResult::ERR; + } + + // TODO: Make pagination asynchronous and make it go to the next page when navigating to the bottom in the list of chapters + // in the GUI. Currently all pages are fetched at once, synchronously. This can be very slow for certain manga like Naruto + // which has 21 pages of chapters... + SearchResult Mangadex::search(const std::string &url, BodyItems &result_items) { +#if 0 + int page = 1; + while(true) { + bool is_last_page; + SearchResult search_result = search_page(url, result_items, page, &is_last_page); + if(search_result != SearchResult::OK) + return search_result; + + ++page; + if(is_last_page) + break; + } + return SearchResult::OK; +#else + CommandArg user_agent_arg = { "-H", useragent_str }; + + std::string manga_id = title_url_extract_manga_id(url); + std::string request_url = "https://mangadex.org/api/?id=" + manga_id + "&type=manga"; + std::string server_response; + if(download_to_string(request_url, server_response, {std::move(user_agent_arg)}, use_tor) != DownloadResult::OK) + return SearchResult::NET_ERR; + + if(server_response.empty()) + return SearchResult::OK; + + Json::Value json_root; + Json::CharReaderBuilder json_builder; + std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader()); + std::string json_errors; + if(!json_reader->parse(&server_response[0], &server_response[server_response.size()], &json_root, &json_errors)) { + fprintf(stderr, "Mangadex search json error: %s\n", json_errors.c_str()); + return SearchResult::ERR; + } + + Json::Value &status_json = json_root["status"]; + if(!status_json.isString() || status_json.asString() != "OK") + return SearchResult::ERR; + + Json::Value &chapter_json = json_root["chapter"]; + if(!chapter_json.isObject()) + return SearchResult::ERR; + + std::vector<std::pair<std::string, Json::Value>> chapters(chapter_json.size()); + for(auto &member_name : chapter_json.getMemberNames()) { + Json::Value chapter = chapter_json[member_name]; + if(chapter.isObject()) + chapters.push_back(std::make_pair(member_name, std::move(chapter))); + } + + std::sort(chapters.begin(), chapters.end(), [](std::pair<std::string, Json::Value> &a, std::pair<std::string, Json::Value> &b) { + Json::Value &a_timestamp_json = a.second["timestamp"]; + Json::Value &b_timestamp_json = b.second["timestamp"]; + int64_t a_timestamp = 0; + int64_t b_timestamp = 0; + if(a_timestamp_json.isInt64()) + a_timestamp = a_timestamp_json.asInt64(); + if(b_timestamp_json.isInt64()) + b_timestamp = b_timestamp_json.asInt64(); + return a_timestamp > b_timestamp; + }); + + int prev_chapter_number = -1; + for(auto it = chapters.begin(); it != chapters.end(); ++it) { + const std::string &chapter_id = it->first; + Json::Value &chapter = it->second; + + Json::Value &lang_code = chapter["lang_code"]; + // TODO: Allow selecting other languages than english + if(!lang_code.isString() || lang_code.asString() != "gb") + continue; + + Json::Value &chapter_number_json = chapter["chapter"]; + if(!chapter_number_json.isString()) + continue; + + int chapter_number = atoi(chapter_number_json.asCString()); + if(chapter_number == 0 || chapter_number == prev_chapter_number) + continue; + prev_chapter_number = chapter_number; + + Json::Value &chapter_title_json = chapter["title"]; + std::string chapter_url = mangadex_url + "/chapter/" + chapter_id; + std::string chapter_name = std::string("Ch. ") + chapter_number_json.asCString(); + if(chapter_title_json.isString()) + chapter_name += std::string(" - ") + chapter_title_json.asCString(); + + auto item = std::make_unique<BodyItem>(std::move(chapter_name)); + item->url = std::move(chapter_url); + result_items.push_back(std::move(item)); + } + return SearchResult::OK; +#endif + } + + static bool get_rememberme_token(std::string &rememberme_token) { + Path mangadex_credentials_path = get_storage_dir().join("credentials").join("mangadex.json"); + std::string mangadex_credentials; + if(file_get_content(mangadex_credentials_path, mangadex_credentials) != 0) { + fprintf(stderr, "Failed to get content of file: %s\n", mangadex_credentials_path.data.c_str()); + return false; + } + + Json::Value json_root; + Json::CharReaderBuilder json_builder; + std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader()); + std::string json_errors; + if(!json_reader->parse(&mangadex_credentials[0], &mangadex_credentials[mangadex_credentials.size()], &json_root, &json_errors)) { + fprintf(stderr, "Mangadex credentials json error: %s\n", json_errors.c_str()); + return false; + } + + if(json_root.isObject()) { + Json::Value &rememberme_token_json = json_root["rememberme_token"]; + if(rememberme_token_json.isString()) { + rememberme_token = rememberme_token_json.asString(); + return true; + } + } + return true; + } + + struct BodyItemImageContext { + BodyItems *body_items; + size_t index; + }; + + // TODO: Implement pagination (go to next page and get all results as well) + SuggestionResult Mangadex::update_search_suggestions(const std::string &text, BodyItems &result_items) { + std::string rememberme_token; + if(!get_rememberme_token(rememberme_token)) + return SuggestionResult::ERR; + + std::string url = "https://mangadex.org/search?title="; + url += url_param_encode(text); + CommandArg cookie_arg = { "-H", "cookie: mangadex_rememberme_token=" + rememberme_token }; + CommandArg user_agent_arg = { "-H", useragent_str }; + + std::string website_data; + if(download_to_string(url, website_data, {std::move(cookie_arg), std::move(user_agent_arg)}, use_tor) != DownloadResult::OK) + return SuggestionResult::NET_ERR; + + QuickMediaHtmlSearch html_search; + int result = quickmedia_html_search_init(&html_search, website_data.c_str()); + if(result != 0) + goto cleanup; + + result = quickmedia_html_find_nodes_xpath(&html_search, "//a", + [](QuickMediaHtmlNode *node, void *userdata) { + auto *item_data = (BodyItems*)userdata; + const char *href = quickmedia_html_node_get_attribute_value(node, "href"); + const char *title = quickmedia_html_node_get_attribute_value(node, "title"); + if(title && href && strncmp(href, "/title/", 7) == 0) { + auto item = std::make_unique<BodyItem>(strip(title)); + item->url = mangadex_url + href; + item_data->push_back(std::move(item)); + } + }, &result_items); + + BodyItemImageContext body_item_image_context; + body_item_image_context.body_items = &result_items; + body_item_image_context.index = 0; + + result = quickmedia_html_find_nodes_xpath(&html_search, "//img", + [](QuickMediaHtmlNode *node, void *userdata) { + auto *item_data = (BodyItemImageContext*)userdata; + const char *src = quickmedia_html_node_get_attribute_value(node, "src"); + if(src && strncmp(src, "/images/manga/", 14) == 0) { + if(item_data->index < item_data->body_items->size()) { + (*item_data->body_items)[item_data->index]->thumbnail_url = mangadex_url + src; + item_data->index++; + } + } + }, &body_item_image_context); + + cleanup: + quickmedia_html_search_deinit(&html_search); + return result == 0 ? SuggestionResult::OK : SuggestionResult::ERR; + } + + ImageResult Mangadex::get_number_of_images(const std::string &url, int &num_images) { + std::lock_guard<std::mutex> lock(image_urls_mutex); + num_images = 0; + ImageResult image_result = get_image_urls_for_chapter(url); + if(image_result != ImageResult::OK) + return image_result; + + num_images = last_chapter_image_urls.size(); + return ImageResult::OK; + } + + ImageResult Mangadex::get_image_urls_for_chapter(const std::string &url) { + if(url == last_chapter_url) + return ImageResult::OK; + + last_chapter_image_urls.clear(); + + CommandArg user_agent_arg = { "-H", useragent_str }; + std::string manga_id = chapter_url_extract_manga_id(url); + std::string request_url = mangadex_url + "/api/?id=" + manga_id + "&server=null&type=chapter"; + std::string server_response; + if(download_to_string(request_url, server_response, {std::move(user_agent_arg)}, use_tor) != DownloadResult::OK) + return ImageResult::NET_ERR; + + if(server_response.empty()) + return ImageResult::OK; + + Json::Value json_root; + Json::CharReaderBuilder json_builder; + std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader()); + std::string json_errors; + if(!json_reader->parse(&server_response[0], &server_response[server_response.size()], &json_root, &json_errors)) { + fprintf(stderr, "Mangadex image urls json error: %s\n", json_errors.c_str()); + return ImageResult::ERR; + } + + Json::Value &status_json = json_root["status"]; + if(!status_json.isString() || status_json.asString() != "OK") + return ImageResult::ERR; + + Json::Value &chapter_hash = json_root["hash"]; + if(!chapter_hash.isString()) + return ImageResult::ERR; + const char *chapter_hash_str = chapter_hash.asCString(); + + Json::Value &server_json = json_root["server"]; + std::string server; + if(server_json.isString()) + server = server_json.asString(); + else + server = mangadex_url + "/data/"; + + Json::Value &page_array = json_root["page_array"]; + if(page_array.isArray()) { + for(const Json::Value &image_name : page_array) { + if(!image_name.isString()) + continue; + + std::string image_url = server + chapter_hash_str + "/" + image_name.asCString(); + last_chapter_image_urls.push_back(std::move(image_url)); + } + } + + last_chapter_url = url; + if(last_chapter_image_urls.empty()) { + last_chapter_url.clear(); + return ImageResult::ERR; + } + return ImageResult::OK; + } + + ImageResult Mangadex::for_each_page_in_chapter(const std::string &chapter_url, PageCallback callback) { + std::vector<std::string> image_urls; + { + std::lock_guard<std::mutex> lock(image_urls_mutex); + ImageResult image_result = get_image_urls_for_chapter(chapter_url); + if(image_result != ImageResult::OK) + return image_result; + + image_urls = last_chapter_image_urls; + } + + for(const std::string &url : image_urls) { + if(!callback(url)) + break; + } + return ImageResult::OK; + } + + bool Mangadex::extract_id_from_url(const std::string &url, std::string &manga_id) { + manga_id = title_url_extract_manga_id(url); + return true; + } +} diff --git a/src/plugins/Manganelo.cpp b/src/plugins/Manganelo.cpp index a97f75a..89f882c 100644 --- a/src/plugins/Manganelo.cpp +++ b/src/plugins/Manganelo.cpp @@ -1,4 +1,5 @@ #include "../../plugins/Manganelo.hpp" +#include "../../include/Notification.hpp" #include <quickmedia/HtmlSearch.h> #include <json/reader.h> @@ -159,4 +160,37 @@ namespace QuickMedia { } return ImageResult::OK; } + + bool Manganelo::extract_id_from_url(const std::string &url, std::string &manga_id) { + bool manganelo_website = false; + if(url.find("mangakakalot") != std::string::npos || url.find("manganelo") != std::string::npos) + manganelo_website = true; + + if(manganelo_website) { + size_t index = url.find("manga/"); + if(index == std::string::npos) { + std::string err_msg = "Url "; + err_msg += url; + err_msg += " doesn't contain manga id"; + show_notification("Manga", err_msg, Urgency::CRITICAL); + return false; + } + + manga_id = url.substr(index + 6); + if(manga_id.size() <= 2) { + std::string err_msg = "Url "; + err_msg += url; + err_msg += " doesn't contain manga id"; + show_notification("Manga", err_msg, Urgency::CRITICAL); + return false; + } + return true; + } else { + std::string err_msg = "Unexpected url "; + err_msg += url; + err_msg += " is not manganelo or mangakakalot"; + show_notification("Manga", err_msg, Urgency::CRITICAL); + return false; + } + } }
\ No newline at end of file |