From ecb76941fe38cb8016388f6b2185312432cc8122 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Thu, 6 May 2021 18:46:28 +0200 Subject: Upgrade mangadex api to the new one There is a bug in mangadex api where order[publishAt]=desc is ignored, so going to the next chapter is messed up. Automedia will be updated after this is fixed. Also manga cover images are missing. --- src/plugins/Mangadex.cpp | 438 ++++++++++++++++++++++------------------------- 1 file changed, 203 insertions(+), 235 deletions(-) (limited to 'src/plugins/Mangadex.cpp') diff --git a/src/plugins/Mangadex.cpp b/src/plugins/Mangadex.cpp index 0d0c601..7332e21 100644 --- a/src/plugins/Mangadex.cpp +++ b/src/plugins/Mangadex.cpp @@ -1,303 +1,271 @@ #include "../../plugins/Mangadex.hpp" -#include "../../include/Storage.hpp" -#include "../../include/Notification.hpp" -#include "../../include/StringUtils.hpp" -#include "../../include/NetUtils.hpp" -#include -#include -#include -#include - -static const std::string mangadex_url = "https://mangadex.org"; - -static rapidjson::Value nullValue(rapidjson::kNullType); -static const rapidjson::Value& GetMember(const rapidjson::Value &obj, const char *key) { - auto it = obj.FindMember(key); - if(it != obj.MemberEnd()) - return it->value; - return nullValue; -} +#include namespace QuickMedia { - static std::string title_url_extract_manga_id(const std::string &url) { - size_t find_index = url.find("/title/"); - if(find_index == std::string::npos) - return ""; - - size_t id_start_index = find_index + 7; - size_t end_index = url.find("/", id_start_index); - if(end_index == std::string::npos) - return url.substr(id_start_index); - - return url.substr(id_start_index, end_index - id_start_index); - } + PluginResult legacy_mangadex_id_to_new_manga_id(Page *page, const std::vector &manga_ids, std::vector> &new_manga_ids) { + Json::Value request_json(Json::objectValue); + request_json["type"] = "manga"; + Json::Value manga_ids_json(Json::arrayValue); + for(int manga_id : manga_ids) { + manga_ids_json.append(manga_id); + } + request_json["ids"] = std::move(manga_ids_json); - static std::string chapter_url_extract_manga_id(const std::string &url) { - size_t find_index = url.find("/chapter/"); - if(find_index == std::string::npos) - return ""; - return url.substr(find_index + 9); - } + Json::StreamWriterBuilder json_builder; + json_builder["commentStyle"] = "None"; + json_builder["indentation"] = ""; + + std::vector additional_args = { + { "-X", "POST" }, + { "-H", "content-type: application/json" }, + { "--data-binary", Json::writeString(json_builder, request_json) } + }; + + Json::Value json_root; + if(page->download_json(json_root, "https://api.mangadex.org/legacy/mapping", std::move(additional_args), true) != DownloadResult::OK) + return PluginResult::NET_ERR; + + if(!json_root.isArray()) + return PluginResult::OK; + + for(const Json::Value &result_item_json : json_root) { + if(!result_item_json.isObject()) + continue; + + const Json::Value &result_json = result_item_json["result"]; + if(!result_json.isString() || strcmp(result_json.asCString(), "ok") != 0) + continue; - static bool get_cookie_filepath(std::string &cookie_filepath) { - Path cookie_path = get_storage_dir().join("cookies"); - if(create_directory_recursive(cookie_path) != 0) { - show_notification("QuickMedia", "Failed to create directory: " + cookie_path.data, Urgency::CRITICAL); - return false; + const Json::Value &data_json = result_item_json["data"]; + if(!data_json.isObject()) + continue; + + const Json::Value &attributes_json = data_json["attributes"]; + if(!attributes_json.isObject()) + continue; + + const Json::Value &legacy_id_json = attributes_json["legacyId"]; + if(!legacy_id_json.isInt()) + continue; + + const Json::Value &new_id_json = attributes_json["newId"]; + if(!new_id_json.isString()) + continue; + + new_manga_ids.push_back(std::make_pair(legacy_id_json.asInt(), new_id_json.asString())); } - cookie_filepath = cookie_path.join("mangadex.txt").data; - return true; + + return PluginResult::OK; } - struct BodyItemChapterContext { - BodyItems *body_items; - int prev_chapter_number; - bool *is_last_page; + class MangadexChapterImagesList : public BodyItemExtra { + public: + std::vector image_urls; }; SearchResult MangadexSearchPage::search(const std::string &str, int page, BodyItems &result_items) { - std::string rememberme_token; - if(!get_rememberme_token(rememberme_token)) - return SearchResult::ERR; - - std::string url = "https://mangadex.org/search?s=0&p=" + std::to_string(page) + "&tag_mode_inc=all&tag_mode_exc=any&title=" + url_param_encode(str) +"#listing"; - CommandArg cookie_arg = { "-H", "cookie: mangadex_rememberme_token=" + rememberme_token }; + std::string url = "https://api.mangadex.org/manga?title=" + url_param_encode(str) + "&limit=100&offset=" + std::to_string(page * 100); - std::string website_data; - if(download_to_string(url, website_data, {std::move(cookie_arg)}, true) != DownloadResult::OK) + Json::Value json_root; + if(download_json(json_root, url, {}, true) != DownloadResult::OK) return SearchResult::NET_ERR; - QuickMediaHtmlSearch html_search; - int result = quickmedia_html_search_init(&html_search, website_data.c_str()); - if(result != 0) - goto cleanup; - - result = quickmedia_html_find_nodes_xpath(&html_search, "//a", - [](QuickMediaHtmlNode *node, void *userdata) { - auto *item_data = (BodyItems*)userdata; - const char *href = quickmedia_html_node_get_attribute_value(node, "href"); - const char *title = quickmedia_html_node_get_attribute_value(node, "title"); - if(title && href && strncmp(href, "/title/", 7) == 0) { - auto item = BodyItem::create(strip(title)); - item->url = mangadex_url + href; - item_data->push_back(std::move(item)); - } - }, &result_items); - - if(result != 0) - goto cleanup; - - BodyItemContext body_item_image_context; - body_item_image_context.body_items = &result_items; - body_item_image_context.index = 0; - - result = quickmedia_html_find_nodes_xpath(&html_search, "//img", - [](QuickMediaHtmlNode *node, void *userdata) { - auto *item_data = (BodyItemContext*)userdata; - const char *src = quickmedia_html_node_get_attribute_value(node, "src"); - if(src && strncmp(src, "/images/manga/", 14) == 0 && item_data->index < item_data->body_items->size()) { - (*item_data->body_items)[item_data->index]->thumbnail_url = mangadex_url + src; - item_data->index++; - } - }, &body_item_image_context); - - if(result != 0) - goto cleanup; - - cleanup: - quickmedia_html_search_deinit(&html_search); - return result == 0 ? SearchResult::OK : SearchResult::ERR; + if(!json_root.isObject()) + return SearchResult::OK; + + const Json::Value &results_json = json_root["results"]; + if(!results_json.isArray()) + return SearchResult::OK; + + for(const Json::Value &result_item_json : results_json) { + if(!result_item_json.isObject()) + continue; + + const Json::Value &result_json = result_item_json["result"]; + if(!result_json.isString() || strcmp(result_json.asCString(), "ok") != 0) + continue; + + const Json::Value &data_json = result_item_json["data"]; + if(!data_json.isObject()) + continue; + + const Json::Value &id_json = data_json["id"]; + if(!id_json.isString()) + continue; + + const Json::Value &attributes_json = data_json["attributes"]; + if(!attributes_json.isObject()) + continue; + + const Json::Value &description_json = attributes_json["description"]; + + const Json::Value &title_json = attributes_json["title"]; + if(!title_json.isObject()) + continue; + + std::string title; + const Json::Value &title_en_json = title_json["en"]; + if(title_en_json.isString()) + title = title_en_json.asString(); + else + title = "No title"; // TODO: Verify if this happens. If it happens, get the title in any available language + + auto body_item = BodyItem::create(std::move(title)); + body_item->url = id_json.asString(); + if(description_json.isString()) { + body_item->set_description(description_json.asString()); + body_item->set_description_color(sf::Color(179, 179, 179)); + } + result_items.push_back(std::move(body_item)); + } + + return SearchResult::OK; } SearchResult MangadexSearchPage::search(const std::string &str, BodyItems &result_items) { - return search(str, 1, result_items); + return search(str, 0, result_items); } PluginResult MangadexSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) { - return search_result_to_plugin_result(search(str, 1 + page, result_items)); + return search_result_to_plugin_result(search(str, page, result_items)); } - PluginResult MangadexSearchPage::submit(const std::string &title, const std::string &url, std::vector &result_tabs) { - std::string request_url = "https://mangadex.org/api/?id=" + title_url_extract_manga_id(url) + "&type=manga"; + static PluginResult get_chapters_for_manga(Page *page, const std::string &manga_id, int page_num, BodyItems &result_items) { + std::string request_url = "https://api.mangadex.org/chapter?manga=" + manga_id + "&limit=100&offset=" + std::to_string(page_num * 100) + "&order[publishAt]=desc"; - rapidjson::Document json_root; - DownloadResult result = download_to_json(request_url, json_root, {}, true); - if(result != DownloadResult::OK) return download_result_to_plugin_result(result); + Json::Value json_root; + if(page->download_json(json_root, request_url, {}, true) != DownloadResult::OK) + return PluginResult::NET_ERR; - if(!json_root.IsObject()) - return PluginResult::ERR; + if(!json_root.isObject()) + return PluginResult::OK; - const rapidjson::Value &status_json = GetMember(json_root, "status"); - if(!status_json.IsString() || strcmp(status_json.GetString(), "OK") != 0) - return PluginResult::ERR; + const Json::Value &results_json = json_root["results"]; + if(!results_json.isArray()) + return PluginResult::OK; - const rapidjson::Value &chapter_json = GetMember(json_root, "chapter"); - if(!chapter_json.IsObject()) - return PluginResult::ERR; + for(const Json::Value &result_item_json : results_json) { + if(!result_item_json.isObject()) + continue; - time_t time_now = time(NULL); - auto body = create_body(); + const Json::Value &result_json = result_item_json["result"]; + if(!result_json.isString() || strcmp(result_json.asCString(), "ok") != 0) + continue; - /* TODO: Pointer */ - std::string prev_chapter_number; - for(auto const &it : chapter_json.GetObject()) { - const std::string &chapter_id = it.name.GetString(); - const rapidjson::Value &chapter = it.value; + const Json::Value &data_json = result_item_json["data"]; + if(!data_json.isObject()) + continue; + + const Json::Value &id_json = data_json["id"]; + if(!id_json.isString()) + continue; - const rapidjson::Value ×tamp_json = chapter["timestamp"]; - if(timestamp_json.IsInt64() && timestamp_json.GetInt64() > time_now) + const Json::Value &attributes_json = data_json["attributes"]; + if(!attributes_json.isObject()) continue; - const rapidjson::Value &lang_code = chapter["lang_code"]; - // TODO: Allow selecting other languages than english - if(!lang_code.IsString() || strcmp(lang_code.GetString(), "gb") != 0) + const Json::Value &translated_language_json = attributes_json["translatedLanguage"]; + if(!translated_language_json.isString() || strcmp(translated_language_json.asCString(), "en") != 0) continue; - const rapidjson::Value &chapter_number_json = chapter["chapter"]; - if(!chapter_number_json.IsString()) + const Json::Value &chapter_json = attributes_json["chapter"]; + if(!chapter_json.isString()) continue; - std::string chapter_number_str = chapter_number_json.GetString(); - if(chapter_number_str == prev_chapter_number) + std::string title = "Ch. " + chapter_json.asString(); + const Json::Value &title_json = attributes_json["title"]; + if(title_json.isString() && title_json.asCString()[0] != '\0') + title += " - " + title_json.asString(); + + const Json::Value &hash_json = attributes_json["hash"]; + if(!hash_json.isString()) continue; - prev_chapter_number = chapter_number_str; - const rapidjson::Value &chapter_title_json = chapter["title"]; - std::string chapter_url = mangadex_url + "/chapter/" + chapter_id; - std::string chapter_name = std::string("Ch. ") + chapter_number_str; - if(chapter_title_json.IsString() && chapter_title_json.GetStringLength() > 0) - chapter_name += std::string(" - ") + chapter_title_json.GetString(); + const Json::Value &attributes_data_json = attributes_json["data"]; + if(!attributes_data_json.isArray()) + continue; + + auto body_item = BodyItem::create(std::move(title)); + body_item->url = id_json.asString(); + auto images_list = std::make_shared(); + for(const Json::Value &data_item_json : attributes_data_json) { + if(!data_item_json.isString()) + continue; - auto item = BodyItem::create(std::move(chapter_name)); - item->url = std::move(chapter_url); - body->items.push_back(std::move(item)); + std::string url = hash_json.asString() + "/" + data_item_json.asString(); + images_list->image_urls.push_back(std::move(url)); + } + body_item->extra = std::move(images_list); + result_items.push_back(std::move(body_item)); } + return PluginResult::OK; + } + + PluginResult MangadexSearchPage::submit(const std::string &title, const std::string &url, std::vector &result_tabs) { + auto body = create_body(); + get_chapters_for_manga(this, url, 0, body->items); result_tabs.push_back(Tab{std::move(body), std::make_unique(program, title, url), create_search_bar("Search...", SEARCH_DELAY_FILTER)}); return PluginResult::OK; } - bool MangadexSearchPage::get_rememberme_token(std::string &rememberme_token_output) { - if(rememberme_token) { - rememberme_token_output = rememberme_token.value(); - return true; - } + PluginResult MangadexChaptersPage::submit(const std::string &title, const std::string &url, std::vector &result_tabs) { + Json::Value json_root; + if(download_json(json_root, "https://api.mangadex.org/at-home/server/" + url, {}, true) != DownloadResult::OK) + return PluginResult::NET_ERR; - Path mangadex_credentials_path = get_storage_dir().join("credentials").join("mangadex.json"); - std::string mangadex_credentials; - if(file_get_content(mangadex_credentials_path, mangadex_credentials) != 0) { - fprintf(stderr, "Failed to get content of file: %s\n", mangadex_credentials_path.data.c_str()); - return false; - } + if(!json_root.isObject()) + return PluginResult::ERR; - Json::Value json_root; - Json::CharReaderBuilder json_builder; - std::unique_ptr json_reader(json_builder.newCharReader()); - std::string json_errors; - if(!json_reader->parse(&mangadex_credentials[0], &mangadex_credentials[mangadex_credentials.size()], &json_root, &json_errors)) { - fprintf(stderr, "Mangadex credentials json error: %s\n", json_errors.c_str()); - return false; - } + const Json::Value &base_url_json = json_root["baseUrl"]; + if(!base_url_json.isString()) + return PluginResult::ERR; + + std::string base_url = base_url_json.asString(); + if(base_url.back() != '/') + base_url += '/'; - if(json_root.isObject()) { - Json::Value &rememberme_token_json = json_root["rememberme_token"]; - if(rememberme_token_json.isString()) - rememberme_token_output = rememberme_token_json.asString(); + auto image_urls = static_cast(submit_body_item->extra.get())->image_urls; + for(auto &image_url : image_urls) { + image_url = base_url + "data/" + image_url; } - rememberme_token = rememberme_token_output; - return true; + result_tabs.push_back(Tab{nullptr, std::make_unique(program, content_title, title, content_url, image_urls), nullptr}); + return PluginResult::OK; } - PluginResult MangadexChaptersPage::submit(const std::string &title, const std::string &url, std::vector &result_tabs) { - result_tabs.push_back(Tab{nullptr, std::make_unique(program, content_title, title, url), nullptr}); - return PluginResult::OK; + PluginResult MangadexChaptersPage::get_page(const std::string&, int page, BodyItems &result_items) { + return get_chapters_for_manga(this, content_url, page, result_items); } bool MangadexChaptersPage::extract_id_from_url(const std::string &url, std::string &manga_id) const { - manga_id = title_url_extract_manga_id(url); + size_t start_index = url.find("manga="); + if(start_index == std::string::npos) { + manga_id = url; + return true; + } + + start_index += 6; + size_t end_index = url.find('&', start_index); + if(end_index == std::string::npos) { + manga_id = url.substr(start_index); + return true; + } + + manga_id = url.substr(start_index, end_index - start_index); return true; } ImageResult MangadexImagesPage::get_number_of_images(int &num_images) { - num_images = 0; - ImageResult image_result = get_image_urls_for_chapter(url); - if(image_result != ImageResult::OK) return image_result; - num_images = chapter_image_urls.size(); + num_images = image_urls.size(); return ImageResult::OK; } ImageResult MangadexImagesPage::for_each_page_in_chapter(PageCallback callback) { - ImageResult image_result = get_image_urls_for_chapter(url); - if(image_result != ImageResult::OK) return image_result; - for(const std::string &url : chapter_image_urls) { + for(const std::string &url : image_urls) { if(!callback(url)) break; } return ImageResult::OK; } - - ImageResult MangadexImagesPage::get_image_urls_for_chapter(const std::string &url) { - if(!chapter_image_urls.empty()) - return ImageResult::OK; - - std::string cookie_filepath; - if(!get_cookie_filepath(cookie_filepath)) - return ImageResult::ERR; - - if(!save_mangadex_cookies(url, cookie_filepath)) - return ImageResult::ERR; - - CommandArg cookie_arg = { "-b", std::move(cookie_filepath) }; - std::string manga_id = chapter_url_extract_manga_id(url); - std::string request_url = mangadex_url + "/api/?id=" + manga_id + "&server=null&type=chapter"; - - Json::Value json_root; - DownloadResult result = download_json(json_root, request_url, {std::move(cookie_arg)}, true); - if(result != DownloadResult::OK) return download_result_to_image_result(result); - - if(!json_root.isObject()) - return ImageResult::ERR; - - Json::Value &status_json = json_root["status"]; - if(!status_json.isString() || status_json.asString() != "OK") - return ImageResult::ERR; - - Json::Value &chapter_hash = json_root["hash"]; - if(!chapter_hash.isString()) - return ImageResult::ERR; - const char *chapter_hash_str = chapter_hash.asCString(); - - Json::Value &server_json = json_root["server"]; - std::string server; - if(server_json.isString()) - server = server_json.asString(); - else - server = mangadex_url + "/data/"; - - Json::Value &page_array = json_root["page_array"]; - if(page_array.isArray()) { - for(const Json::Value &image_name : page_array) { - if(!image_name.isString()) - continue; - - std::string image_url = server + chapter_hash_str + "/" + image_name.asCString(); - chapter_image_urls.push_back(std::move(image_url)); - } - } - - if(chapter_image_urls.empty()) - return ImageResult::ERR; - - return ImageResult::OK; - } - - bool MangadexImagesPage::save_mangadex_cookies(const std::string &url, const std::string &cookie_filepath) { - CommandArg cookie_arg = { "-c", cookie_filepath }; - std::string server_response; - if(download_to_string(url, server_response, {std::move(cookie_arg)}, true) != DownloadResult::OK) - return false; - - return true; - } } -- cgit v1.2.3