Upgrade mangadex api to the new one

There is a bug in mangadex api where order[publishAt]=desc is ignored,
so going to the next chapter is messed up. Automedia will be updated
after this is fixed. Also manga cover images are missing.

1 files changed, 203 insertions, 235 deletions

diff --git a/src/plugins/Mangadex.cpp b/src/plugins/Mangadex.cpp
index 0d0c601..7332e21 100644
--- a/src/plugins/Mangadex.cpp
+++ b/src/plugins/Mangadex.cpp
@@ -1,303 +1,271 @@
 #include "../../plugins/Mangadex.hpp"
-#include "../../include/Storage.hpp"
-#include "../../include/Notification.hpp"
-#include "../../include/StringUtils.hpp"
-#include "../../include/NetUtils.hpp"
-#include <rapidjson/document.h>
-#include <rapidjson/reader.h>
-#include <quickmedia/HtmlSearch.h>
-#include <json/reader.h>
-
-static const std::string mangadex_url = "https://mangadex.org";
-
-static rapidjson::Value nullValue(rapidjson::kNullType);
-static const rapidjson::Value& GetMember(const rapidjson::Value &obj, const char *key) {
-    auto it = obj.FindMember(key);
-    if(it != obj.MemberEnd())
-        return it->value;
-    return nullValue;
-}
+#include <json/writer.h>
 
 namespace QuickMedia {
-    static std::string title_url_extract_manga_id(const std::string &url) {
-        size_t find_index = url.find("/title/");
-        if(find_index == std::string::npos)
-            return "";
-        
-        size_t id_start_index = find_index + 7;
-        size_t end_index = url.find("/", id_start_index);
-        if(end_index == std::string::npos)
-            return url.substr(id_start_index);
-        
-        return url.substr(id_start_index, end_index - id_start_index);
-    }
+    PluginResult legacy_mangadex_id_to_new_manga_id(Page *page, const std::vector<int> &manga_ids, std::vector<std::pair<int, std::string>> &new_manga_ids) {
+        Json::Value request_json(Json::objectValue);
+        request_json["type"] = "manga";
+        Json::Value manga_ids_json(Json::arrayValue);
+        for(int manga_id : manga_ids) {
+            manga_ids_json.append(manga_id);
+        }
+        request_json["ids"] = std::move(manga_ids_json);
 
-    static std::string chapter_url_extract_manga_id(const std::string &url) {
-        size_t find_index = url.find("/chapter/");
-        if(find_index == std::string::npos)
-            return "";
-        return url.substr(find_index + 9);
-    }
+        Json::StreamWriterBuilder json_builder;
+        json_builder["commentStyle"] = "None";
+        json_builder["indentation"] = "";
+
+        std::vector<CommandArg> additional_args = {
+            { "-X", "POST" },
+            { "-H", "content-type: application/json" },
+            { "--data-binary", Json::writeString(json_builder, request_json) }
+        };
+
+        Json::Value json_root;
+        if(page->download_json(json_root, "https://api.mangadex.org/legacy/mapping", std::move(additional_args), true) != DownloadResult::OK)
+            return PluginResult::NET_ERR;
+
+        if(!json_root.isArray())
+            return PluginResult::OK;
+
+        for(const Json::Value &result_item_json : json_root) {
+            if(!result_item_json.isObject())
+                continue;
+
+            const Json::Value &result_json = result_item_json["result"];
+            if(!result_json.isString() || strcmp(result_json.asCString(), "ok") != 0)
+                continue;
 
-    static bool get_cookie_filepath(std::string &cookie_filepath) {
-        Path cookie_path = get_storage_dir().join("cookies");
-        if(create_directory_recursive(cookie_path) != 0) {
-            show_notification("QuickMedia", "Failed to create directory: " + cookie_path.data, Urgency::CRITICAL);
-            return false;
+            const Json::Value &data_json = result_item_json["data"];
+            if(!data_json.isObject())
+                continue;
+
+            const Json::Value &attributes_json = data_json["attributes"];
+            if(!attributes_json.isObject())
+                continue;
+
+            const Json::Value &legacy_id_json = attributes_json["legacyId"];
+            if(!legacy_id_json.isInt())
+                continue;
+
+            const Json::Value &new_id_json = attributes_json["newId"];
+            if(!new_id_json.isString())
+                continue;
+
+            new_manga_ids.push_back(std::make_pair<int, std::string>(legacy_id_json.asInt(), new_id_json.asString()));
         }
-        cookie_filepath = cookie_path.join("mangadex.txt").data;
-        return true;
+
+        return PluginResult::OK;
     }
 
-    struct BodyItemChapterContext {
-        BodyItems *body_items;
-        int prev_chapter_number;
-        bool *is_last_page;
+    class MangadexChapterImagesList : public BodyItemExtra {
+    public:
+        std::vector<std::string> image_urls;
     };
 
     SearchResult MangadexSearchPage::search(const std::string &str, int page, BodyItems &result_items) {
-        std::string rememberme_token;
-        if(!get_rememberme_token(rememberme_token))
-            return SearchResult::ERR;
-
-        std::string url = "https://mangadex.org/search?s=0&p=" + std::to_string(page) + "&tag_mode_inc=all&tag_mode_exc=any&title=" + url_param_encode(str) +"#listing";
-        CommandArg cookie_arg = { "-H", "cookie: mangadex_rememberme_token=" + rememberme_token };
+        std::string url = "https://api.mangadex.org/manga?title=" + url_param_encode(str) + "&limit=100&offset=" + std::to_string(page * 100);
 
-        std::string website_data;
-        if(download_to_string(url, website_data, {std::move(cookie_arg)}, true) != DownloadResult::OK)
+        Json::Value json_root;
+        if(download_json(json_root, url, {}, true) != DownloadResult::OK)
             return SearchResult::NET_ERR;
 
-        QuickMediaHtmlSearch html_search;
-        int result = quickmedia_html_search_init(&html_search, website_data.c_str());
-        if(result != 0)
-            goto cleanup;
-
-        result = quickmedia_html_find_nodes_xpath(&html_search, "//a",
-            [](QuickMediaHtmlNode *node, void *userdata) {
-                auto *item_data = (BodyItems*)userdata;
-                const char *href = quickmedia_html_node_get_attribute_value(node, "href");
-                const char *title = quickmedia_html_node_get_attribute_value(node, "title");
-                if(title && href && strncmp(href, "/title/", 7) == 0) {
-                    auto item = BodyItem::create(strip(title));
-                    item->url = mangadex_url + href;
-                    item_data->push_back(std::move(item));
-                }
-            }, &result_items);
-
-        if(result != 0)
-            goto cleanup;
-
-        BodyItemContext body_item_image_context;
-        body_item_image_context.body_items = &result_items;
-        body_item_image_context.index = 0;
-
-        result = quickmedia_html_find_nodes_xpath(&html_search, "//img",
-            [](QuickMediaHtmlNode *node, void *userdata) {
-                auto *item_data = (BodyItemContext*)userdata;
-                const char *src = quickmedia_html_node_get_attribute_value(node, "src");
-                if(src && strncmp(src, "/images/manga/", 14) == 0 && item_data->index < item_data->body_items->size()) {
-                    (*item_data->body_items)[item_data->index]->thumbnail_url = mangadex_url + src;
-                    item_data->index++;
-                }
-            }, &body_item_image_context);
-
-        if(result != 0)
-            goto cleanup;
-
-        cleanup:
-        quickmedia_html_search_deinit(&html_search);
-        return result == 0 ? SearchResult::OK : SearchResult::ERR;
+        if(!json_root.isObject())
+            return SearchResult::OK;
+
+        const Json::Value &results_json = json_root["results"];
+        if(!results_json.isArray())
+            return SearchResult::OK;
+
+        for(const Json::Value &result_item_json : results_json) {
+            if(!result_item_json.isObject())
+                continue;
+
+            const Json::Value &result_json = result_item_json["result"];
+            if(!result_json.isString() || strcmp(result_json.asCString(), "ok") != 0)
+                continue;
+
+            const Json::Value &data_json = result_item_json["data"];
+            if(!data_json.isObject())
+                continue;
+            
+            const Json::Value &id_json = data_json["id"];
+            if(!id_json.isString())
+                continue;
+
+            const Json::Value &attributes_json = data_json["attributes"];
+            if(!attributes_json.isObject())
+                continue;
+
+            const Json::Value &description_json = attributes_json["description"];
+
+            const Json::Value &title_json = attributes_json["title"];
+            if(!title_json.isObject())
+                continue;
+
+            std::string title;
+            const Json::Value &title_en_json = title_json["en"];
+            if(title_en_json.isString())
+                title = title_en_json.asString();
+            else
+                title = "No title"; // TODO: Verify if this happens. If it happens, get the title in any available language
+
+            auto body_item = BodyItem::create(std::move(title));
+            body_item->url = id_json.asString();
+            if(description_json.isString()) {
+                body_item->set_description(description_json.asString());
+                body_item->set_description_color(sf::Color(179, 179, 179));
+            }
+            result_items.push_back(std::move(body_item));
+        }
+
+        return SearchResult::OK;
     }
 
     SearchResult MangadexSearchPage::search(const std::string &str, BodyItems &result_items) {
-        return search(str, 1, result_items);
+        return search(str, 0, result_items);
     }
 
     PluginResult MangadexSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) {
-        return search_result_to_plugin_result(search(str, 1 + page, result_items));
+        return search_result_to_plugin_result(search(str, page, result_items));
     }
 
-    PluginResult MangadexSearchPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
-        std::string request_url = "https://mangadex.org/api/?id=" + title_url_extract_manga_id(url) + "&type=manga";
+    static PluginResult get_chapters_for_manga(Page *page, const std::string &manga_id, int page_num, BodyItems &result_items) {
+        std::string request_url = "https://api.mangadex.org/chapter?manga=" + manga_id + "&limit=100&offset=" + std::to_string(page_num * 100) + "&order[publishAt]=desc";
 
-        rapidjson::Document json_root;
-        DownloadResult result = download_to_json(request_url, json_root, {}, true);
-        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
+        Json::Value json_root;
+        if(page->download_json(json_root, request_url, {}, true) != DownloadResult::OK)
+            return PluginResult::NET_ERR;
 
-        if(!json_root.IsObject())
-            return PluginResult::ERR;
+        if(!json_root.isObject())
+            return PluginResult::OK;
 
-        const rapidjson::Value &status_json = GetMember(json_root, "status");
-        if(!status_json.IsString() || strcmp(status_json.GetString(), "OK") != 0)
-            return PluginResult::ERR;
+        const Json::Value &results_json = json_root["results"];
+        if(!results_json.isArray())
+            return PluginResult::OK;
 
-        const rapidjson::Value &chapter_json = GetMember(json_root, "chapter");
-        if(!chapter_json.IsObject())
-            return PluginResult::ERR;
+        for(const Json::Value &result_item_json : results_json) {
+            if(!result_item_json.isObject())
+                continue;
 
-        time_t time_now = time(NULL);
-        auto body = create_body();
+            const Json::Value &result_json = result_item_json["result"];
+            if(!result_json.isString() || strcmp(result_json.asCString(), "ok") != 0)
+                continue;
 
-        /* TODO: Pointer */
-        std::string prev_chapter_number;
-        for(auto const &it : chapter_json.GetObject()) {
-            const std::string &chapter_id = it.name.GetString();
-            const rapidjson::Value &chapter = it.value;
+            const Json::Value &data_json = result_item_json["data"];
+            if(!data_json.isObject())
+                continue;
+
+            const Json::Value &id_json = data_json["id"];
+            if(!id_json.isString())
+                continue;
 
-            const rapidjson::Value &timestamp_json = chapter["timestamp"];
-            if(timestamp_json.IsInt64() && timestamp_json.GetInt64() > time_now)
+            const Json::Value &attributes_json = data_json["attributes"];
+            if(!attributes_json.isObject())
                 continue;
 
-            const rapidjson::Value &lang_code = chapter["lang_code"];
-            // TODO: Allow selecting other languages than english
-            if(!lang_code.IsString() || strcmp(lang_code.GetString(), "gb") != 0)
+            const Json::Value &translated_language_json = attributes_json["translatedLanguage"];
+            if(!translated_language_json.isString() || strcmp(translated_language_json.asCString(), "en") != 0)
                 continue;
 
-            const rapidjson::Value &chapter_number_json = chapter["chapter"];
-            if(!chapter_number_json.IsString())
+            const Json::Value &chapter_json = attributes_json["chapter"];
+            if(!chapter_json.isString())
                 continue;
 
-            std::string chapter_number_str = chapter_number_json.GetString();
-            if(chapter_number_str == prev_chapter_number)
+            std::string title = "Ch. " + chapter_json.asString();
+            const Json::Value &title_json = attributes_json["title"];
+            if(title_json.isString() && title_json.asCString()[0] != '\0')
+                title += " - " + title_json.asString();
+
+            const Json::Value &hash_json = attributes_json["hash"];
+            if(!hash_json.isString())
                 continue;
-            prev_chapter_number = chapter_number_str;
 
-            const rapidjson::Value &chapter_title_json = chapter["title"];
-            std::string chapter_url = mangadex_url + "/chapter/" + chapter_id;
-            std::string chapter_name = std::string("Ch. ") + chapter_number_str;
-            if(chapter_title_json.IsString() && chapter_title_json.GetStringLength() > 0)
-                chapter_name += std::string(" - ") + chapter_title_json.GetString();
+            const Json::Value &attributes_data_json = attributes_json["data"];
+            if(!attributes_data_json.isArray())
+                continue;
+
+            auto body_item = BodyItem::create(std::move(title));
+            body_item->url = id_json.asString();
+            auto images_list = std::make_shared<MangadexChapterImagesList>();
+            for(const Json::Value &data_item_json : attributes_data_json) {
+                if(!data_item_json.isString())
+                    continue;
 
-            auto item = BodyItem::create(std::move(chapter_name));
-            item->url = std::move(chapter_url);
-            body->items.push_back(std::move(item));
+                std::string url = hash_json.asString() + "/" + data_item_json.asString();
+                images_list->image_urls.push_back(std::move(url));
+            }
+            body_item->extra = std::move(images_list);
+            result_items.push_back(std::move(body_item));
         }
 
+        return PluginResult::OK;
+    }
+
+    PluginResult MangadexSearchPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
+        auto body = create_body();
+        get_chapters_for_manga(this, url, 0, body->items);
         result_tabs.push_back(Tab{std::move(body), std::make_unique<MangadexChaptersPage>(program, title, url), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
         return PluginResult::OK;
     }
 
-    bool MangadexSearchPage::get_rememberme_token(std::string &rememberme_token_output) {
-        if(rememberme_token) {
-            rememberme_token_output = rememberme_token.value();
-            return true;
-        }
+    PluginResult MangadexChaptersPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
+        Json::Value json_root;
+        if(download_json(json_root, "https://api.mangadex.org/at-home/server/" + url, {}, true) != DownloadResult::OK)
+            return PluginResult::NET_ERR;
 
-        Path mangadex_credentials_path = get_storage_dir().join("credentials").join("mangadex.json");
-        std::string mangadex_credentials;
-        if(file_get_content(mangadex_credentials_path, mangadex_credentials) != 0) {
-            fprintf(stderr, "Failed to get content of file: %s\n", mangadex_credentials_path.data.c_str());
-            return false;
-        }
+        if(!json_root.isObject())
+            return PluginResult::ERR;
 
-        Json::Value json_root;
-        Json::CharReaderBuilder json_builder;
-        std::unique_ptr<Json::CharReader> json_reader(json_builder.newCharReader());
-        std::string json_errors;
-        if(!json_reader->parse(&mangadex_credentials[0], &mangadex_credentials[mangadex_credentials.size()], &json_root, &json_errors)) {
-            fprintf(stderr, "Mangadex credentials json error: %s\n", json_errors.c_str());
-            return false;
-        }
+        const Json::Value &base_url_json = json_root["baseUrl"];
+        if(!base_url_json.isString())
+            return PluginResult::ERR;
+
+        std::string base_url = base_url_json.asString();
+        if(base_url.back() != '/')
+            base_url += '/';
 
-        if(json_root.isObject()) {
-            Json::Value &rememberme_token_json = json_root["rememberme_token"];
-            if(rememberme_token_json.isString())
-                rememberme_token_output = rememberme_token_json.asString();
+        auto image_urls = static_cast<MangadexChapterImagesList*>(submit_body_item->extra.get())->image_urls;
+        for(auto &image_url : image_urls) {
+            image_url = base_url + "data/" + image_url;
         }
 
-        rememberme_token = rememberme_token_output;
-        return true;
+        result_tabs.push_back(Tab{nullptr, std::make_unique<MangadexImagesPage>(program, content_title, title, content_url, image_urls), nullptr});
+        return PluginResult::OK;
     }
 
-    PluginResult MangadexChaptersPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
-        result_tabs.push_back(Tab{nullptr, std::make_unique<MangadexImagesPage>(program, content_title, title, url), nullptr});
-        return PluginResult::OK;
+    PluginResult MangadexChaptersPage::get_page(const std::string&, int page, BodyItems &result_items) {
+        return get_chapters_for_manga(this, content_url, page, result_items);
     }
 
     bool MangadexChaptersPage::extract_id_from_url(const std::string &url, std::string &manga_id) const {
-        manga_id = title_url_extract_manga_id(url);
+        size_t start_index = url.find("manga=");
+        if(start_index == std::string::npos) {
+            manga_id = url;
+            return true;
+        }
+
+        start_index += 6;
+        size_t end_index = url.find('&', start_index);
+        if(end_index == std::string::npos) {
+            manga_id = url.substr(start_index);
+            return true;
+        }
+        
+        manga_id = url.substr(start_index, end_index - start_index);
         return true;
     }
 
     ImageResult MangadexImagesPage::get_number_of_images(int &num_images) {
-        num_images = 0;
-        ImageResult image_result = get_image_urls_for_chapter(url);
-        if(image_result != ImageResult::OK) return image_result;
-        num_images = chapter_image_urls.size();
+        num_images = image_urls.size();
         return ImageResult::OK;
     }
 
     ImageResult MangadexImagesPage::for_each_page_in_chapter(PageCallback callback) {
-        ImageResult image_result = get_image_urls_for_chapter(url);
-        if(image_result != ImageResult::OK) return image_result;
-        for(const std::string &url : chapter_image_urls) {
+        for(const std::string &url : image_urls) {
             if(!callback(url))
                 break;
         }
         return ImageResult::OK;
     }
-
-    ImageResult MangadexImagesPage::get_image_urls_for_chapter(const std::string &url) {
-        if(!chapter_image_urls.empty())
-            return ImageResult::OK;
-
-        std::string cookie_filepath;
-        if(!get_cookie_filepath(cookie_filepath))
-            return ImageResult::ERR;
-
-        if(!save_mangadex_cookies(url, cookie_filepath))
-            return ImageResult::ERR;
-
-        CommandArg cookie_arg = { "-b", std::move(cookie_filepath) };
-        std::string manga_id = chapter_url_extract_manga_id(url);
-        std::string request_url = mangadex_url + "/api/?id=" + manga_id + "&server=null&type=chapter";
-
-        Json::Value json_root;
-        DownloadResult result = download_json(json_root, request_url, {std::move(cookie_arg)}, true);
-        if(result != DownloadResult::OK) return download_result_to_image_result(result);
-
-        if(!json_root.isObject())
-            return ImageResult::ERR;
-
-        Json::Value &status_json = json_root["status"];
-        if(!status_json.isString() || status_json.asString() != "OK")
-            return ImageResult::ERR;
-
-        Json::Value &chapter_hash = json_root["hash"];
-        if(!chapter_hash.isString())
-            return ImageResult::ERR;
-        const char *chapter_hash_str = chapter_hash.asCString();
-
-        Json::Value &server_json = json_root["server"];
-        std::string server;
-        if(server_json.isString())
-            server = server_json.asString();
-        else
-            server = mangadex_url + "/data/";
-
-        Json::Value &page_array = json_root["page_array"];
-        if(page_array.isArray()) {
-            for(const Json::Value &image_name : page_array) {
-                if(!image_name.isString())
-                    continue;
-
-                std::string image_url = server + chapter_hash_str + "/" + image_name.asCString();
-                chapter_image_urls.push_back(std::move(image_url));
-            }
-        }
-
-        if(chapter_image_urls.empty())
-            return ImageResult::ERR;
-
-        return ImageResult::OK;
-    }
-
-    bool MangadexImagesPage::save_mangadex_cookies(const std::string &url, const std::string &cookie_filepath) {
-        CommandArg cookie_arg = { "-c", cookie_filepath };
-        std::string server_response;
-        if(download_to_string(url, server_response, {std::move(cookie_arg)}, true) != DownloadResult::OK)
-            return false;
-
-        return true;
-    }
 }