path: root/src/plugins/MangaGeneric.cpp

#include "../../plugins/MangaGeneric.hpp"
#include "../../include/StringUtils.hpp"
#include "../../include/Theme.hpp"
#include <quickmedia/HtmlSearch.h>
#include <assert.h>

namespace QuickMedia {
    struct HtmlSearchUserdata {
        BodyItems *body_items;
        const char *field1 = nullptr;
        const char *field2 = nullptr;
        const char *field2_contains = nullptr;
    };

    enum class MergeType {
        THUMBNAIL,
        DESCRIPTION
    };

    struct HtmlMergeUserdata {
        MergeType type;
        BodyItemContext body_item_image_context;
        const char *field_name = nullptr;
        const char *field_contains = nullptr;
        const char *desc_prefix = nullptr;
    };

    struct HtmlListPageImagesUserdata {
        std::vector<std::string> *urls;
        const char *field_name = nullptr;
        const char *field_contains = nullptr;
    };

    struct HtmlPageCountUserdata {
        int num_pages = 0;
        const char *field_name = nullptr;
        const char *field_contains = nullptr;
    };

    struct HtmlPageImageUserdata {
        std::string *url = nullptr;
        const char *field_name = nullptr;
        const char *field_contains = nullptr;
    };

    static bool string_view_contains(const QuickMediaStringView str, const char *sub) {
        return memmem(str.data, str.size, sub, strlen(sub));
    }

    static QuickMediaStringView html_attr_or_inner_text(QuickMediaMatchNode *node, const char *field_name) {
        if(strcmp(field_name, "text") == 0)
            return quickmedia_html_node_get_text(node);
        else
            return quickmedia_html_node_get_attribute_value(node, field_name);
    }

    static void body_items_prepend_website_url(BodyItems &body_items, const std::string &website_url) {
        for(auto &body_item : body_items) {
            if(string_starts_with(body_item->url, "//"))
                body_item->url = "https://" + body_item->url.substr(2);
            else if(string_starts_with(body_item->url, "/"))
                body_item->url = website_url + body_item->url.substr(1);

            if(string_starts_with(body_item->thumbnail_url, "//"))
                body_item->thumbnail_url = "https://" + body_item->thumbnail_url.substr(2);
            else if(string_starts_with(body_item->thumbnail_url, "/"))
                body_item->thumbnail_url = website_url + body_item->thumbnail_url.substr(1);
        }
    }

    static int html_append_search(QuickMediaHtmlSearch *html_search, const char *html_query, HtmlSearchUserdata *search_userdata) {
        return quickmedia_html_find_nodes_xpath(html_search, html_query,
            [](QuickMediaMatchNode *node, void *userdata) {
                HtmlSearchUserdata *search_userdata = (HtmlSearchUserdata*)userdata;
                QuickMediaStringView field1_value = html_attr_or_inner_text(node, search_userdata->field1);
                if(search_userdata->field2) {
                    QuickMediaStringView field2_value = html_attr_or_inner_text(node, search_userdata->field2);
                    if(field1_value.data && field2_value.data && (!search_userdata->field2_contains || string_view_contains(field2_value, search_userdata->field2_contains))) {
                        std::string field1_fixed(field1_value.data, field1_value.size);
                        html_unescape_sequences(field1_fixed);
                        auto item = BodyItem::create(std::move(field1_fixed));
                        item->url = std::string(field2_value.data, field2_value.size);
                        search_userdata->body_items->push_back(std::move(item));
                    }
                } else {
                    if(field1_value.data) {
                        std::string field1_fixed(field1_value.data, field1_value.size);
                        html_unescape_sequences(field1_fixed);
                        auto item = BodyItem::create(std::move(field1_fixed));
                        search_userdata->body_items->push_back(std::move(item));
                    }
                }
                return 0;
            }, search_userdata);
    }

    static int html_body_item_merge(QuickMediaHtmlSearch *html_search, const char *html_query, HtmlMergeUserdata *merge_userdata) {
        return quickmedia_html_find_nodes_xpath(html_search, html_query,
            [](QuickMediaMatchNode *node, void *userdata) {
                HtmlMergeUserdata *merge_userdata = (HtmlMergeUserdata*)userdata;
                BodyItemContext &body_item_image_context = merge_userdata->body_item_image_context;
                QuickMediaStringView field_value = html_attr_or_inner_text(node, merge_userdata->field_name);
                if(body_item_image_context.index < body_item_image_context.body_items->size()
                    && field_value.data && (!merge_userdata->field_contains || string_view_contains(field_value, merge_userdata->field_contains)))
                {
                    std::string field_stripped(field_value.data, field_value.size);
                    if(merge_userdata->type == MergeType::THUMBNAIL) {
                        (*body_item_image_context.body_items)[body_item_image_context.index]->thumbnail_url = std::move(field_stripped);
                    } else if(merge_userdata->type == MergeType::DESCRIPTION) {
                        const char *prefix = merge_userdata->desc_prefix ? merge_userdata->desc_prefix : "";
                        (*body_item_image_context.body_items)[body_item_image_context.index]->set_description(prefix + std::move(field_stripped));
                        (*body_item_image_context.body_items)[body_item_image_context.index]->set_description_color(get_current_theme().faded_text_color);
                    }
                    body_item_image_context.index++;
                }
                return 0;
            }, merge_userdata);
    }

    static int html_get_page_url(QuickMediaHtmlSearch *html_search, const char *html_query, HtmlPageImageUserdata *page_image_userdata) {
        return quickmedia_html_find_nodes_xpath(html_search, html_query,
            [](QuickMediaMatchNode *node, void *userdata) {
                HtmlPageImageUserdata *page_image_userdata = (HtmlPageImageUserdata*)userdata;
                QuickMediaStringView field1_value = html_attr_or_inner_text(node, page_image_userdata->field_name);
                if(page_image_userdata->url->empty() && field1_value.data && (!page_image_userdata->field_contains || string_view_contains(field1_value, page_image_userdata->field_contains))) {
                    *page_image_userdata->url = std::string(field1_value.data, field1_value.size);
                }
                return 0;
            }, page_image_userdata);
    }

    MangaGenericSearchPage::MangaGenericSearchPage(Program *program, const char *service_name, const char *website_url, bool fail_on_http_error) :
        Page(program), service_name(service_name), website_url(website_url ? website_url : ""), fail_on_http_error(fail_on_http_error)
    {
        if(!this->website_url.empty()) {
            if(this->website_url.back() != '/')
                this->website_url.push_back('/');
        }
    }

    SearchResult MangaGenericSearchPage::search(const std::string &str, BodyItems &result_items) {
        return plugin_result_to_search_result(get_page(str, 0, result_items));
    }

    PluginResult MangaGenericSearchPage::get_page(const std::string &url, bool is_post, const std::vector<MangaFormDataStr> &form_data, BodyItems &result_items) {
        std::vector<CommandArg> args;
        if(!website_url.empty())
            args.push_back({ "-H", "referer: " + website_url });

        if(is_post) {
            args.push_back({ "-X", "POST" });
            args.push_back({ "-H", "x-requested-with: XMLHttpRequest" });
            for(const MangaFormDataStr &form : form_data) {
                args.push_back({ "--form-string", std::string(form.key) + "=" + form.value });
            }

            assert(search_query.json_handler);
            std::string err_msg;
            Json::Value json_root;
            if(download_json(json_root, url, args, true, fail_on_http_error ? nullptr : &err_msg) != DownloadResult::OK)
                return PluginResult::NET_ERR;

            result_items = search_query.json_handler(json_root);
            body_items_prepend_website_url(result_items, website_url);
            return PluginResult::OK;
        }

        std::string target_url;
        std::string website_data;
        if(download_to_string(url, website_data, args, true, fail_on_http_error) != DownloadResult::OK)
            return PluginResult::NET_ERR;

        if(website_data.empty())
            return PluginResult::OK;

        QuickMediaHtmlSearch html_search;
        int result = quickmedia_html_search_init(&html_search, website_data.c_str(), website_data.size());
        if(result != 0)
            goto cleanup;

        for(const TextQuery &text_query : text_queries) {
            if(!text_query.html_query || !text_query.title_field) {
                assert(false);
                result = -1;
                goto cleanup;
            }

            BodyItems new_result_items;
            HtmlSearchUserdata search_userdata;
            search_userdata.body_items = &new_result_items;
            search_userdata.field1 = text_query.title_field;
            search_userdata.field2 = text_query.url_field;
            search_userdata.field2_contains = text_query.url_contains;

            result = html_append_search(&html_search, text_query.html_query, &search_userdata);
            if(result != 0)
                goto cleanup;

            if(!text_query.url_field && !new_result_items.empty()) {
                if(target_url.empty()) {
                    std::string response_headers;
                    DownloadResult download_result = download_head_to_string(url, response_headers, true); 
                    if(download_result != DownloadResult::OK) {
                        result = -1;
                        goto cleanup;
                    }

                    target_url = header_extract_value(response_headers, "location");
                    if(target_url.empty()) {
                        fprintf(stderr, "Failed to extract target location from %s HEAD\n", url.c_str());
                        result = -1;
                        goto cleanup;
                    }
                }

                for(auto &new_body_item : new_result_items) {
                    new_body_item->url = target_url;
                }
            }

            result_items.insert(result_items.end(), std::move_iterator(new_result_items.begin()), std::move_iterator(new_result_items.end()));
        }

        for(const DescriptionQuery &description_query : description_queries) {
            assert(description_query.html_query && description_query.field_name);
            if(description_query.html_query && description_query.field_name) {
                HtmlMergeUserdata merge_userdata;
                merge_userdata.type = MergeType::DESCRIPTION;
                merge_userdata.body_item_image_context.body_items = &result_items;
                merge_userdata.body_item_image_context.index = 0;
                merge_userdata.field_name = description_query.field_name;
                merge_userdata.field_contains = nullptr;
                result = html_body_item_merge(&html_search, description_query.html_query, &merge_userdata);
                if(result != 0)
                    goto cleanup;
            }
        }

        for(const ThumbnailQuery &thumbnail_query : thumbnail_queries) {
            assert(thumbnail_query.html_query && thumbnail_query.field_name);
            if(thumbnail_query.html_query && thumbnail_query.field_name) {
                HtmlMergeUserdata merge_userdata;
                merge_userdata.type = MergeType::THUMBNAIL;
                merge_userdata.body_item_image_context.body_items = &result_items;
                merge_userdata.body_item_image_context.index = 0;
                merge_userdata.field_name = thumbnail_query.field_name;
                merge_userdata.field_contains = thumbnail_query.field_contains;
                result = html_body_item_merge(&html_search, thumbnail_query.html_query, &merge_userdata);
                if(result != 0)
                    goto cleanup;
            }
        }

        body_items_prepend_website_url(result_items, website_url);

        cleanup:
        quickmedia_html_search_deinit(&html_search);
        if(result == 0) {
            return PluginResult::OK;
        } else {
            result_items.clear();
            return PluginResult::ERR;
        }
    }

    PluginResult MangaGenericSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) {
        if(search_query.is_post) {
            if(page != 0)
                return PluginResult::OK;

            std::vector<MangaFormDataStr> form_data(search_query.form_data.size());
            for(size_t i = 0; i < form_data.size(); ++i) {
                MangaFormDataStr &form = form_data[i];
                form.key = search_query.form_data[i].key;
                form.value = search_query.form_data[i].value;
                string_replace_all(form.value, "%s", str);
            }
            return get_page(search_query.search_template, search_query.is_post, form_data, result_items);
        } else {
            std::string url = search_query.search_template;
            string_replace_all(url, "%s", url_param_encode(str));
            size_t num_replaced = string_replace_all(url, "%p", std::to_string(search_query.page_start + page));
            if(num_replaced == 0 && page != 0)
                return PluginResult::OK;
            else
                return get_page(url, false, {}, result_items);
        }
    }

    PluginResult MangaGenericSearchPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
        if(!list_chapters_query.html_query || !list_chapters_query.title_field || !list_chapters_query.url_field) {
            assert(false);
            return PluginResult::ERR;
        }

        std::vector<CommandArg> args;
        if(!website_url.empty())
            args.push_back({ "-H", "referer: " + website_url });

        std::map<std::string, std::string> creators; // key = name, value = url
        BodyItems chapters_items;
        HtmlSearchUserdata search_userdata;
        search_userdata.body_items = &chapters_items;
        search_userdata.field1 = list_chapters_query.title_field;
        search_userdata.field2 = list_chapters_query.url_field;
        search_userdata.field2_contains = list_chapters_query.url_contains;

        std::string website_data;
        if(download_to_string(url, website_data, args, true, fail_on_http_error) != DownloadResult::OK)
            return PluginResult::NET_ERR;

        QuickMediaHtmlSearch html_search;
        int result = quickmedia_html_search_init(&html_search, website_data.c_str(), website_data.size());
        if(result != 0)
            goto cleanup;

        result = html_append_search(&html_search, list_chapters_query.html_query, &search_userdata);
        if(result != 0)
            goto cleanup;

        assert(!list_chapters_query.uploaded_time_html_query || list_chapters_query.uploaded_time_field_name);
        if(list_chapters_query.uploaded_time_html_query && list_chapters_query.uploaded_time_field_name) {
            HtmlMergeUserdata merge_userdata;
            merge_userdata.type = MergeType::DESCRIPTION;
            merge_userdata.body_item_image_context.body_items = &chapters_items;
            merge_userdata.body_item_image_context.index = 0;
            merge_userdata.field_name = list_chapters_query.uploaded_time_field_name;
            merge_userdata.field_contains = list_chapters_query.uploaded_time_field_contains;
            merge_userdata.desc_prefix = "Uploaded: ";
            result = html_body_item_merge(&html_search, list_chapters_query.uploaded_time_html_query, &merge_userdata);
        }

        struct HtmlAuthorsUserdata {
            std::map<std::string, std::string> *creators;
            const AuthorsQuery *authors_query;
        };

        for(const AuthorsQuery &authors_query : authors_queries) {
            if(authors_query.html_query && authors_query.title_field && authors_query.url_field) {
                HtmlAuthorsUserdata authors_userdata;
                authors_userdata.creators = &creators;
                authors_userdata.authors_query = &authors_query;

                quickmedia_html_find_nodes_xpath(&html_search, authors_query.html_query,
                    [](QuickMediaMatchNode *node, void *userdata) {
                        HtmlAuthorsUserdata *authors_userdata = (HtmlAuthorsUserdata*)userdata;
                        QuickMediaStringView title_value = html_attr_or_inner_text(node, authors_userdata->authors_query->title_field);
                        QuickMediaStringView url_value = html_attr_or_inner_text(node, authors_userdata->authors_query->url_field);
                        if(title_value.data && url_value.data && (!authors_userdata->authors_query->url_contains || string_view_contains(url_value, authors_userdata->authors_query->url_contains)))
                            (*authors_userdata->creators)[std::string(title_value.data, title_value.size)] = std::string(url_value.data, url_value.size);
                        return 0;
                    }, &authors_userdata);
            }
        }

        body_items_prepend_website_url(chapters_items, website_url);

        for(auto &it : creators) {
            if(string_starts_with(it.second, "//"))
                it.second = "https://" + it.second.substr(2);
            else if(string_starts_with(it.second, "/"))
                it.second = website_url + it.second.substr(1);
        }

        cleanup:
        quickmedia_html_search_deinit(&html_search);
        if(result != 0)
            return PluginResult::ERR;

        auto body = create_body();
        body->set_items(std::move(chapters_items));
        result_tabs.push_back(Tab{std::move(body), std::make_unique<MangaGenericChaptersPage>(program, title, url, manga_id_extractor, service_name, website_url, &list_page_query, fail_on_http_error), create_search_bar("Search...", SEARCH_DELAY_FILTER)});

        for(auto &it : creators) {
            Creator creator;
            creator.name = it.first;
            creator.url = it.second;
            result_tabs.push_back(Tab{create_body(), std::make_unique<MangaGenericCreatorPage>(program, this, std::move(creator)), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
        }

        return PluginResult::OK;
    }

    PluginResult MangaGenericChaptersPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
        result_tabs.push_back(Tab{nullptr, std::make_unique<MangaGenericImagesPage>(program, content_title, title, url, service_name, website_url, list_page_query, fail_on_http_error), nullptr});
        return PluginResult::OK;
    }

    bool MangaGenericChaptersPage::extract_id_from_url(const std::string &url, std::string &manga_id) const {
        size_t start_index = url.rfind(manga_id_extractor.prefix);
        if(start_index == std::string::npos)
            return false;

        start_index += strlen(manga_id_extractor.prefix);
        if(!manga_id_extractor.end) {
            manga_id = url.substr(start_index);
            return true;
        }
        
        size_t end_index = url.find(manga_id_extractor.end, start_index);
        if(end_index == std::string::npos) {
            manga_id = url.substr(start_index);
            return true;
        }
        
        manga_id = url.substr(start_index, end_index - start_index);
        return true;
    }

    PluginResult MangaGenericCreatorPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
        return search_page->submit(title, url, result_tabs);
    }

    PluginResult MangaGenericCreatorPage::lazy_fetch(BodyItems &result_items) {
        return search_page->get_page(creator.url, false, {}, result_items);
    }

    static bool is_number(const char *str) {
        while(*str) {
            char c = *str;
            if(c < '0' || c > '9')
                return false;
            ++str;
        }
        return true;
    }

    ImageResult MangaGenericImagesPage::get_number_of_images(int &num_images) {
        num_images = 0;
        switch(list_page_query->type) {
            case ListPageQueryType::IMAGES: {
                ImageResult result = get_page_image_urls();
                if(result != ImageResult::OK) return result;
                num_images = chapter_image_urls.size();
                return ImageResult::OK;
            }
            case ListPageQueryType::PAGINATION: {
                const ListPagePaginationQuery *list_page_pagination_query = &list_page_query->pagination_query;
                if(!list_page_pagination_query->pages_html_query || !list_page_pagination_query->pages_field_name
                    || !list_page_pagination_query->image_html_query || !list_page_pagination_query->image_field_name
                    || !list_page_pagination_query->next_page_html_query || !list_page_pagination_query->next_page_field_name)
                {
                    assert(false);
                    return ImageResult::ERR;
                }

                if(chapter_num_pages != -1) {
                    num_images = chapter_num_pages;
                    return ImageResult::OK;
                }

                current_image_url.clear();
                next_page_url.clear();

                HtmlPageCountUserdata page_count_userdata;
                page_count_userdata.num_pages = 0;
                page_count_userdata.field_name = list_page_pagination_query->pages_field_name;
                page_count_userdata.field_contains = nullptr;

                HtmlPageImageUserdata page_image_userdata;
                page_image_userdata.url = &current_image_url;
                page_image_userdata.field_name = list_page_pagination_query->image_field_name;
                page_image_userdata.field_contains = list_page_pagination_query->image_field_contains;

                HtmlPageImageUserdata next_page_userdata;
                next_page_userdata.url = &next_page_url;
                next_page_userdata.field_name = list_page_pagination_query->next_page_field_name;
                next_page_userdata.field_contains = list_page_pagination_query->next_page_field_contains;

                std::vector<CommandArg> args;
                if(!website_url.empty())
                    args.push_back({ "-H", "referer: " + website_url });

                std::string website_data;
                if(download_to_string(url, website_data, args, true, fail_on_http_error) != DownloadResult::OK)
                    return ImageResult::NET_ERR;

                QuickMediaHtmlSearch html_search;
                int result = quickmedia_html_search_init(&html_search, website_data.c_str(), website_data.size());
                if(result != 0)
                    goto cleanup;

                result = quickmedia_html_find_nodes_xpath(&html_search, list_page_pagination_query->pages_html_query,
                    [](QuickMediaMatchNode *node, void *userdata) {
                        HtmlPageCountUserdata *page_count_userdata = (HtmlPageCountUserdata*)userdata;
                        QuickMediaStringView field1_value = html_attr_or_inner_text(node, page_count_userdata->field_name);
                        if(field1_value.data) {
                            std::string field_value_stripped(field1_value.data, field1_value.size);
                            if(is_number(field_value_stripped.c_str()))
                                page_count_userdata->num_pages = strtoll(field_value_stripped.c_str(), nullptr, 10);
                        }
                        return 0;
                    }, &page_count_userdata);

                if(result != 0 || page_count_userdata.num_pages == 0) {
                    result = -1;
                    goto cleanup;
                }

                result = html_get_page_url(&html_search, list_page_pagination_query->image_html_query, &page_image_userdata);
                if(result != 0 || current_image_url.empty()) {
                    result = -1;
                    goto cleanup;
                }

                result = html_get_page_url(&html_search, list_page_pagination_query->next_page_html_query, &next_page_userdata);
                if(next_page_url.empty())
                    result = -1;

                cleanup:
                quickmedia_html_search_deinit(&html_search);
                if(result != 0) {
                    current_image_url.clear();
                    next_page_url.clear();
                    return ImageResult::ERR;
                }

                if(string_starts_with(current_image_url, "//"))
                    current_image_url = "https://" + current_image_url.substr(2);
                else if(string_starts_with(current_image_url, "/"))
                    current_image_url = website_url + current_image_url.substr(1);

                num_images = page_count_userdata.num_pages;
                chapter_num_pages = num_images;
                return ImageResult::OK;
            }
            case ListPageQueryType::CUSTOM: {
                ImageResult result = get_page_image_urls();
                if(result != ImageResult::OK) return result;
                num_images = chapter_image_urls.size();
                return ImageResult::OK;
            }
        }
        return ImageResult::OK;
    }

    ImageResult MangaGenericImagesPage::for_each_page_in_chapter(PageCallback callback) {
        switch(list_page_query->type) {
            case ListPageQueryType::IMAGES: {
                ImageResult result = get_page_image_urls();
                if(result != ImageResult::OK) return result;
                for(const std::string &url : chapter_image_urls) {
                    if(!callback(url))
                        break;
                }
                return ImageResult::OK;
            }
            case ListPageQueryType::PAGINATION: {
                const ListPagePaginationQuery *list_page_pagination_query = &list_page_query->pagination_query;
                if(!list_page_pagination_query->image_html_query || !list_page_pagination_query->image_field_name
                    || !list_page_pagination_query->next_page_html_query || !list_page_pagination_query->next_page_field_name)
                {
                    assert(false);
                    return ImageResult::ERR;
                }

                int num_images = 0;
                ImageResult result = get_number_of_images(num_images);
                if(result != ImageResult::OK) return result;

                if(!callback(current_image_url))
                    return ImageResult::OK;

                for(int i = 0; i < num_images; ++i) {
                    std::string full_url = url + next_page_url;
                    current_image_url.clear();
                    next_page_url.clear();

                    HtmlPageImageUserdata page_image_userdata;
                    page_image_userdata.url = &current_image_url;
                    page_image_userdata.field_name = list_page_pagination_query->image_field_name;
                    page_image_userdata.field_contains = list_page_pagination_query->image_field_contains;

                    HtmlPageImageUserdata next_page_userdata;
                    next_page_userdata.url = &next_page_url;
                    next_page_userdata.field_name = list_page_pagination_query->next_page_field_name;
                    next_page_userdata.field_contains = list_page_pagination_query->next_page_field_contains;

                    std::vector<CommandArg> args;
                    if(!website_url.empty())
                        args.push_back({ "-H", "referer: " + website_url });

                    std::string image_src;
                    std::string website_data;
                    DownloadErrorHandler error_callback = [](std::string&){ return true; };
                    if(fail_on_http_error)
                        error_callback = nullptr;
                    if(download_to_string_cache(full_url, website_data, args, true, error_callback) != DownloadResult::OK)
                        return ImageResult::ERR;

                    QuickMediaHtmlSearch html_search;
                    int result = quickmedia_html_search_init(&html_search, website_data.c_str(), website_data.size());
                    if(result != 0)
                        goto cleanup;

                    html_get_page_url(&html_search, list_page_pagination_query->image_html_query, &page_image_userdata);
                    html_get_page_url(&html_search, list_page_pagination_query->next_page_html_query, &next_page_userdata);

                    cleanup:
                    quickmedia_html_search_deinit(&html_search);

                    if(string_starts_with(current_image_url, "//"))
                        current_image_url = "https://" + current_image_url.substr(2);
                    else if(string_starts_with(current_image_url, "/"))
                        current_image_url = website_url + current_image_url.substr(1);

                    if(!callback(current_image_url))
                        break;
                }

                return ImageResult::OK;
            }
            case ListPageQueryType::CUSTOM: {
                ImageResult result = get_page_image_urls();
                if(result != ImageResult::OK) return result;
                for(const std::string &url : chapter_image_urls) {
                    if(!callback(url))
                        break;
                }
                return ImageResult::OK;
            }
        }
        return ImageResult::OK;
    }

    ImageResult MangaGenericImagesPage::get_page_image_urls() {
        if(!chapter_image_urls.empty())
            return ImageResult::OK;

        std::vector<CommandArg> args;
        if(!website_url.empty())
            args.push_back({ "-H", "referer: " + website_url });

        std::string website_data;
        if(download_to_string(url, website_data, args, true, fail_on_http_error) != DownloadResult::OK)
            return ImageResult::NET_ERR;

        if(list_page_query->type == ListPageQueryType::IMAGES) {
            const ListPageImagesQuery *list_page_images_query = &list_page_query->images_query;
            if(!list_page_images_query->html_query || !list_page_images_query->field_name) {
                assert(false);
                return ImageResult::ERR;
            }

            HtmlListPageImagesUserdata list_page_images_userdata;
            list_page_images_userdata.urls = &chapter_image_urls;
            list_page_images_userdata.field_name = list_page_images_query->field_name;
            list_page_images_userdata.field_contains = list_page_images_query->field_contains;

            QuickMediaHtmlSearch html_search;
            int result = quickmedia_html_search_init(&html_search, website_data.c_str(), website_data.size());
            if(result != 0)
                goto cleanup;

            result = quickmedia_html_find_nodes_xpath(&html_search, list_page_images_query->html_query,
                [](QuickMediaMatchNode *node, void *userdata) {
                    HtmlListPageImagesUserdata *list_page_images_userdata = (HtmlListPageImagesUserdata*)userdata;
                    QuickMediaStringView field1_value = html_attr_or_inner_text(node, list_page_images_userdata->field_name);
                    if(field1_value.data && (!list_page_images_userdata->field_contains || string_view_contains(field1_value, list_page_images_userdata->field_contains))) {
                        list_page_images_userdata->urls->push_back(std::string(field1_value.data, field1_value.size));
                    }
                    return 0;
                }, &list_page_images_userdata);

            if(result == 0 && !chapter_image_urls.empty() && list_page_images_query->post_handler)
                list_page_images_query->post_handler(chapter_image_urls);

            cleanup:
            quickmedia_html_search_deinit(&html_search);
            if(result != 0 || chapter_image_urls.empty()) {
                chapter_image_urls.clear();
                return ImageResult::ERR;
            }
        } else if(list_page_query->type == ListPageQueryType::CUSTOM) {
            const ListPageCustomQuery *list_page_custom_query = &list_page_query->custom_query;
            if(!list_page_custom_query->handler) {
                assert(false);
                return ImageResult::ERR;
            }
            chapter_image_urls = list_page_custom_query->handler(website_data);
        } else {
            assert(false);
            return ImageResult::ERR;
        }

        for(std::string &url : chapter_image_urls) {
            if(string_starts_with(url, "//"))
                url = "https://" + url.substr(2);
            else if(string_starts_with(url, "/"))
                url = website_url + url.substr(1);
        }

        chapter_num_pages = chapter_image_urls.size();
        return ImageResult::OK;
    }


    MangaGenericSearchPage& MangaGenericSearchPage::search_handler(const char *search_template, int page_start) {
        search_query.search_template = search_template;
        search_query.page_start = page_start;
        search_query.form_data.clear();
        search_query.is_post = false;
        search_query.json_handler = nullptr;
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::search_post_handler(const char *url, std::vector<MangaFormData> form_data, SearchQueryJsonHandler result_handler) {
        search_query.search_template = url;
        search_query.page_start = 1;
        search_query.form_data = std::move(form_data);
        search_query.is_post = true;
        search_query.json_handler = std::move(result_handler);
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::text_handler(std::vector<TextQuery> queries) {
        text_queries = std::move(queries);
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::description_handler(std::vector<DescriptionQuery> queries) {
        description_queries = std::move(queries);
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::thumbnail_handler(std::vector<ThumbnailQuery> queries) {
        thumbnail_queries = std::move(queries);
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::authors_handler(std::vector<AuthorsQuery> queries) {
        authors_queries = std::move(queries);
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::list_chapters_handler(const char *html_query, const char *title_field, const char *url_field, const char *url_contains) {
        list_chapters_query.html_query = html_query;
        list_chapters_query.title_field = title_field;
        list_chapters_query.url_field = url_field;
        list_chapters_query.url_contains = url_contains;
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::list_chapters_uploaded_time_handler(const char *html_query, const char *field_name, const char *field_contains) {
        list_chapters_query.uploaded_time_html_query = html_query;
        list_chapters_query.uploaded_time_field_name = field_name;
        list_chapters_query.uploaded_time_field_contains = field_contains;
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::list_page_images_handler(const char *html_query, const char *field_name, const char *field_contains, ListPageImagesQueryPost post_handler) {
        list_page_query.type = ListPageQueryType::IMAGES;
        list_page_query.images_query.html_query = html_query;
        list_page_query.images_query.field_name = field_name;
        list_page_query.images_query.field_contains = field_contains;
        list_page_query.images_query.post_handler = std::move(post_handler);
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::list_page_images_pagination_handler(
        const char *pages_html_query, const char *pages_field_name,
        const char *image_html_query, const char *image_field_name, const char *image_field_contains,
        const char *next_page_html_query, const char *next_page_field_name, const char *next_page_field_contains)
    {
        list_page_query.type = ListPageQueryType::PAGINATION;
        list_page_query.pagination_query.pages_html_query = pages_html_query;
        list_page_query.pagination_query.pages_field_name = pages_field_name;

        list_page_query.pagination_query.image_html_query = image_html_query;
        list_page_query.pagination_query.image_field_name = image_field_name;
        list_page_query.pagination_query.image_field_contains = image_field_contains;

        list_page_query.pagination_query.next_page_html_query = next_page_html_query;
        list_page_query.pagination_query.next_page_field_name = next_page_field_name;
        list_page_query.pagination_query.next_page_field_contains = next_page_field_contains;
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::list_page_images_custom_handler(ListPageCustomHandler handler) {
        assert(handler);
        list_page_query.type = ListPageQueryType::CUSTOM;
        list_page_query.custom_query.handler = std::move(handler);
        return *this;
    }

    MangaGenericSearchPage& MangaGenericSearchPage::manga_id_handler(const char *prefix, const char *end) {
        manga_id_extractor.prefix = prefix;
        manga_id_extractor.end = end;
        return *this;
    }
}