From 1ba58ba74a767a91b28f44eb75db41455adcaa70 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Tue, 27 Apr 2021 14:00:35 +0200 Subject: Add authors page for mangakatana and mangatown, attempt to fix image loading getting stuck, misc fixed related to async in plugins --- src/plugins/MangaGeneric.cpp | 92 ++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 37 deletions(-) (limited to 'src/plugins/MangaGeneric.cpp') diff --git a/src/plugins/MangaGeneric.cpp b/src/plugins/MangaGeneric.cpp index 58bedf4..bf8a4c4 100644 --- a/src/plugins/MangaGeneric.cpp +++ b/src/plugins/MangaGeneric.cpp @@ -145,9 +145,13 @@ namespace QuickMedia { } PluginResult MangaGenericSearchPage::get_page(const std::string &url, BodyItems &result_items) { + std::vector args; + if(!website_url.empty()) + args.push_back({ "-H", "referer: " + website_url }); + std::string target_url; std::string website_data; - if(download_to_string(url, website_data, {}, true, fail_on_http_error) != DownloadResult::OK) + if(download_to_string(url, website_data, args, true, fail_on_http_error) != DownloadResult::OK) return PluginResult::NET_ERR; if(website_data.empty()) @@ -265,7 +269,11 @@ namespace QuickMedia { return PluginResult::ERR; } - std::vector creators; + std::vector args; + if(!website_url.empty()) + args.push_back({ "-H", "referer: " + website_url }); + + std::map creators; // key = name, value = url BodyItems chapters_items; HtmlSearchUserdata search_userdata; search_userdata.body_items = &chapters_items; @@ -274,7 +282,7 @@ namespace QuickMedia { search_userdata.field2_contains = list_chapters_query.url_contains; std::string website_data; - if(download_to_string(url, website_data, {}, true, fail_on_http_error) != DownloadResult::OK) + if(download_to_string(url, website_data, args, true, fail_on_http_error) != DownloadResult::OK) return PluginResult::NET_ERR; QuickMediaHtmlSearch html_search; @@ -298,28 +306,26 @@ namespace QuickMedia { result = html_body_item_merge(&html_search, list_chapters_query.uploaded_time_html_query, &merge_userdata); } - if(authors_query.html_query && authors_query.title_field && authors_query.url_field) { - struct HtmlAuthorsUserdata { - std::vector *creators; - AuthorsQuery *authors_query; - }; + struct HtmlAuthorsUserdata { + std::map *creators; + const AuthorsQuery *authors_query; + }; - HtmlAuthorsUserdata authors_userdata; - authors_userdata.creators = &creators; - authors_userdata.authors_query = &authors_query; + for(const AuthorsQuery &authors_query : authors_queries) { + if(authors_query.html_query && authors_query.title_field && authors_query.url_field) { + HtmlAuthorsUserdata authors_userdata; + authors_userdata.creators = &creators; + authors_userdata.authors_query = &authors_query; - quickmedia_html_find_nodes_xpath(&html_search, authors_query.html_query, - [](QuickMediaHtmlNode *node, void *userdata) { - HtmlAuthorsUserdata *authors_userdata = (HtmlAuthorsUserdata*)userdata; - const char *title_value = html_attr_or_inner_text(node, authors_userdata->authors_query->title_field); - const char *url_value = html_attr_or_inner_text(node, authors_userdata->authors_query->url_field); - if(title_value && url_value && (!authors_userdata->authors_query->url_contains || strstr(url_value, authors_userdata->authors_query->url_contains))) { - Creator creator; - creator.name = strip(title_value); - creator.url = strip(url_value); - authors_userdata->creators->push_back(std::move(creator)); - } - }, &authors_userdata); + quickmedia_html_find_nodes_xpath(&html_search, authors_query.html_query, + [](QuickMediaHtmlNode *node, void *userdata) { + HtmlAuthorsUserdata *authors_userdata = (HtmlAuthorsUserdata*)userdata; + const char *title_value = html_attr_or_inner_text(node, authors_userdata->authors_query->title_field); + const char *url_value = html_attr_or_inner_text(node, authors_userdata->authors_query->url_field); + if(title_value && url_value && (!authors_userdata->authors_query->url_contains || strstr(url_value, authors_userdata->authors_query->url_contains))) + (*authors_userdata->creators)[strip(title_value)] = strip(url_value); + }, &authors_userdata); + } } for(auto &body_item : chapters_items) { @@ -334,11 +340,11 @@ namespace QuickMedia { body_item->thumbnail_url = website_url + body_item->thumbnail_url.substr(1); } - for(auto &creator : creators) { - if(string_starts_with(creator.url, "//")) - creator.url = "https://" + creator.url.substr(2); - else if(string_starts_with(creator.url, "/")) - creator.url = website_url + creator.url.substr(1); + for(auto &it : creators) { + if(string_starts_with(it.second, "//")) + it.second = "https://" + it.second.substr(2); + else if(string_starts_with(it.second, "/")) + it.second = website_url + it.second.substr(1); } cleanup: @@ -350,7 +356,10 @@ namespace QuickMedia { body->items = std::move(chapters_items); result_tabs.push_back(Tab{std::move(body), std::make_unique(program, title, url, manga_id_extractor, service_name, website_url, &list_page_query, fail_on_http_error), create_search_bar("Search...", SEARCH_DELAY_FILTER)}); - for(Creator &creator : creators) { + for(auto &it : creators) { + Creator creator; + creator.name = it.first; + creator.url = it.second; result_tabs.push_back(Tab{create_body(), std::make_unique(program, this, std::move(creator)), create_search_bar("Search...", SEARCH_DELAY_FILTER)}); } @@ -443,8 +452,12 @@ namespace QuickMedia { next_page_userdata.field_name = list_page_pagination_query->next_page_field_name; next_page_userdata.field_contains = list_page_pagination_query->next_page_field_contains; + std::vector args; + if(!website_url.empty()) + args.push_back({ "-H", "referer: " + website_url }); + std::string website_data; - if(download_to_string(url, website_data, {}, true, fail_on_http_error) != DownloadResult::OK) + if(download_to_string(url, website_data, args, true, fail_on_http_error) != DownloadResult::OK) return ImageResult::NET_ERR; QuickMediaHtmlSearch html_search; @@ -547,12 +560,16 @@ namespace QuickMedia { next_page_userdata.field_name = list_page_pagination_query->next_page_field_name; next_page_userdata.field_contains = list_page_pagination_query->next_page_field_contains; + std::vector args; + if(!website_url.empty()) + args.push_back({ "-H", "referer: " + website_url }); + std::string image_src; std::string website_data; DownloadErrorHandler error_callback = [](std::string&){ return true; }; if(fail_on_http_error) error_callback = nullptr; - if(download_to_string_cache(full_url, website_data, {}, true, error_callback) != DownloadResult::OK) + if(download_to_string_cache(full_url, website_data, args, true, error_callback) != DownloadResult::OK) return ImageResult::ERR; QuickMediaHtmlSearch html_search; @@ -594,8 +611,12 @@ namespace QuickMedia { if(!chapter_image_urls.empty()) return ImageResult::OK; + std::vector args; + if(!website_url.empty()) + args.push_back({ "-H", "referer: " + website_url }); + std::string website_data; - if(download_to_string(url, website_data, {}, true, fail_on_http_error) != DownloadResult::OK) + if(download_to_string(url, website_data, args, true, fail_on_http_error) != DownloadResult::OK) return ImageResult::NET_ERR; if(list_page_query->type == ListPageQueryType::IMAGES) { @@ -678,11 +699,8 @@ namespace QuickMedia { return *this; } - MangaGenericSearchPage& MangaGenericSearchPage::authors_handler(const char *html_query, const char *title_field, const char *url_field, const char *url_contains) { - authors_query.html_query = html_query; - authors_query.title_field = title_field; - authors_query.url_field = url_field; - authors_query.url_contains = url_contains; + MangaGenericSearchPage& MangaGenericSearchPage::authors_handler(std::vector queries) { + authors_queries = std::move(queries); return *this; } -- cgit v1.2.3