#include "../../plugins/NyaaSi.hpp" #include "../../include/Program.hpp" #include "../../include/Storage.hpp" #include "../../include/Notification.hpp" #include "../../include/StringUtils.hpp" #include "../../include/NetUtils.hpp" #include "../../include/Utils.hpp" #include namespace QuickMedia { // Return end of td tag, or std::string::npos static size_t find_td_get_value(const std::string &str, size_t start_index, size_t end_index, std::string &result) { size_t td_begin = str.find("= end_index) return std::string::npos; size_t td_end = str.find("", td_begin + 3); if(td_end == std::string::npos || td_end >= end_index) return std::string::npos; size_t value_begin = str.find('>', td_begin + 3); if(value_begin == std::string::npos || value_begin >= td_end) return std::string::npos; result = str.substr(value_begin + 1, td_end - (value_begin + 1)); return td_end + 5; } static bool is_only_numbers(const char *str, size_t size) { for(size_t i = 0; i < size; ++i) { if(str[i] < '0' || str[i] > '9') return false; } return true; } static std::shared_ptr create_front_page_item(const std::string &title, const std::string &category) { auto body_item = BodyItem::create(title); body_item->url = category; return body_item; } void get_nyaa_si_categories(BodyItems &result_items) { result_items.push_back(create_front_page_item("All categories", "0_0")); result_items.push_back(create_front_page_item("Anime", "1_0")); result_items.push_back(create_front_page_item(" Anime - Music video", "1_1")); result_items.push_back(create_front_page_item(" Anime - English translated", "1_2")); result_items.push_back(create_front_page_item(" Anime - Non-english translated", "1_3")); result_items.push_back(create_front_page_item(" Anime - Raw", "1_4")); result_items.push_back(create_front_page_item("Audio", "2_0")); result_items.push_back(create_front_page_item(" Audio - Lossless", "2_1")); result_items.push_back(create_front_page_item(" Anime - Lossy", "2_2")); result_items.push_back(create_front_page_item("Literature", "3_0")); result_items.push_back(create_front_page_item(" Literature - English translated", "3_1")); result_items.push_back(create_front_page_item(" Literature - Non-english translated", "3_1")); result_items.push_back(create_front_page_item(" Literature - Raw", "3_3")); result_items.push_back(create_front_page_item("Live Action", "4_0")); result_items.push_back(create_front_page_item(" Live Action - English translated", "4_1")); result_items.push_back(create_front_page_item(" Live Action - Non-english translated", "4_3")); result_items.push_back(create_front_page_item(" Live Action - Idol/Promotional video", "4_2")); result_items.push_back(create_front_page_item(" Live Action - Raw", "4_4")); result_items.push_back(create_front_page_item("Pictures", "5_0")); result_items.push_back(create_front_page_item(" Pictures - Graphics", "5_1")); result_items.push_back(create_front_page_item(" Pictures - Photos", "5_2")); result_items.push_back(create_front_page_item("Software", "6_0")); result_items.push_back(create_front_page_item(" Software - Applications", "6_1")); result_items.push_back(create_front_page_item(" Software - Games", "6_2")); } void get_sukebei_categories(BodyItems &result_items) { result_items.push_back(create_front_page_item("All categories", "0_0")); result_items.push_back(create_front_page_item("Art", "1_0")); result_items.push_back(create_front_page_item(" Anime", "1_1")); result_items.push_back(create_front_page_item(" Doujinshi", "1_2")); result_items.push_back(create_front_page_item(" Games", "1_3")); result_items.push_back(create_front_page_item(" Manga", "1_4")); result_items.push_back(create_front_page_item(" Pictures", "1_5")); result_items.push_back(create_front_page_item("Real Life", "2_0")); result_items.push_back(create_front_page_item(" Photobooks and Pictures", "2_1")); result_items.push_back(create_front_page_item(" Videos", "2_2")); } static time_t nyaa_si_time_to_unix_time(const char *time_str) { int year = 0; int month = 0; int day = 0; int hour = 0; int minute = 0; sscanf(time_str, "%d-%d-%d %d:%d", &year, &month, &day, &hour, &minute); if(year == 0) return 0; struct tm time; memset(&time, 0, sizeof(time)); time.tm_year = year - 1900; time.tm_mon = month - 1; time.tm_mday = day; time.tm_hour = hour; time.tm_min = minute; time.tm_sec = 0; return timegm(&time); } static const std::array sort_type_names = { "Size desc 🡇", "Uploaded date desc 🡇", "Seeders desc 🡇", "Leechers desc 🡇", "Downloads desc 🡇", "Size asc 🡅", "Uploaded date asc 🡅", "Seeders asc 🡅", "Leechers asc 🡅", "Downloads asc 🡅" }; static const std::array sort_params = { "s=size&o=desc", "s=id&o=desc", "s=seeders&o=desc", "s=leechers&o=desc", "s=downloads&o=desc", "s=size&o=asc", "s=id&o=asc", "s=seeders&o=asc", "s=leechers&o=asc", "s=downloads&o=asc", }; static std::shared_ptr create_sort_body_item(const std::string &title, NyaaSiSortType sort_type) { auto body_item = BodyItem::create(title); body_item->userdata = (void*)sort_type; return body_item; } static void sort_page_create_body_items(Body *body, NyaaSiSortType sort_type) { for(size_t i = 0; i < sort_type_names.size(); ++i) { std::string prefix = " "; if((NyaaSiSortType)i == sort_type) prefix = "* "; body->append_item(create_sort_body_item(prefix + sort_type_names[i], (NyaaSiSortType)i)); } } // TODO: Also show the number of comments for each torrent. TODO: Optimize? // TODO: Show each field as seperate columns instead of seperating by | static SearchResult search_page(const std::string &domain, const std::string &list_url, const std::string &text, int page, NyaaSiSortType sort_type, BodyItems &result_items) { std::string full_url = "https://" + domain + "/?c=" + list_url + "&f=0&p=" + std::to_string(page) + "&q="; full_url += url_param_encode(text); full_url += std::string("&") + sort_params[(size_t)sort_type]; std::string website_data; if(download_to_string(full_url, website_data, {}, true) != DownloadResult::OK) return SearchResult::NET_ERR; const bool is_sukebei = (domain == "sukebei.nyaa.si"); size_t tbody_begin = website_data.find(""); if(tbody_begin == std::string::npos) return SearchResult::OK; size_t tbody_end = website_data.find("", tbody_begin + 7); if(tbody_end == std::string::npos) return SearchResult::ERR; size_t index = tbody_begin + 7; while(index < tbody_end) { size_t tr_begin = website_data.find("= tbody_end) break; size_t tr_end = website_data.find("", tr_begin + 3); if(tr_end == std::string::npos || tr_end >= tbody_end) return SearchResult::ERR; index = tr_begin + 3; bool is_trusted = false; bool is_remake = false; size_t tr_class_begin = website_data.find("class=\"", index); if(tr_class_begin != std::string::npos && tr_class_begin < tr_end) { size_t tr_class_end = website_data.find('"', tr_class_begin + 7); size_t class_length = tr_class_end - (tr_class_begin + 7); if(strncmp(website_data.c_str() + tr_class_begin + 7, "success", class_length) == 0) is_trusted = true; else if(strncmp(website_data.c_str() + tr_class_begin + 7, "danger", class_length) == 0) is_remake = true; index = tr_class_end + 1; } size_t category_begin = website_data.find("/?c=", index); if(category_begin == std::string::npos || category_begin >= tr_end) return SearchResult::ERR; size_t category_end = website_data.find('"', category_begin + 4); if(category_end == std::string::npos || category_end >= tr_end) return SearchResult::ERR; index = category_end + 1; size_t view_begin = website_data.find("/view/", index); if(view_begin == std::string::npos || view_begin >= tr_end) return SearchResult::ERR; size_t view_end = website_data.find('"', view_begin + 6); if(view_end == std::string::npos || view_end >= tr_end) return SearchResult::ERR; std::string view_url = website_data.substr(view_begin, view_end - view_begin); // Torrents with comments have two /view/, one for comments and one for the title if(!is_only_numbers(website_data.c_str() + view_begin + 6, view_end - (view_begin + 6))) { size_t view_begin2 = website_data.find("/view/", view_end + 1); if(view_begin2 == std::string::npos || view_begin2 >= tr_end) return SearchResult::ERR; size_t view_end2 = website_data.find('"', view_begin2 + 6); if(view_end2 == std::string::npos || view_end2 >= tr_end) return SearchResult::ERR; view_end = view_end2; } size_t title_begin = website_data.find('>', view_end + 1); if(title_begin == std::string::npos || title_begin >= tr_end) return SearchResult::ERR; size_t title_end = website_data.find("", title_begin + 1); if(title_end == std::string::npos || title_end >= tr_end) return SearchResult::ERR; std::string title = website_data.substr(title_begin + 1, title_end - (title_begin + 1)); html_unescape_sequences(title); title = strip(title); index = title_end + 4; size_t magnet_begin = website_data.find("magnet:?xt", index); if(magnet_begin == std::string::npos || magnet_begin >= tr_end) return SearchResult::ERR; size_t magnet_end = website_data.find('"', magnet_begin + 10); if(magnet_end == std::string::npos || magnet_end >= tr_end) return SearchResult::ERR; index = magnet_end + 1; std::string size; index = find_td_get_value(website_data, index, tr_end, size); if(index == std::string::npos) return SearchResult::ERR; std::string timestamp; index = find_td_get_value(website_data, index, tr_end, timestamp); if(index == std::string::npos) return SearchResult::ERR; std::string seeders; index = find_td_get_value(website_data, index, tr_end, seeders); if(index == std::string::npos) return SearchResult::ERR; std::string leechers; index = find_td_get_value(website_data, index, tr_end, leechers); if(index == std::string::npos) return SearchResult::ERR; std::string completed; index = find_td_get_value(website_data, index, tr_end, completed); if(index == std::string::npos) return SearchResult::ERR; index = tr_end + 5; std::string description = "Size: " + size + " | Published: " + unix_time_to_local_time_str(nyaa_si_time_to_unix_time(timestamp.c_str())) + " | Seeders: " + seeders + " | Leechers: " + leechers + " | Completed: " + completed; auto body_item = BodyItem::create(std::move(title)); body_item->thumbnail_url = "https://" + domain + "/static/img/icons/" + (is_sukebei ? "sukebei" : "nyaa") + "/" + website_data.substr(category_begin + 4, category_end - (category_begin + 4)) + ".png"; body_item->set_description(std::move(description)); body_item->url = "https://" + domain + std::move(view_url); if(is_trusted) body_item->set_title_color(mgl::Color(43, 255, 47)); else if(is_remake) body_item->set_title_color(mgl::Color(255, 45, 47)); body_item->thumbnail_size = mgl::vec2i(80, 28); result_items.push_back(std::move(body_item)); } return SearchResult::OK; } PluginResult NyaaSiCategoryPage::submit(const SubmitArgs &args, std::vector &result_tabs) { std::string domain = is_sukebei ? "sukebei.nyaa.si" : "nyaa.si"; BodyItems result_items; SearchResult search_result = search_page(domain, args.url, "", 1, NyaaSiSortType::UPLOAD_DATE_DESC, result_items); if(search_result != SearchResult::OK) return search_result_to_plugin_result(search_result); auto search_page = std::make_unique(program, strip(args.title), args.url, std::move(domain)); NyaaSiSearchPage *search_page_p = search_page.get(); auto body = create_body(); body->set_items(std::move(result_items)); result_tabs.push_back(Tab{std::move(body), std::move(search_page), create_search_bar("Search...", 500)}); auto sort_order_page_body = create_body(); Body *sort_order_page_body_p = sort_order_page_body.get(); sort_page_create_body_items(sort_order_page_body_p, NyaaSiSortType::UPLOAD_DATE_DESC); result_tabs.push_back(Tab{std::move(sort_order_page_body), std::make_unique(program, sort_order_page_body_p, search_page_p), nullptr}); return PluginResult::OK; } NyaaSiSearchPage::NyaaSiSearchPage(Program *program, std::string category_name, std::string category_id, std::string domain) : Page(program), category_name(std::move(category_name)), category_id(std::move(category_id)), domain(std::move(domain)) { set_sort_type(NyaaSiSortType::UPLOAD_DATE_DESC); } SearchResult NyaaSiSearchPage::search(const std::string &str, BodyItems &result_items) { return search_page(domain, category_id, str, 1, sort_type, result_items); } PluginResult NyaaSiSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) { return search_result_to_plugin_result(search_page(domain, category_id, str, 1 + page, sort_type, result_items)); } struct ResultItemExtra { BodyItems *result_items; const std::string *domain; }; PluginResult NyaaSiSearchPage::submit(const SubmitArgs &args, std::vector &result_tabs) { size_t comments_start_index; std::string title; BodyItems result_items; auto torrent_item = BodyItem::create("💾 Download"); std::string magnet_url; std::string description; ResultItemExtra result_item_extra; result_item_extra.result_items = &result_items; result_item_extra.domain = &domain; std::string website_data; if(download_to_string(args.url, website_data, {}, true) != DownloadResult::OK) return PluginResult::NET_ERR; QuickMediaHtmlSearch html_search; int result = quickmedia_html_search_init(&html_search, website_data.c_str(), website_data.size()); if(result != 0) goto cleanup; result = quickmedia_html_find_nodes_xpath(&html_search, "//h3[class='panel-title']", [](QuickMediaMatchNode *node, void *userdata) { std::string *title = (std::string*)userdata; QuickMediaStringView text = quickmedia_html_node_get_text(node); if(title->empty() && text.data) { title->assign(text.data, text.size); } return 0; }, &title); if(result != 0) goto cleanup; if(title.empty()) { fprintf(stderr, "Error: nyaa.si: failed to get title\n"); result = -1; goto cleanup; } result = quickmedia_html_find_nodes_xpath(&html_search, "//div[class='panel-body']//div[class='row']//a", [](QuickMediaMatchNode *node, void *userdata) { ResultItemExtra *item_data = (ResultItemExtra*)userdata; QuickMediaStringView href = quickmedia_html_node_get_attribute_value(node, "href"); QuickMediaStringView text = quickmedia_html_node_get_text(node); if(item_data->result_items->empty() && href.data && text.data && href.size >= 6 && memcmp(href.data, "/user/", 6) == 0) { auto body_item = BodyItem::create(""); body_item->set_description("Submitter: " + std::string(text.data, text.size)); body_item->url = "https://" + *item_data->domain + "/" + std::string(href.data, href.size); item_data->result_items->push_back(std::move(body_item)); } return 0; }, &result_item_extra); if(result != 0) goto cleanup; if(result_items.empty()) { auto body_item = BodyItem::create(""); body_item->set_description("Submitter: Anonymous"); result_items.push_back(std::move(body_item)); } result_items.front()->set_title(title); result = quickmedia_html_find_nodes_xpath(&html_search, "//div[id='torrent-description']", [](QuickMediaMatchNode *node, void *userdata) { std::string *description = (std::string*)userdata; QuickMediaStringView text = quickmedia_html_node_get_text(node); if(description->empty() && text.data) { std::string desc(text.data, text.size); html_unescape_sequences(desc); *description = std::move(desc); } return 0; }, &description); if(result != 0) goto cleanup; if(!description.empty()) result_items.front()->set_description(result_items.front()->get_description() + "\nDescription:\n" + description); result = quickmedia_html_find_nodes_xpath(&html_search, "//div[class='container']//a", [](QuickMediaMatchNode *node, void *userdata) { std::string *magnet_url = (std::string*)userdata; QuickMediaStringView href = quickmedia_html_node_get_attribute_value(node, "href"); if(magnet_url->empty() && href.data && href.size >= 8 && memcmp(href.data, "magnet:?", 8) == 0) { magnet_url->assign(href.data, href.size); } return 0; }, &magnet_url); if(result != 0) goto cleanup; if(magnet_url.empty()) { fprintf(stderr, "Error: %s: failed to get magnet link\n", domain.c_str()); result = -1; goto cleanup; } html_unescape_sequences(magnet_url); torrent_item->url = std::move(magnet_url); result_items.push_back(std::move(torrent_item)); comments_start_index = result_items.size(); result = quickmedia_html_find_nodes_xpath(&html_search, "//div[id='comments']//a", [](QuickMediaMatchNode *node, void *userdata) { auto *item_data = (BodyItems*)userdata; QuickMediaStringView href = quickmedia_html_node_get_attribute_value(node, "href"); QuickMediaStringView text = quickmedia_html_node_get_text(node); if(href.data && text.data && href.size >= 6 && memcmp(href.data, "/user/", 6) == 0) { auto body_item = BodyItem::create(std::string(text.data, text.size)); //body_item->url = "https://nyaa.si/" + std::string(href); item_data->push_back(std::move(body_item)); } return 0; }, &result_items); if(result != 0 || result_items.size() == comments_start_index) goto cleanup; BodyItemContext body_item_image_context; body_item_image_context.body_items = &result_items; body_item_image_context.index = comments_start_index; result = quickmedia_html_find_nodes_xpath(&html_search, "//div[id='comments']//img[class='avatar']", [](QuickMediaMatchNode *node, void *userdata) { auto *item_data = (BodyItemContext*)userdata; QuickMediaStringView src = quickmedia_html_node_get_attribute_value(node, "src"); if(src.data && item_data->index < item_data->body_items->size()) { (*item_data->body_items)[item_data->index]->thumbnail_url.assign(src.data, src.size); (*item_data->body_items)[item_data->index]->thumbnail_size = mgl::vec2i(120, 120); item_data->index++; } return 0; }, &body_item_image_context); if(result != 0) goto cleanup; body_item_image_context.index = comments_start_index; result = quickmedia_html_find_nodes_xpath(&html_search, "//div[id='comments']//div[class='comment-content']", [](QuickMediaMatchNode *node, void *userdata) { auto *item_data = (BodyItemContext*)userdata; QuickMediaStringView text = quickmedia_html_node_get_text(node); if(text.data && item_data->index < item_data->body_items->size()) { std::string desc(text.data, text.size); html_unescape_sequences(desc); (*item_data->body_items)[item_data->index]->set_description(std::move(desc)); item_data->index++; } return 0; }, &body_item_image_context); cleanup: quickmedia_html_search_deinit(&html_search); if(result != 0) return PluginResult::ERR; auto body = create_body(); body->set_items(std::move(result_items)); result_tabs.push_back(Tab{std::move(body), std::make_unique(program), nullptr}); return PluginResult::OK; } void NyaaSiSearchPage::set_sort_type(NyaaSiSortType sort_type) { this->sort_type = sort_type; title = category_name + " | " + sort_type_names[(size_t)sort_type]; title.erase(title.end() - 5, title.end()); // Erase emoji character and space at the end. TODO: Remove this when tabs support emojis. } PluginResult NyaaSiSortOrderPage::submit(const SubmitArgs &args, std::vector&) { const NyaaSiSortType sort_type = (NyaaSiSortType)(size_t)args.userdata; body->clear_items(); sort_page_create_body_items(body, sort_type); search_page->set_sort_type(sort_type); search_page->needs_refresh = true; return PluginResult::OK; } PluginResult NyaaSiTorrentPage::submit(const SubmitArgs &submit_args, std::vector &result_tabs) { (void)result_tabs; if(strncmp(submit_args.url.c_str(), "magnet:?", 8) == 0) { if(!is_program_executable_by_name("xdg-open")) { show_notification("QuickMedia", "xdg-utils which provides xdg-open needs to be installed to download torrents", Urgency::CRITICAL); return PluginResult::ERR; } const char *args[] = { "xdg-open", submit_args.url.c_str(), nullptr }; exec_program_async(args, nullptr); } return PluginResult::OK; } }