From 76e1aebbe075287a8297194b38343467c76dd964 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Mon, 26 Apr 2021 18:37:00 +0200 Subject: Fix soundcloud (fetch client id), add authors to mangakatana, some other fixes --- src/plugins/MangaGeneric.cpp | 159 ++++++++++++++++++++++++++++--------------- src/plugins/Matrix.cpp | 33 +++++---- src/plugins/Soundcloud.cpp | 68 +++++++++++++++++- 3 files changed, 192 insertions(+), 68 deletions(-) (limited to 'src/plugins') diff --git a/src/plugins/MangaGeneric.cpp b/src/plugins/MangaGeneric.cpp index 24472e5..58bedf4 100644 --- a/src/plugins/MangaGeneric.cpp +++ b/src/plugins/MangaGeneric.cpp @@ -49,11 +49,6 @@ namespace QuickMedia { return quickmedia_html_node_get_attribute_value(node, field_name); } - static bool starts_with(const std::string &str, const char *sub) { - size_t sub_len = strlen(sub); - return str.size() >= sub_len && memcmp(str.c_str(), sub, sub_len) == 0; - } - static int html_append_search(QuickMediaHtmlSearch *html_search, const char *html_query, HtmlSearchUserdata *search_userdata) { return quickmedia_html_find_nodes_xpath(html_search, html_query, [](QuickMediaHtmlNode *node, void *userdata) { @@ -149,12 +144,8 @@ namespace QuickMedia { return plugin_result_to_search_result(get_page(str, 0, result_items)); } - PluginResult MangaGenericSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) { + PluginResult MangaGenericSearchPage::get_page(const std::string &url, BodyItems &result_items) { std::string target_url; - std::string url = search_query.search_template; - string_replace_all(url, "%s", url_param_encode(str)); - string_replace_all(url, "%p", std::to_string(search_query.page_start + page)); - std::string website_data; if(download_to_string(url, website_data, {}, true, fail_on_http_error) != DownloadResult::OK) return PluginResult::NET_ERR; @@ -184,36 +175,6 @@ namespace QuickMedia { if(result != 0) goto cleanup; - for(const DescriptionQuery &description_query : description_queries) { - assert(description_query.html_query && description_query.field_name); - if(description_query.html_query && description_query.field_name) { - HtmlMergeUserdata merge_userdata; - merge_userdata.type = MergeType::DESCRIPTION; - merge_userdata.body_item_image_context.body_items = &new_result_items; - merge_userdata.body_item_image_context.index = 0; - merge_userdata.field_name = description_query.field_name; - merge_userdata.field_contains = nullptr; - result = html_body_item_merge(&html_search, description_query.html_query, &merge_userdata); - if(result != 0) - goto cleanup; - } - } - - for(const ThumbnailQuery &thumbnail_query : thumbnail_queries) { - assert(thumbnail_query.html_query && thumbnail_query.field_name); - if(thumbnail_query.html_query && thumbnail_query.field_name) { - HtmlMergeUserdata merge_userdata; - merge_userdata.type = MergeType::THUMBNAIL; - merge_userdata.body_item_image_context.body_items = &new_result_items; - merge_userdata.body_item_image_context.index = 0; - merge_userdata.field_name = thumbnail_query.field_name; - merge_userdata.field_contains = thumbnail_query.field_contains; - result = html_body_item_merge(&html_search, thumbnail_query.html_query, &merge_userdata); - if(result != 0) - goto cleanup; - } - } - if(!text_query.url_field && !new_result_items.empty()) { if(target_url.empty()) { std::string response_headers; @@ -239,15 +200,45 @@ namespace QuickMedia { result_items.insert(result_items.end(), std::move_iterator(new_result_items.begin()), std::move_iterator(new_result_items.end())); } + for(const DescriptionQuery &description_query : description_queries) { + assert(description_query.html_query && description_query.field_name); + if(description_query.html_query && description_query.field_name) { + HtmlMergeUserdata merge_userdata; + merge_userdata.type = MergeType::DESCRIPTION; + merge_userdata.body_item_image_context.body_items = &result_items; + merge_userdata.body_item_image_context.index = 0; + merge_userdata.field_name = description_query.field_name; + merge_userdata.field_contains = nullptr; + result = html_body_item_merge(&html_search, description_query.html_query, &merge_userdata); + if(result != 0) + goto cleanup; + } + } + + for(const ThumbnailQuery &thumbnail_query : thumbnail_queries) { + assert(thumbnail_query.html_query && thumbnail_query.field_name); + if(thumbnail_query.html_query && thumbnail_query.field_name) { + HtmlMergeUserdata merge_userdata; + merge_userdata.type = MergeType::THUMBNAIL; + merge_userdata.body_item_image_context.body_items = &result_items; + merge_userdata.body_item_image_context.index = 0; + merge_userdata.field_name = thumbnail_query.field_name; + merge_userdata.field_contains = thumbnail_query.field_contains; + result = html_body_item_merge(&html_search, thumbnail_query.html_query, &merge_userdata); + if(result != 0) + goto cleanup; + } + } + for(auto &body_item : result_items) { - if(starts_with(body_item->url, "//")) + if(string_starts_with(body_item->url, "//")) body_item->url = "https://" + body_item->url.substr(2); - else if(starts_with(body_item->url, "/")) + else if(string_starts_with(body_item->url, "/")) body_item->url = website_url + body_item->url.substr(1); - if(starts_with(body_item->thumbnail_url, "//")) + if(string_starts_with(body_item->thumbnail_url, "//")) body_item->thumbnail_url = "https://" + body_item->thumbnail_url.substr(2); - else if(starts_with(body_item->thumbnail_url, "/")) + else if(string_starts_with(body_item->thumbnail_url, "/")) body_item->thumbnail_url = website_url + body_item->thumbnail_url.substr(1); } @@ -261,12 +252,20 @@ namespace QuickMedia { } } + PluginResult MangaGenericSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) { + std::string url = search_query.search_template; + string_replace_all(url, "%s", url_param_encode(str)); + string_replace_all(url, "%p", std::to_string(search_query.page_start + page)); + return get_page(url, result_items); + } + PluginResult MangaGenericSearchPage::submit(const std::string &title, const std::string &url, std::vector &result_tabs) { if(!list_chapters_query.html_query || !list_chapters_query.title_field || !list_chapters_query.url_field) { assert(false); return PluginResult::ERR; } + std::vector creators; BodyItems chapters_items; HtmlSearchUserdata search_userdata; search_userdata.body_items = &chapters_items; @@ -299,18 +298,49 @@ namespace QuickMedia { result = html_body_item_merge(&html_search, list_chapters_query.uploaded_time_html_query, &merge_userdata); } + if(authors_query.html_query && authors_query.title_field && authors_query.url_field) { + struct HtmlAuthorsUserdata { + std::vector *creators; + AuthorsQuery *authors_query; + }; + + HtmlAuthorsUserdata authors_userdata; + authors_userdata.creators = &creators; + authors_userdata.authors_query = &authors_query; + + quickmedia_html_find_nodes_xpath(&html_search, authors_query.html_query, + [](QuickMediaHtmlNode *node, void *userdata) { + HtmlAuthorsUserdata *authors_userdata = (HtmlAuthorsUserdata*)userdata; + const char *title_value = html_attr_or_inner_text(node, authors_userdata->authors_query->title_field); + const char *url_value = html_attr_or_inner_text(node, authors_userdata->authors_query->url_field); + if(title_value && url_value && (!authors_userdata->authors_query->url_contains || strstr(url_value, authors_userdata->authors_query->url_contains))) { + Creator creator; + creator.name = strip(title_value); + creator.url = strip(url_value); + authors_userdata->creators->push_back(std::move(creator)); + } + }, &authors_userdata); + } + for(auto &body_item : chapters_items) { - if(starts_with(body_item->url, "//")) + if(string_starts_with(body_item->url, "//")) body_item->url = "https://" + body_item->url.substr(2); - else if(starts_with(body_item->url, "/")) + else if(string_starts_with(body_item->url, "/")) body_item->url = website_url + body_item->url.substr(1); - if(starts_with(body_item->thumbnail_url, "//")) + if(string_starts_with(body_item->thumbnail_url, "//")) body_item->thumbnail_url = "https://" + body_item->thumbnail_url.substr(2); - else if(starts_with(body_item->thumbnail_url, "/")) + else if(string_starts_with(body_item->thumbnail_url, "/")) body_item->thumbnail_url = website_url + body_item->thumbnail_url.substr(1); } + for(auto &creator : creators) { + if(string_starts_with(creator.url, "//")) + creator.url = "https://" + creator.url.substr(2); + else if(string_starts_with(creator.url, "/")) + creator.url = website_url + creator.url.substr(1); + } + cleanup: quickmedia_html_search_deinit(&html_search); if(result != 0) @@ -319,6 +349,11 @@ namespace QuickMedia { auto body = create_body(); body->items = std::move(chapters_items); result_tabs.push_back(Tab{std::move(body), std::make_unique(program, title, url, manga_id_extractor, service_name, website_url, &list_page_query, fail_on_http_error), create_search_bar("Search...", SEARCH_DELAY_FILTER)}); + + for(Creator &creator : creators) { + result_tabs.push_back(Tab{create_body(), std::make_unique(program, this, std::move(creator)), create_search_bar("Search...", SEARCH_DELAY_FILTER)}); + } + return PluginResult::OK; } @@ -348,6 +383,14 @@ namespace QuickMedia { return true; } + PluginResult MangaGenericCreatorPage::submit(const std::string &title, const std::string &url, std::vector &result_tabs) { + return search_page->submit(title, url, result_tabs); + } + + PluginResult MangaGenericCreatorPage::lazy_fetch(BodyItems &result_items) { + return search_page->get_page(creator.url, result_items); + } + static bool is_number(const char *str) { while(*str) { char c = *str; @@ -443,9 +486,9 @@ namespace QuickMedia { return ImageResult::ERR; } - if(starts_with(current_image_url, "//")) + if(string_starts_with(current_image_url, "//")) current_image_url = "https://" + current_image_url.substr(2); - else if(starts_with(current_image_url, "/")) + else if(string_starts_with(current_image_url, "/")) current_image_url = website_url + current_image_url.substr(1); num_images = page_count_userdata.num_pages; @@ -523,9 +566,9 @@ namespace QuickMedia { cleanup: quickmedia_html_search_deinit(&html_search); - if(starts_with(current_image_url, "//")) + if(string_starts_with(current_image_url, "//")) current_image_url = "https://" + current_image_url.substr(2); - else if(starts_with(current_image_url, "/")) + else if(string_starts_with(current_image_url, "/")) current_image_url = website_url + current_image_url.substr(1); if(!callback(current_image_url)) @@ -603,9 +646,9 @@ namespace QuickMedia { } for(std::string &url : chapter_image_urls) { - if(starts_with(url, "//")) + if(string_starts_with(url, "//")) url = "https://" + url.substr(2); - else if(starts_with(url, "/")) + else if(string_starts_with(url, "/")) url = website_url + url.substr(1); } @@ -635,6 +678,14 @@ namespace QuickMedia { return *this; } + MangaGenericSearchPage& MangaGenericSearchPage::authors_handler(const char *html_query, const char *title_field, const char *url_field, const char *url_contains) { + authors_query.html_query = html_query; + authors_query.title_field = title_field; + authors_query.url_field = url_field; + authors_query.url_contains = url_contains; + return *this; + } + MangaGenericSearchPage& MangaGenericSearchPage::list_chapters_handler(const char *html_query, const char *title_field, const char *url_field, const char *url_contains) { list_chapters_query.html_query = html_query; list_chapters_query.title_field = title_field; diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp index 57c879e..0c06458 100644 --- a/src/plugins/Matrix.cpp +++ b/src/plugins/Matrix.cpp @@ -489,14 +489,15 @@ namespace QuickMedia { } static std::string message_to_room_description_text(Message *message) { + std::string body = strip(message->body); if(message->type == MessageType::REACTION) - return "Reacted with: " + extract_first_line_elipses(message->body, 150); + return "Reacted with: " + extract_first_line_elipses(body, 150); else if(message->related_event_type == RelatedEventType::REPLY) - return extract_first_line_elipses(remove_reply_formatting(message->body), 150); + return extract_first_line_elipses(remove_reply_formatting(body), 150); else if(message->related_event_type == RelatedEventType::EDIT) - return "Edited: " + extract_first_line_elipses(remove_reply_formatting(message->body), 150); + return "Edited: " + extract_first_line_elipses(remove_reply_formatting(body), 150); else - return extract_first_line_elipses(message->body, 150); + return extract_first_line_elipses(body, 150); } void MatrixQuickMedia::update_room_description(RoomData *room, Messages &new_messages, bool is_initial_sync, bool sync_is_cache, Body *chat_body, bool messages_tab_visible) { @@ -1090,18 +1091,20 @@ namespace QuickMedia { } } - void Matrix::start_sync(MatrixDelegate *delegate, bool &cached) { + bool Matrix::start_sync(MatrixDelegate *delegate, bool &cached) { cached = true; if(sync_running) - return; + return true; assert(!this->delegate); assert(!access_token.empty()); // Need to be logged in this->delegate = delegate; Path matrix_cache_dir = get_cache_dir().join("matrix"); - if(create_directory_recursive(matrix_cache_dir) != 0) + if(create_directory_recursive(matrix_cache_dir) != 0) { fprintf(stderr, "Failed to create matrix cache directory\n"); + return false; + } matrix_cache_dir.join("sync_data.json"); cached = (get_file_type(matrix_cache_dir) == FileType::REGULAR); @@ -1196,6 +1199,7 @@ namespace QuickMedia { result = parse_sync_response(json_root, false, initial_sync); if(result != PluginResult::OK) { fprintf(stderr, "Failed to parse sync response\n"); + initial_sync = false; goto sync_end; } @@ -1206,6 +1210,7 @@ namespace QuickMedia { } else { //set_next_batch("Invalid"); fprintf(stderr, "Matrix: missing next batch\n"); + initial_sync = false; goto sync_end; } @@ -1288,6 +1293,8 @@ namespace QuickMedia { std::this_thread::sleep_for(std::chrono::milliseconds(500)); } }); + + return true; } void Matrix::stop_sync() { @@ -1313,6 +1320,10 @@ namespace QuickMedia { delegate = nullptr; sync_failed = false; sync_fail_reason.clear(); + set_next_batch(""); + invites.clear(); + filter_cached.reset(); + my_events_transaction_ids.clear(); } bool Matrix::is_initial_sync_finished() const { @@ -3382,7 +3393,7 @@ namespace QuickMedia { } PluginResult Matrix::logout() { - assert(!sync_running); + stop_sync(); Path session_path = get_storage_dir().join(SERVICE_NAME).join("session.json"); remove(session_path.data.c_str()); @@ -3402,10 +3413,6 @@ namespace QuickMedia { homeserver.clear(); homeserver_domain.clear(); upload_limit.reset(); - set_next_batch(""); - invites.clear(); - filter_cached.reset(); - my_events_transaction_ids.clear(); return PluginResult::OK; } @@ -3950,7 +3957,7 @@ namespace QuickMedia { // TODO: Use at param? which is room->get_prev_batch(); char url[512]; - snprintf(url, sizeof(url), "%s/_matrix/client/r0/rooms/%s/members?membership=join", homeserver.c_str(), room->id.c_str()); + snprintf(url, sizeof(url), "%s/_matrix/client/r0/rooms/%s/members?not_membership=leave", homeserver.c_str(), room->id.c_str()); rapidjson::Document json_root; DownloadResult download_result = download_json(json_root, url, std::move(additional_args), true); diff --git a/src/plugins/Soundcloud.cpp b/src/plugins/Soundcloud.cpp index a0184b4..6740504 100644 --- a/src/plugins/Soundcloud.cpp +++ b/src/plugins/Soundcloud.cpp @@ -2,9 +2,11 @@ #include "../../include/NetUtils.hpp" #include "../../include/StringUtils.hpp" #include "../../include/Scale.hpp" +#include +#include namespace QuickMedia { - static std::string client_id = "Na04L87fnpWDMVCCW2ngWldN4JMoLTAc"; + static std::string client_id; class SoundcloudPlaylist : public BodyItemExtra { public: @@ -218,6 +220,11 @@ namespace QuickMedia { return PluginResult::OK; } + PluginResult SoundcloudSearchPage::submit(const std::string &title, const std::string &url, std::vector &result_tabs) { + submit_page.submit_body_item = submit_body_item; + return submit_page.submit(title, url, result_tabs); + } + SearchResult SoundcloudSearchPage::search(const std::string &str, BodyItems &result_items) { query_urn.clear(); PluginResult result = get_page(str, 0, result_items); @@ -268,6 +275,65 @@ namespace QuickMedia { return PluginResult::OK; } + PluginResult SoundcloudSearchPage::lazy_fetch(BodyItems&) { + std::string website_data; + DownloadResult download_result = download_to_string("https://soundcloud.com/", website_data, {}, true); + if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result); + + if(client_id.empty()) { + std::vector script_sources; + QuickMediaHtmlSearch html_search; + int result = quickmedia_html_search_init(&html_search, website_data.c_str()); + if(result != 0) + goto cleanup; + + result = quickmedia_html_find_nodes_xpath(&html_search, "//script", + [](QuickMediaHtmlNode *node, void *userdata) { + std::vector *script_sources = (std::vector*)userdata; + const char *src = quickmedia_html_node_get_attribute_value(node, "src"); + if(src) + script_sources->push_back(strip(src)); + }, &script_sources); + + cleanup: + quickmedia_html_search_deinit(&html_search); + if(result != 0) + return PluginResult::ERR; + + std::vector> async_download_threads; + for(std::string &script_source : script_sources) { + if(string_starts_with(script_source, "//")) + script_source = "https://" + script_source.substr(2); + else if(string_starts_with(script_source, "/")) + script_source = "https://soundcloud.com/" + script_source.substr(1); + + async_download_threads.push_back(std::async(std::launch::async, [script_source]() -> std::string { + std::string website_data; + DownloadResult download_result = download_to_string(script_source, website_data, {}, true); + if(download_result != DownloadResult::OK) return ""; + + size_t index = website_data.find("web-auth?client_id="); + if(index == std::string::npos) return ""; + + index += 19; + size_t end = website_data.find('&', index); + if(end == std::string::npos) end = website_data.size(); + return website_data.substr(index, end - index); + })); + } + + for(auto &download_thread : async_download_threads) { + if(download_thread.valid()) { + std::string fetched_client_id = download_thread.get(); + if(client_id.empty() && !fetched_client_id.empty()) + client_id = std::move(fetched_client_id); + } + } + } + + return PluginResult::OK; + } + PluginResult SoundcloudUserPage::get_page(const std::string&, int page, BodyItems &result_items) { while(current_page < page) { PluginResult plugin_result = get_continuation_page(result_items); -- cgit v1.2.3