Fix soundcloud (fetch client id), add authors to mangakatana, some other fixes

3 files changed, 192 insertions, 68 deletions

diff --git a/src/plugins/MangaGeneric.cpp b/src/plugins/MangaGeneric.cpp
index 24472e5..58bedf4 100644
--- a/src/plugins/MangaGeneric.cpp
+++ b/src/plugins/MangaGeneric.cpp
@@ -49,11 +49,6 @@ namespace QuickMedia {
             return quickmedia_html_node_get_attribute_value(node, field_name);
     }
 
-    static bool starts_with(const std::string &str, const char *sub) {
-        size_t sub_len = strlen(sub);
-        return str.size() >= sub_len && memcmp(str.c_str(), sub, sub_len) == 0;
-    }
-
     static int html_append_search(QuickMediaHtmlSearch *html_search, const char *html_query, HtmlSearchUserdata *search_userdata) {
         return quickmedia_html_find_nodes_xpath(html_search, html_query,
             [](QuickMediaHtmlNode *node, void *userdata) {
@@ -149,12 +144,8 @@ namespace QuickMedia {
         return plugin_result_to_search_result(get_page(str, 0, result_items));
     }
 
-    PluginResult MangaGenericSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) {
+    PluginResult MangaGenericSearchPage::get_page(const std::string &url, BodyItems &result_items) {
         std::string target_url;
-        std::string url = search_query.search_template;
-        string_replace_all(url, "%s", url_param_encode(str));
-        string_replace_all(url, "%p", std::to_string(search_query.page_start + page));
-
         std::string website_data;
         if(download_to_string(url, website_data, {}, true, fail_on_http_error) != DownloadResult::OK)
             return PluginResult::NET_ERR;
@@ -184,36 +175,6 @@ namespace QuickMedia {
             if(result != 0)
                 goto cleanup;
 
-            for(const DescriptionQuery &description_query : description_queries) {
-                assert(description_query.html_query && description_query.field_name);
-                if(description_query.html_query && description_query.field_name) {
-                    HtmlMergeUserdata merge_userdata;
-                    merge_userdata.type = MergeType::DESCRIPTION;
-                    merge_userdata.body_item_image_context.body_items = &new_result_items;
-                    merge_userdata.body_item_image_context.index = 0;
-                    merge_userdata.field_name = description_query.field_name;
-                    merge_userdata.field_contains = nullptr;
-                    result = html_body_item_merge(&html_search, description_query.html_query, &merge_userdata);
-                    if(result != 0)
-                        goto cleanup;
-                }
-            }
-
-            for(const ThumbnailQuery &thumbnail_query : thumbnail_queries) {
-                assert(thumbnail_query.html_query && thumbnail_query.field_name);
-                if(thumbnail_query.html_query && thumbnail_query.field_name) {
-                    HtmlMergeUserdata merge_userdata;
-                    merge_userdata.type = MergeType::THUMBNAIL;
-                    merge_userdata.body_item_image_context.body_items = &new_result_items;
-                    merge_userdata.body_item_image_context.index = 0;
-                    merge_userdata.field_name = thumbnail_query.field_name;
-                    merge_userdata.field_contains = thumbnail_query.field_contains;
-                    result = html_body_item_merge(&html_search, thumbnail_query.html_query, &merge_userdata);
-                    if(result != 0)
-                        goto cleanup;
-                }
-            }
-
             if(!text_query.url_field && !new_result_items.empty()) {
                 if(target_url.empty()) {
                     std::string response_headers;
@@ -239,15 +200,45 @@ namespace QuickMedia {
             result_items.insert(result_items.end(), std::move_iterator(new_result_items.begin()), std::move_iterator(new_result_items.end()));
         }
 
+        for(const DescriptionQuery &description_query : description_queries) {
+            assert(description_query.html_query && description_query.field_name);
+            if(description_query.html_query && description_query.field_name) {
+                HtmlMergeUserdata merge_userdata;
+                merge_userdata.type = MergeType::DESCRIPTION;
+                merge_userdata.body_item_image_context.body_items = &result_items;
+                merge_userdata.body_item_image_context.index = 0;
+                merge_userdata.field_name = description_query.field_name;
+                merge_userdata.field_contains = nullptr;
+                result = html_body_item_merge(&html_search, description_query.html_query, &merge_userdata);
+                if(result != 0)
+                    goto cleanup;
+            }
+        }
+
+        for(const ThumbnailQuery &thumbnail_query : thumbnail_queries) {
+            assert(thumbnail_query.html_query && thumbnail_query.field_name);
+            if(thumbnail_query.html_query && thumbnail_query.field_name) {
+                HtmlMergeUserdata merge_userdata;
+                merge_userdata.type = MergeType::THUMBNAIL;
+                merge_userdata.body_item_image_context.body_items = &result_items;
+                merge_userdata.body_item_image_context.index = 0;
+                merge_userdata.field_name = thumbnail_query.field_name;
+                merge_userdata.field_contains = thumbnail_query.field_contains;
+                result = html_body_item_merge(&html_search, thumbnail_query.html_query, &merge_userdata);
+                if(result != 0)
+                    goto cleanup;
+            }
+        }
+
         for(auto &body_item : result_items) {
-            if(starts_with(body_item->url, "//"))
+            if(string_starts_with(body_item->url, "//"))
                 body_item->url = "https://" + body_item->url.substr(2);
-            else if(starts_with(body_item->url, "/"))
+            else if(string_starts_with(body_item->url, "/"))
                 body_item->url = website_url + body_item->url.substr(1);
 
-            if(starts_with(body_item->thumbnail_url, "//"))
+            if(string_starts_with(body_item->thumbnail_url, "//"))
                 body_item->thumbnail_url = "https://" + body_item->thumbnail_url.substr(2);
-            else if(starts_with(body_item->thumbnail_url, "/"))
+            else if(string_starts_with(body_item->thumbnail_url, "/"))
                 body_item->thumbnail_url = website_url + body_item->thumbnail_url.substr(1);
         }
 
@@ -261,12 +252,20 @@ namespace QuickMedia {
         }
     }
 
+    PluginResult MangaGenericSearchPage::get_page(const std::string &str, int page, BodyItems &result_items) {
+        std::string url = search_query.search_template;
+        string_replace_all(url, "%s", url_param_encode(str));
+        string_replace_all(url, "%p", std::to_string(search_query.page_start + page));
+        return get_page(url, result_items);
+    }
+
     PluginResult MangaGenericSearchPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
         if(!list_chapters_query.html_query || !list_chapters_query.title_field || !list_chapters_query.url_field) {
             assert(false);
             return PluginResult::ERR;
         }
 
+        std::vector<Creator> creators;
         BodyItems chapters_items;
         HtmlSearchUserdata search_userdata;
         search_userdata.body_items = &chapters_items;
@@ -299,18 +298,49 @@ namespace QuickMedia {
             result = html_body_item_merge(&html_search, list_chapters_query.uploaded_time_html_query, &merge_userdata);
         }
 
+        if(authors_query.html_query && authors_query.title_field && authors_query.url_field) {
+            struct HtmlAuthorsUserdata {
+                std::vector<Creator> *creators;
+                AuthorsQuery *authors_query;
+            };
+
+            HtmlAuthorsUserdata authors_userdata;
+            authors_userdata.creators = &creators;
+            authors_userdata.authors_query = &authors_query;
+
+            quickmedia_html_find_nodes_xpath(&html_search, authors_query.html_query,
+                [](QuickMediaHtmlNode *node, void *userdata) {
+                    HtmlAuthorsUserdata *authors_userdata = (HtmlAuthorsUserdata*)userdata;
+                    const char *title_value = html_attr_or_inner_text(node, authors_userdata->authors_query->title_field);
+                    const char *url_value = html_attr_or_inner_text(node, authors_userdata->authors_query->url_field);
+                    if(title_value && url_value && (!authors_userdata->authors_query->url_contains || strstr(url_value, authors_userdata->authors_query->url_contains))) {
+                        Creator creator;
+                        creator.name = strip(title_value);
+                        creator.url = strip(url_value);
+                        authors_userdata->creators->push_back(std::move(creator));
+                    }
+                }, &authors_userdata);
+        }
+
         for(auto &body_item : chapters_items) {
-            if(starts_with(body_item->url, "//"))
+            if(string_starts_with(body_item->url, "//"))
                 body_item->url = "https://" + body_item->url.substr(2);
-            else if(starts_with(body_item->url, "/"))
+            else if(string_starts_with(body_item->url, "/"))
                 body_item->url = website_url + body_item->url.substr(1);
 
-            if(starts_with(body_item->thumbnail_url, "//"))
+            if(string_starts_with(body_item->thumbnail_url, "//"))
                 body_item->thumbnail_url = "https://" + body_item->thumbnail_url.substr(2);
-            else if(starts_with(body_item->thumbnail_url, "/"))
+            else if(string_starts_with(body_item->thumbnail_url, "/"))
                 body_item->thumbnail_url = website_url + body_item->thumbnail_url.substr(1);
         }
 
+        for(auto &creator : creators) {
+            if(string_starts_with(creator.url, "//"))
+                creator.url = "https://" + creator.url.substr(2);
+            else if(string_starts_with(creator.url, "/"))
+                creator.url = website_url + creator.url.substr(1);
+        }
+
         cleanup:
         quickmedia_html_search_deinit(&html_search);
         if(result != 0)
@@ -319,6 +349,11 @@ namespace QuickMedia {
         auto body = create_body();
         body->items = std::move(chapters_items);
         result_tabs.push_back(Tab{std::move(body), std::make_unique<MangaGenericChaptersPage>(program, title, url, manga_id_extractor, service_name, website_url, &list_page_query, fail_on_http_error), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+
+        for(Creator &creator : creators) {
+            result_tabs.push_back(Tab{create_body(), std::make_unique<MangaGenericCreatorPage>(program, this, std::move(creator)), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+        }
+
         return PluginResult::OK;
     }
 
@@ -348,6 +383,14 @@ namespace QuickMedia {
         return true;
     }
 
+    PluginResult MangaGenericCreatorPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
+        return search_page->submit(title, url, result_tabs);
+    }
+
+    PluginResult MangaGenericCreatorPage::lazy_fetch(BodyItems &result_items) {
+        return search_page->get_page(creator.url, result_items);
+    }
+
     static bool is_number(const char *str) {
         while(*str) {
             char c = *str;
@@ -443,9 +486,9 @@ namespace QuickMedia {
                     return ImageResult::ERR;
                 }
 
-                if(starts_with(current_image_url, "//"))
+                if(string_starts_with(current_image_url, "//"))
                     current_image_url = "https://" + current_image_url.substr(2);
-                else if(starts_with(current_image_url, "/"))
+                else if(string_starts_with(current_image_url, "/"))
                     current_image_url = website_url + current_image_url.substr(1);
 
                 num_images = page_count_userdata.num_pages;
@@ -523,9 +566,9 @@ namespace QuickMedia {
                     cleanup:
                     quickmedia_html_search_deinit(&html_search);
 
-                    if(starts_with(current_image_url, "//"))
+                    if(string_starts_with(current_image_url, "//"))
                         current_image_url = "https://" + current_image_url.substr(2);
-                    else if(starts_with(current_image_url, "/"))
+                    else if(string_starts_with(current_image_url, "/"))
                         current_image_url = website_url + current_image_url.substr(1);
 
                     if(!callback(current_image_url))
@@ -603,9 +646,9 @@ namespace QuickMedia {
         }
 
         for(std::string &url : chapter_image_urls) {
-            if(starts_with(url, "//"))
+            if(string_starts_with(url, "//"))
                 url = "https://" + url.substr(2);
-            else if(starts_with(url, "/"))
+            else if(string_starts_with(url, "/"))
                 url = website_url + url.substr(1);
         }
 
@@ -635,6 +678,14 @@ namespace QuickMedia {
         return *this;
     }
 
+    MangaGenericSearchPage& MangaGenericSearchPage::authors_handler(const char *html_query, const char *title_field, const char *url_field, const char *url_contains) {
+        authors_query.html_query = html_query;
+        authors_query.title_field = title_field;
+        authors_query.url_field = url_field;
+        authors_query.url_contains = url_contains;
+        return *this;
+    }
+
     MangaGenericSearchPage& MangaGenericSearchPage::list_chapters_handler(const char *html_query, const char *title_field, const char *url_field, const char *url_contains) {
         list_chapters_query.html_query = html_query;
         list_chapters_query.title_field = title_field;
diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp
index 57c879e..0c06458 100644
--- a/src/plugins/Matrix.cpp
+++ b/src/plugins/Matrix.cpp
@@ -489,14 +489,15 @@ namespace QuickMedia {
     }
 
     static std::string message_to_room_description_text(Message *message) {
+        std::string body = strip(message->body);
         if(message->type == MessageType::REACTION)
-            return "Reacted with: " + extract_first_line_elipses(message->body, 150);
+            return "Reacted with: " + extract_first_line_elipses(body, 150);
         else if(message->related_event_type == RelatedEventType::REPLY)
-            return extract_first_line_elipses(remove_reply_formatting(message->body), 150);
+            return extract_first_line_elipses(remove_reply_formatting(body), 150);
         else if(message->related_event_type == RelatedEventType::EDIT)
-            return "Edited: " + extract_first_line_elipses(remove_reply_formatting(message->body), 150);
+            return "Edited: " + extract_first_line_elipses(remove_reply_formatting(body), 150);
         else
-            return extract_first_line_elipses(message->body, 150);
+            return extract_first_line_elipses(body, 150);
     }
 
     void MatrixQuickMedia::update_room_description(RoomData *room, Messages &new_messages, bool is_initial_sync, bool sync_is_cache, Body *chat_body, bool messages_tab_visible) {
@@ -1090,18 +1091,20 @@ namespace QuickMedia {
         }
     }
 
-    void Matrix::start_sync(MatrixDelegate *delegate, bool &cached) {
+    bool Matrix::start_sync(MatrixDelegate *delegate, bool &cached) {
         cached = true;
         if(sync_running)
-            return;
+            return true;
 
         assert(!this->delegate);
         assert(!access_token.empty()); // Need to be logged in
         this->delegate = delegate;
 
         Path matrix_cache_dir = get_cache_dir().join("matrix");
-        if(create_directory_recursive(matrix_cache_dir) != 0)
+        if(create_directory_recursive(matrix_cache_dir) != 0) {
             fprintf(stderr, "Failed to create matrix cache directory\n");
+            return false;
+        }
         matrix_cache_dir.join("sync_data.json");
 
         cached = (get_file_type(matrix_cache_dir) == FileType::REGULAR);
@@ -1196,6 +1199,7 @@ namespace QuickMedia {
                 result = parse_sync_response(json_root, false, initial_sync);
                 if(result != PluginResult::OK) {
                     fprintf(stderr, "Failed to parse sync response\n");
+                    initial_sync = false;
                     goto sync_end;
                 }
 
@@ -1206,6 +1210,7 @@ namespace QuickMedia {
                 } else {
                     //set_next_batch("Invalid");
                     fprintf(stderr, "Matrix: missing next batch\n");
+                    initial_sync = false;
                     goto sync_end;
                 }
 
@@ -1288,6 +1293,8 @@ namespace QuickMedia {
                     std::this_thread::sleep_for(std::chrono::milliseconds(500));
             }
         });
+
+        return true;
     }
 
     void Matrix::stop_sync() {
@@ -1313,6 +1320,10 @@ namespace QuickMedia {
         delegate = nullptr;
         sync_failed = false;
         sync_fail_reason.clear();
+        set_next_batch("");
+        invites.clear();
+        filter_cached.reset();
+        my_events_transaction_ids.clear();
     }
 
     bool Matrix::is_initial_sync_finished() const {
@@ -3382,7 +3393,7 @@ namespace QuickMedia {
     }
 
     PluginResult Matrix::logout() {
-        assert(!sync_running);
+        stop_sync();
         Path session_path = get_storage_dir().join(SERVICE_NAME).join("session.json");
         remove(session_path.data.c_str());
 
@@ -3402,10 +3413,6 @@ namespace QuickMedia {
         homeserver.clear();
         homeserver_domain.clear();
         upload_limit.reset();
-        set_next_batch("");
-        invites.clear();
-        filter_cached.reset();
-        my_events_transaction_ids.clear();
 
         return PluginResult::OK;
     }
@@ -3950,7 +3957,7 @@ namespace QuickMedia {
 
         // TODO: Use at param? which is room->get_prev_batch();
         char url[512];
-        snprintf(url, sizeof(url), "%s/_matrix/client/r0/rooms/%s/members?membership=join", homeserver.c_str(), room->id.c_str());
+        snprintf(url, sizeof(url), "%s/_matrix/client/r0/rooms/%s/members?not_membership=leave", homeserver.c_str(), room->id.c_str());
 
         rapidjson::Document json_root;
         DownloadResult download_result = download_json(json_root, url, std::move(additional_args), true);
diff --git a/src/plugins/Soundcloud.cpp b/src/plugins/Soundcloud.cpp
index a0184b4..6740504 100644
--- a/src/plugins/Soundcloud.cpp
+++ b/src/plugins/Soundcloud.cpp
@@ -2,9 +2,11 @@
 #include "../../include/NetUtils.hpp"
 #include "../../include/StringUtils.hpp"
 #include "../../include/Scale.hpp"
+#include <quickmedia/HtmlSearch.h>
+#include <thread>
 
 namespace QuickMedia {
-    static std::string client_id = "Na04L87fnpWDMVCCW2ngWldN4JMoLTAc";
+    static std::string client_id;
 
     class SoundcloudPlaylist : public BodyItemExtra {
     public:
@@ -218,6 +220,11 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
+    PluginResult SoundcloudSearchPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
+        submit_page.submit_body_item = submit_body_item;
+        return submit_page.submit(title, url, result_tabs);
+    }
+
     SearchResult SoundcloudSearchPage::search(const std::string &str, BodyItems &result_items) {
         query_urn.clear();
         PluginResult result = get_page(str, 0, result_items);
@@ -268,6 +275,65 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
+    PluginResult SoundcloudSearchPage::lazy_fetch(BodyItems&) {
+        std::string website_data;
+        DownloadResult download_result = download_to_string("https://soundcloud.com/", website_data, {}, true);
+        if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result);
+
+        if(client_id.empty()) {
+            std::vector<std::string> script_sources;
+            QuickMediaHtmlSearch html_search;
+            int result = quickmedia_html_search_init(&html_search, website_data.c_str());
+            if(result != 0)
+                goto cleanup;
+
+            result = quickmedia_html_find_nodes_xpath(&html_search, "//script",
+                [](QuickMediaHtmlNode *node, void *userdata) {
+                    std::vector<std::string> *script_sources = (std::vector<std::string>*)userdata;
+                    const char *src = quickmedia_html_node_get_attribute_value(node, "src");
+                    if(src)
+                        script_sources->push_back(strip(src));
+                }, &script_sources);
+
+            cleanup:
+            quickmedia_html_search_deinit(&html_search);
+            if(result != 0)
+                return PluginResult::ERR;
+
+            std::vector<std::future<std::string>> async_download_threads;
+            for(std::string &script_source : script_sources) {
+                if(string_starts_with(script_source, "//"))
+                    script_source = "https://" + script_source.substr(2);
+                else if(string_starts_with(script_source, "/"))
+                    script_source = "https://soundcloud.com/" + script_source.substr(1);
+
+                async_download_threads.push_back(std::async(std::launch::async, [script_source]() -> std::string {
+                    std::string website_data;
+                    DownloadResult download_result = download_to_string(script_source, website_data, {}, true);
+                    if(download_result != DownloadResult::OK) return "";
+
+                    size_t index = website_data.find("web-auth?client_id=");
+                    if(index == std::string::npos) return "";
+
+                    index += 19;
+                    size_t end = website_data.find('&', index);
+                    if(end == std::string::npos) end = website_data.size();
+                    return website_data.substr(index, end - index);
+                }));
+            }
+
+            for(auto &download_thread : async_download_threads) {
+                if(download_thread.valid()) {
+                    std::string fetched_client_id = download_thread.get();
+                    if(client_id.empty() && !fetched_client_id.empty())
+                        client_id = std::move(fetched_client_id);
+                }
+            }
+        }
+        
+        return PluginResult::OK;
+    }
+
     PluginResult SoundcloudUserPage::get_page(const std::string&, int page, BodyItems &result_items) {
         while(current_page < page) {
             PluginResult plugin_result = get_continuation_page(result_items);