Matrix: fetch additional messages in the background for all rooms, filter room list when updated after sync

2 files changed, 126 insertions, 30 deletions

diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 865f6fe..5d980f3 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -981,6 +981,11 @@ namespace QuickMedia {
             if(after_submit_handler)
                 after_submit_handler();
 
+            if(tabs[selected_tab].page->clear_search_after_submit() && tabs[selected_tab].search_bar) {
+                tabs[selected_tab].search_bar->clear();
+                tabs[selected_tab].search_bar->onTextUpdateCallback("");
+            }
+
             if(tabs[selected_tab].page->is_single_page()) {
                 tabs[selected_tab].search_bar->clear();
                 if(new_tabs.size() == 1)
@@ -3224,16 +3229,27 @@ namespace QuickMedia {
             }
         };
 
-        auto process_pinned_events = [&tabs](const std::optional<std::vector<std::string>> &pinned_events) {
-            if(!pinned_events || pinned_events->empty())
+        auto pinned_body_items_contains_event = [&tabs](const std::string &event_id) {
+            for(auto &body_item : tabs[PINNED_TAB_INDEX].body->items) {
+                if(static_cast<PinnedEventData*>(body_item->userdata)->event_id == event_id)
+                    return true;
+            }
+            return false;
+        };
+
+        auto process_pinned_events = [&tabs, &pinned_body_items_contains_event](const std::optional<std::vector<std::string>> &pinned_events) {
+            if(!pinned_events)
                 return;
 
             bool empty_before = tabs[PINNED_TAB_INDEX].body->items.empty();
             int selected_before = tabs[PINNED_TAB_INDEX].body->get_selected_item();
-            tabs[PINNED_TAB_INDEX].body->items.clear();
+            auto prev_pinned_body_items = tabs[PINNED_TAB_INDEX].body->items;
 
             // TODO: Add message to rooms messages when there are new pinned events
             for(const std::string &event : pinned_events.value()) {
+                if(pinned_body_items_contains_event(event))
+                    continue;
+
                 auto body = BodyItem::create("");
                 body->set_description("Loading message...");
                 PinnedEventData *event_data = new PinnedEventData();
@@ -3243,6 +3259,11 @@ namespace QuickMedia {
                 tabs[PINNED_TAB_INDEX].body->items.push_back(std::move(body));
             }
 
+            for(auto &prev_body_item : prev_pinned_body_items) {
+                if(!pinned_body_items_contains_event(static_cast<PinnedEventData*>(prev_body_item->userdata)->event_id))
+                    delete (PinnedEventData*)prev_body_item->userdata;
+            }
+
             if(empty_before)
                 tabs[PINNED_TAB_INDEX].body->select_last_item();
             else
@@ -3610,7 +3631,7 @@ namespace QuickMedia {
             return false;
         };
 
-        auto cleanup_tasks = [&set_read_marker_future, &fetch_message_future, &typing_state_queue, &typing_state_thread, &post_task_queue, &post_thread, &unreferenced_event_by_room, &tabs]() {
+        auto cleanup_tasks = [&set_read_marker_future, &fetch_message_future, &typing_state_queue, &typing_state_thread, &post_task_queue, &post_thread, &tabs]() {
             set_read_marker_future.cancel();
             fetch_message_future.cancel();
             typing_state_queue.close();
@@ -3630,6 +3651,7 @@ namespace QuickMedia {
                 for(auto &body_item : tabs[PINNED_TAB_INDEX].body->items) {
                     delete (PinnedEventData*)body_item->userdata;
                 }
+                tabs[PINNED_TAB_INDEX].body->clear_items();
             }
 
             //tabs.clear();
@@ -4281,7 +4303,6 @@ namespace QuickMedia {
                     std::vector<std::string> pinned_events;
                     matrix->get_all_pinned_events(current_room, pinned_events);
                     process_pinned_events(std::move(pinned_events));
-                    tabs[PINNED_TAB_INDEX].body->select_last_item();
                     typing_state_queue.restart();
                     typing_state_thread = std::thread(typing_state_handler);
                     post_task_queue.restart();
@@ -4314,11 +4335,13 @@ namespace QuickMedia {
 
         auto rooms_tags_body = create_body();
         rooms_tags_body->thumbnail_mask_shader = &circle_mask_shader;
-        auto matrix_rooms_tag_page = std::make_unique<MatrixRoomTagsPage>(this, rooms_tags_body.get());
+        auto matrix_rooms_tage_page_search_bar = create_search_bar("Search...", SEARCH_DELAY_FILTER);
+        auto matrix_rooms_tag_page = std::make_unique<MatrixRoomTagsPage>(this, rooms_tags_body.get(), matrix_rooms_tage_page_search_bar.get());
 
         auto invites_body = create_body();
         invites_body->thumbnail_mask_shader = &circle_mask_shader;
-        auto matrix_invites_page = std::make_unique<MatrixInvitesPage>(this, matrix, invites_body.get());
+        auto matrix_invites_page_search_bar = create_search_bar("Search...", SEARCH_DELAY_FILTER);
+        auto matrix_invites_page = std::make_unique<MatrixInvitesPage>(this, matrix, invites_body.get(), matrix_invites_page_search_bar.get());
 
         MatrixQuickMedia matrix_handler(this, matrix, matrix_rooms_page.get(), matrix_rooms_tag_page.get(), matrix_invites_page.get());
         bool sync_cached = false;
@@ -4327,8 +4350,8 @@ namespace QuickMedia {
 
         std::vector<Tab> tabs;
         tabs.push_back(Tab{std::move(rooms_body), std::move(matrix_rooms_page), std::move(matrix_rooms_page_search_bar)});
-        tabs.push_back(Tab{std::move(rooms_tags_body), std::move(matrix_rooms_tag_page), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
-        tabs.push_back(Tab{std::move(invites_body), std::move(matrix_invites_page), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+        tabs.push_back(Tab{std::move(rooms_tags_body), std::move(matrix_rooms_tag_page), std::move(matrix_rooms_tage_page_search_bar)});
+        tabs.push_back(Tab{std::move(invites_body), std::move(matrix_invites_page), std::move(matrix_invites_page_search_bar)});
 
         while(window.isOpen()) {
             page_loop(tabs);
diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp
index 9cfc1bb..32700a7 100644
--- a/src/plugins/Matrix.cpp
+++ b/src/plugins/Matrix.cpp
@@ -21,7 +21,8 @@
 static const char* SERVICE_NAME = "matrix";
 static const char* OTHERS_ROOM_TAG = "tld.name.others";
 // Filter without account data
-static const char* FILTER = "{\"presence\":{\"limit\":0,\"types\":[\"\"]},\"account_data\":{\"limit\":0,\"types\":[\"\"]},\"room\":{\"state\":{\"not_types\":[\"m.room.related_groups\",\"m.room.power_levels\",\"m.room.join_rules\",\"m.room.history_visibility\"],\"lazy_load_members\":true},\"timeline\":{\"limit\":1,\"lazy_load_members\":true},\"ephemeral\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true},\"account_data\":{\"limit\":1,\"types\":[\"m.fully_read\",\"m.tag\"],\"lazy_load_members\":true}}}";
+static const char* INITIAL_FILTER = "{\"presence\":{\"limit\":0,\"types\":[\"\"]},\"account_data\":{\"limit\":0,\"types\":[\"\"]},\"room\":{\"state\":{\"not_types\":[\"m.room.related_groups\",\"m.room.power_levels\",\"m.room.join_rules\",\"m.room.history_visibility\"],\"lazy_load_members\":true},\"timeline\":{\"limit\":1,\"lazy_load_members\":true},\"ephemeral\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true},\"account_data\":{\"limit\":1,\"types\":[\"m.fully_read\",\"m.tag\"],\"lazy_load_members\":true}}}";
+static const char* CONTINUE_FILTER = "{\"presence\":{\"limit\":0,\"types\":[\"\"]},\"account_data\":{\"limit\":0,\"types\":[\"\"]},\"room\":{\"state\":{\"not_types\":[\"m.room.related_groups\",\"m.room.power_levels\",\"m.room.join_rules\",\"m.room.history_visibility\"],\"lazy_load_members\":true},\"timeline\":{\"limit\":20,\"lazy_load_members\":true},\"ephemeral\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true},\"account_data\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true}}}";
 
 static rapidjson::Value nullValue(rapidjson::kNullType);
 static const rapidjson::Value& GetMember(const rapidjson::Value &obj, const char *key) {
@@ -217,7 +218,7 @@ namespace QuickMedia {
 
     void RoomData::clear_data() {
         std::lock_guard<std::recursive_mutex> lock(room_mutex);
-        fetched_messages_by_event_id.clear();
+        //fetched_messages_by_event_id.clear();
         user_info_by_user_id.clear();
         messages.clear();
         messages_read_index = 0;
@@ -469,6 +470,9 @@ namespace QuickMedia {
                 body->clear_items();
             }
 
+            if(!pending_remove_body_items.empty() || !room_body_items.empty())
+                filter_on_update = true;
+
             for(const std::string &room_id : pending_remove_body_items) {
                 remove_body_item_by_url(body->items, room_id);
                 // TODO: There can be a race condition where current_chat_page is set after entering a room and then we will enter a room we left
@@ -488,6 +492,11 @@ namespace QuickMedia {
             sort_room_body_items(body->items);
         }
         matrix_delegate->update(MatrixPageType::ROOM_LIST);
+        if(filter_on_update) {
+            filter_on_update = false;
+            if(search_bar)
+                body->filter_search_fuzzy(search_bar->get_text());
+        }
     }
 
     void MatrixRoomsPage::add_body_item(std::shared_ptr<BodyItem> body_item) {
@@ -578,8 +587,10 @@ namespace QuickMedia {
 
                 if(tag_body_it->second.room_body_items.empty()) {
                     auto room_body_item_it = std::find(body->items.begin(), body->items.end(), tag_body_it->second.tag_item);
-                    if(room_body_item_it != body->items.end())
+                    if(room_body_item_it != body->items.end()) {
                         body->items.erase(room_body_item_it);
+                        filter_on_update = true;
+                    }
                     tag_body_items_by_name.erase(tag_body_it);
                 }
             }
@@ -598,6 +609,7 @@ namespace QuickMedia {
                         body->items.push_back(tag_body_item);
                         tag_data = &tag_body_items_by_name[it.first];
                         tag_data->tag_item = tag_body_item;
+                        filter_on_update = true;
                     }
                 } else {
                     tag_data = &tag_body_it->second;
@@ -610,6 +622,11 @@ namespace QuickMedia {
             add_room_body_items_by_tags.clear();
         }
         matrix_delegate->update(MatrixPageType::ROOM_LIST);
+        if(filter_on_update) {
+            filter_on_update = false;
+            if(search_bar)
+                body->filter_search_fuzzy(search_bar->get_text());
+        }
     }
 
     void MatrixRoomTagsPage::add_room_body_item_to_tag(std::shared_ptr<BodyItem> body_item, const std::string &tag) {
@@ -662,7 +679,7 @@ namespace QuickMedia {
             current_rooms_page->sort_rooms();
     }
 
-    MatrixInvitesPage::MatrixInvitesPage(Program *program, Matrix *matrix, Body *body) : Page(program), matrix(matrix), body(body) {
+    MatrixInvitesPage::MatrixInvitesPage(Program *program, Matrix *matrix, Body *body, SearchBar *search_bar) : Page(program), matrix(matrix), body(body), search_bar(search_bar) {
 
     }
 
@@ -706,6 +723,9 @@ namespace QuickMedia {
             body->clear_items();
         }
 
+        if(!pending_remove_body_items.empty() || !body_items.empty())
+            filter_on_update = true;
+
         for(const std::string &room_id : pending_remove_body_items) {
             remove_body_item_by_url(body->items, room_id);
         }
@@ -718,6 +738,12 @@ namespace QuickMedia {
             prev_invite_count = body->items.size();
             title = "Invites (" + std::to_string(body->items.size()) + ")";
         }
+
+        if(filter_on_update) {
+            filter_on_update = false;
+            if(search_bar)
+                body->filter_search_fuzzy(search_bar->get_text());
+        }
     }
 
     void MatrixInvitesPage::add_body_item(std::shared_ptr<BodyItem> body_item) {
@@ -847,7 +873,7 @@ namespace QuickMedia {
                     rapidjson::ParseResult parse_result = doc.ParseStream<rapidjson::kParseStopWhenDoneFlag>(is);
                     if(parse_result.IsError())
                         break;
-                    if(parse_sync_response(doc) != PluginResult::OK)
+                    if(parse_sync_response(doc, false) != PluginResult::OK)
                         fprintf(stderr, "Failed to parse cached sync response\n");
                 }
                 fclose(sync_cache_file);
@@ -857,6 +883,7 @@ namespace QuickMedia {
             // Filter with account data
             // {"presence":{"limit":0,"types":[""]},"account_data":{"not_types":["im.vector.setting.breadcrumbs","m.push_rules","im.vector.setting.allowed_widgets","io.element.recent_emoji"]},"room":{"state":{"limit":1,"not_types":["m.room.related_groups","m.room.power_levels","m.room.join_rules","m.room.history_visibility"],"lazy_load_members":true},"timeline":{"limit":3,"lazy_load_members":true},"ephemeral":{"limit":0,"types":[""],"lazy_load_members":true},"account_data":{"limit":1,"types":["m.fully_read"],"lazy_load_members":true}}}
 
+#if 0
             bool filter_cached = false;
             Path filter_path = get_storage_dir().join("matrix").join("filter");
             if(get_file_type(filter_path) == FileType::REGULAR)
@@ -869,13 +896,39 @@ namespace QuickMedia {
                 filter = get_filter_cached();
             else
                 filter = FILTER;
-            std::string filter_encoded = url_param_encode(filter);
+#endif
+            std::string filter_encoded = url_param_encode(INITIAL_FILTER);
 
             std::vector<CommandArg> additional_args = {
                 { "-H", "Authorization: Bearer " + access_token },
                 { "-m", "35" }
             };
 
+            sync_additional_messages_thread = std::thread([this]() {
+                std::vector<CommandArg> additional_args = {
+                    { "-H", "Authorization: Bearer " + access_token },
+                    { "-m", "35" }
+                };
+
+                char url[1024];
+                std::string filter_encoded = url_param_encode(CONTINUE_FILTER);
+                snprintf(url, sizeof(url), "%s/_matrix/client/r0/sync?filter=%s&timeout=0", homeserver.c_str(), filter_encoded.c_str());
+
+                rapidjson::Document json_root;
+                std::string err_msg;
+                DownloadResult download_result = download_json(json_root, url, additional_args, true, &err_msg);
+                if(download_result != DownloadResult::OK) {
+                    fprintf(stderr, "/sync for additional messages failed\n");
+                    return;
+                }
+
+                // TODO: Test?
+                //if(next_batch.empty())
+                //    clear_sync_cache_for_new_sync();
+
+                parse_sync_response(json_root, true);
+            });
+
             const rapidjson::Value *next_batch_json;
             PluginResult result;
             bool initial_sync = true;
@@ -916,7 +969,7 @@ namespace QuickMedia {
                 if(next_batch.empty())
                     clear_sync_cache_for_new_sync();
 
-                result = parse_sync_response(json_root);
+                result = parse_sync_response(json_root, false);
                 if(result != PluginResult::OK) {
                     fprintf(stderr, "Failed to parse sync response\n");
                     goto sync_end;
@@ -952,12 +1005,16 @@ namespace QuickMedia {
                         const rapidjson::Value &notification_json = GetMember(json_root, "notifications");
                         parse_notifications(notification_json);
                     });
+
+                    filter_encoded = url_param_encode(CONTINUE_FILTER);
                 }
 
+#if 0
                 if(!filter_cached) {
                     filter_cached = true;
                     filter_encoded = url_param_encode(get_filter_cached());
                 }
+#endif
 
                 sync_end:
                 if(sync_running)
@@ -987,6 +1044,11 @@ namespace QuickMedia {
             program_kill_in_thread(sync_thread.get_id());
             sync_thread.join();
         }
+
+        if(sync_additional_messages_thread.joinable()) {
+            program_kill_in_thread(sync_additional_messages_thread.get_id());
+            sync_additional_messages_thread.join();
+        }
         
         if(notification_thread.joinable()) {
             program_kill_in_thread(notification_thread.get_id());
@@ -1064,7 +1126,7 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    PluginResult Matrix::parse_sync_response(const rapidjson::Document &root) {
+    PluginResult Matrix::parse_sync_response(const rapidjson::Document &root, bool is_additional_messages_sync) {
         if(!root.IsObject())
             return PluginResult::ERR;
 
@@ -1074,7 +1136,7 @@ namespace QuickMedia {
         // TODO: Include "Direct messages" as a tag using |dm_rooms| above
 
         const rapidjson::Value &rooms_json = GetMember(root, "rooms");
-        parse_sync_room_data(rooms_json);
+        parse_sync_room_data(rooms_json, is_additional_messages_sync);
 
         return PluginResult::OK;
     }
@@ -1173,7 +1235,7 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    PluginResult Matrix::parse_sync_room_data(const rapidjson::Value &rooms_json) {
+    PluginResult Matrix::parse_sync_room_data(const rapidjson::Value &rooms_json, bool is_additional_messages_sync) {
         if(!rooms_json.IsObject())
             return PluginResult::OK;
 
@@ -1204,14 +1266,15 @@ namespace QuickMedia {
                     const rapidjson::Value &events_json = GetMember(state_json, "events");
                     events_add_user_info(events_json, room);
                     events_set_room_name(events_json, room);
-                    events_add_pinned_events(events_json, room);
+                    if(!is_additional_messages_sync)
+                        events_add_pinned_events(events_json, room);
                 }
 
                 const rapidjson::Value &account_data_json = GetMember(it.value, "account_data");
 
                 const rapidjson::Value &timeline_json = GetMember(it.value, "timeline");
                 if(timeline_json.IsObject()) {
-                    if(!room->has_prev_batch()) {
+                    if(is_additional_messages_sync && !room->has_prev_batch()) {
                         // This may be non-existent if this is the first event in the room
                         const rapidjson::Value &prev_batch_json = GetMember(timeline_json, "prev_batch");
                         if(prev_batch_json.IsString())
@@ -1243,7 +1306,8 @@ namespace QuickMedia {
                         delegate->join_room(room);
 
                     events_add_messages(events_json, room, MessageDirection::AFTER, has_unread_notifications);
-                    events_add_pinned_events(events_json, room);
+                    if(!is_additional_messages_sync)
+                        events_add_pinned_events(events_json, room);
                 } else {
                     if(account_data_json.IsObject()) {
                         const rapidjson::Value &events_json = GetMember(account_data_json, "events");
@@ -1277,11 +1341,13 @@ namespace QuickMedia {
             }
         }
 
-        const rapidjson::Value &leave_json = GetMember(rooms_json, "leave");
-        remove_rooms(leave_json);
+        if(!is_additional_messages_sync) {
+            const rapidjson::Value &leave_json = GetMember(rooms_json, "leave");
+            remove_rooms(leave_json);
 
-        const rapidjson::Value &invite_json = GetMember(rooms_json, "invite");
-        add_invites(invite_json);
+            const rapidjson::Value &invite_json = GetMember(rooms_json, "invite");
+            add_invites(invite_json);
+        }
 
         return PluginResult::OK;
     }
@@ -1881,6 +1947,7 @@ namespace QuickMedia {
         bool has_room_name = room_data->has_name();
         bool has_room_avatar_url = room_data->has_avatar_url();
 
+        // TODO: Fix. This can be incorrect because this method can be called before we have added all users to the room
         std::vector<std::shared_ptr<UserInfo>> users_excluding_me;
         if(!has_room_name || !has_room_avatar_url)
             users_excluding_me = room_data->get_users_excluding_me(user_id); // TODO: What about thread safety with user_id? its reset in /logout
@@ -2083,8 +2150,8 @@ namespace QuickMedia {
                     invite.new_invite = !next_batch.empty();
 
                     std::string room_id_str(room_id.GetString(), room_id.GetStringLength());
-                    delegate->add_invite(room_id_str, invite);
-                    set_invite(room_id_str, std::move(invite));
+                    if(set_invite(room_id_str, invite))
+                        delegate->add_invite(room_id_str, std::move(invite));
                     break;
                 }
             }
@@ -3149,9 +3216,10 @@ namespace QuickMedia {
         room_data_by_id.erase(room_it);
     }
 
-    void Matrix::set_invite(const std::string &room_id, Invite invite) {
+    bool Matrix::set_invite(const std::string &room_id, Invite invite) {
         std::lock_guard<std::mutex> lock(invite_mutex);
-        invites[room_id] = std::move(invite);
+        auto res = invites.insert(std::make_pair(room_id, std::move(invite)));
+        return res.second;
     }
 
     bool Matrix::remove_invite(const std::string &room_id) {
@@ -3219,6 +3287,7 @@ namespace QuickMedia {
 
     // TODO: GET the filter to check if its valid?
     std::string Matrix::get_filter_cached() {
+        #if 0
         if(filter_cached)
             return filter_cached.value();
 
@@ -3255,6 +3324,10 @@ namespace QuickMedia {
             file_overwrite_atomic(filter_path, filter_cached.value());
         }
         return filter_cached.value();
+        #else
+        assert(false);
+        return INITIAL_FILTER;
+        #endif
     }
 
     DownloadResult Matrix::download_json(rapidjson::Document &result, const std::string &url, std::vector<CommandArg> additional_args, bool use_browser_useragent, std::string *err_msg) const {