Matrix: attempt to fix fetching of previous messages before sync is finished

3 files changed, 64 insertions, 43 deletions

diff --git a/plugins/Matrix.hpp b/plugins/Matrix.hpp
index e60889f..e6a35e2 100644
--- a/plugins/Matrix.hpp
+++ b/plugins/Matrix.hpp
@@ -63,6 +63,7 @@ namespace QuickMedia {
         sf::Vector2i thumbnail_size; // Set to {0, 0} if not specified
         RelatedEventType related_event_type = RelatedEventType::NONE;
         bool mentions_me = false;
+        bool cache = false;
         time_t timestamp = 0; // In milliseconds
         MessageType type;
     };
@@ -81,10 +82,10 @@ namespace QuickMedia {
         void set_user_display_name(std::shared_ptr<UserInfo> &user, std::string display_name);
         void set_user_avatar_url(std::shared_ptr<UserInfo> &user, std::string avatar_url);
 
-        // Ignores duplicates
-        void prepend_messages_reverse(const std::vector<std::shared_ptr<Message>> &new_messages);
-        // Ignores duplicates
-        void append_messages(const std::vector<std::shared_ptr<Message>> &new_messages);
+        // Ignores duplicates, returns the number of added messages
+        size_t prepend_messages_reverse(const std::vector<std::shared_ptr<Message>> &new_messages);
+        // Ignores duplicates, returns the number of added messages
+        size_t append_messages(const std::vector<std::shared_ptr<Message>> &new_messages);
 
         std::shared_ptr<Message> get_message_by_id(const std::string &id);
 
@@ -483,11 +484,12 @@ namespace QuickMedia {
         PluginResult parse_notifications(const rapidjson::Value &notifications_json);
         PluginResult parse_sync_account_data(const rapidjson::Value &account_data_json, std::optional<std::set<std::string>> &dm_rooms);
         PluginResult parse_sync_room_data(const rapidjson::Value &rooms_json, bool is_additional_messages_sync);
-        PluginResult get_previous_room_messages(RoomData *room_data, bool latest_messages);
+        PluginResult get_previous_room_messages(RoomData *room_data, bool latest_messages, size_t &num_new_messages);
         void events_add_user_info(const rapidjson::Value &events_json, RoomData *room_data);
         std::shared_ptr<UserInfo> parse_user_info(const rapidjson::Value &json, const std::string &user_id, RoomData *room_data);
         void events_set_user_read_marker(const rapidjson::Value &events_json, RoomData *room_data, std::shared_ptr<UserInfo> &me);
-        void events_add_messages(const rapidjson::Value &events_json, RoomData *room_data, MessageDirection message_dir, bool has_unread_notifications);
+        // Returns the number of messages added
+        size_t events_add_messages(const rapidjson::Value &events_json, RoomData *room_data, MessageDirection message_dir, bool has_unread_notifications);
         void events_set_room_name(const rapidjson::Value &events_json, RoomData *room_data);
         void set_room_name_to_users_if_empty(RoomData *room, const std::string &room_creator_user_id);
         void events_add_pinned_events(const rapidjson::Value &events_json, RoomData *room_data);
diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp
index d8c8559..1e35d9e 100644
--- a/src/plugins/Matrix.cpp
+++ b/src/plugins/Matrix.cpp
@@ -161,24 +161,30 @@ namespace QuickMedia {
         user->resolve_state = UserResolveState::RESOLVED;
     }
 
-    void RoomData::prepend_messages_reverse(const std::vector<std::shared_ptr<Message>> &new_messages) {
+    size_t RoomData::prepend_messages_reverse(const std::vector<std::shared_ptr<Message>> &new_messages) {
         std::lock_guard<std::recursive_mutex> lock(room_mutex);
+        size_t num_new_messages = 0;
         for(auto it = new_messages.begin(); it != new_messages.end(); ++it) {
             if(message_by_event_id.find((*it)->event_id) == message_by_event_id.end()) {
                 message_by_event_id.insert(std::make_pair((*it)->event_id, *it));
                 messages.insert(messages.begin(), std::move(*it));
+                ++num_new_messages;
             }
         }
+        return num_new_messages;
     }
 
-    void RoomData::append_messages(const std::vector<std::shared_ptr<Message>> &new_messages) {
+    size_t RoomData::append_messages(const std::vector<std::shared_ptr<Message>> &new_messages) {
         std::lock_guard<std::recursive_mutex> lock(room_mutex);
+        size_t num_new_messages = 0;
         for(auto it = new_messages.begin(); it != new_messages.end(); ++it) {
             if(message_by_event_id.find((*it)->event_id) == message_by_event_id.end()) {
                 message_by_event_id.insert(std::make_pair((*it)->event_id, *it));
                 messages.push_back(std::move(*it));
+                ++num_new_messages;
             }
         }
+        return num_new_messages;
     }
 
     std::shared_ptr<Message> RoomData::get_message_by_id(const std::string &id) {
@@ -272,13 +278,26 @@ namespace QuickMedia {
     }
 
     void RoomData::clear_data() {
-        std::lock_guard<std::recursive_mutex> lock(room_mutex);
+        std::lock_guard<std::recursive_mutex> room_lock(room_mutex);
+        std::lock_guard<std::recursive_mutex> user_lock(user_mutex);
         //fetched_messages_by_event_id.clear();
-        userdata = nullptr;
-        user_info_by_user_id.clear();
-        messages.clear();
-        messages_read_index = 0;
-        message_by_event_id.clear();
+        //userdata = nullptr;
+        //user_info_by_user_id.clear();
+        size_t i = 0;
+        for(auto it = messages.begin(); it != messages.end();) {
+            if((*it)->cache) {
+                message_by_event_id.erase((*it)->event_id);
+                it = messages.erase(it);
+                if(i <= messages_read_index)
+                    --messages_read_index;
+            } else {
+                ++it;
+            }
+            ++i;
+        }
+        //messages.clear();
+        //messages_read_index = 0;
+        //message_by_event_id.clear();
         pinned_events.clear();
         tags.clear();
     }
@@ -421,12 +440,12 @@ namespace QuickMedia {
     void MatrixQuickMedia::clear_data() {
         std::lock_guard<std::mutex> lock(pending_room_messages_mutex);
         std::lock_guard<std::mutex> room_body_lock(room_body_items_mutex);
-        room_body_item_by_room.clear();
-        pending_room_messages.clear();
-        rooms_page->clear_data();
-        room_tags_page->clear_data();
+        //room_body_item_by_room.clear();
+        //pending_room_messages.clear();
+        //rooms_page->clear_data();
+        //room_tags_page->clear_data();
         invites_page->clear_data();
-        unread_notifications.clear();
+        //unread_notifications.clear();
     }
 
     static std::shared_ptr<Message> get_last_message_by_timestamp(const Messages &messages) {
@@ -1194,22 +1213,18 @@ namespace QuickMedia {
     }
 
     PluginResult Matrix::get_previous_room_messages(RoomData *room, Messages &messages, bool latest_messages) {
-        room->acquire_room_lock();
-        size_t num_messages_before = room->get_messages_thread_unsafe().size();
-        room->release_room_lock();
-        PluginResult result = get_previous_room_messages(room, latest_messages);
+        size_t num_new_messages = 0;
+        PluginResult result = get_previous_room_messages(room, latest_messages, num_new_messages);
         if(result != PluginResult::OK)
             return result;
 
         room->acquire_room_lock();
-        size_t num_messages_after = room->get_messages_thread_unsafe().size();
-        size_t num_new_messages = num_messages_after - num_messages_before;
         messages.insert(messages.end(), room->get_messages_thread_unsafe().begin(), room->get_messages_thread_unsafe().begin() + num_new_messages);
         room->messages_read_index += num_new_messages;
         //assert(room->messages_read_index <= num_messages_after);
         // TODO: BUG
-        if(room->messages_read_index >= num_messages_after)
-            room->messages_read_index = num_messages_after;
+        if(room->messages_read_index >= room->get_messages_thread_unsafe().size())
+            room->messages_read_index = room->get_messages_thread_unsafe().size();
         room->release_room_lock();
         return PluginResult::OK;
     }
@@ -1361,7 +1376,7 @@ namespace QuickMedia {
 
                 const rapidjson::Value &timeline_json = GetMember(it.value, "timeline");
                 if(timeline_json.IsObject()) {
-                    if(is_additional_messages_sync && !room->has_prev_batch()) {
+                    if(!sync_is_cache && !room->has_prev_batch()) {
                         // This may be non-existent if this is the first event in the room
                         const rapidjson::Value &prev_batch_json = GetMember(timeline_json, "prev_batch");
                         if(prev_batch_json.IsString())
@@ -1628,9 +1643,9 @@ namespace QuickMedia {
         return false;
     }
 
-    void Matrix::events_add_messages(const rapidjson::Value &events_json, RoomData *room_data, MessageDirection message_dir, bool has_unread_notifications) {
+    size_t Matrix::events_add_messages(const rapidjson::Value &events_json, RoomData *room_data, MessageDirection message_dir, bool has_unread_notifications) {
         if(!events_json.IsArray())
-            return;
+            return 0;
 
         // TODO: Preallocate
         std::vector<std::shared_ptr<Message>> new_messages;
@@ -1639,17 +1654,20 @@ namespace QuickMedia {
 
         for(const rapidjson::Value &event_item_json : events_json.GetArray()) {
             std::shared_ptr<Message> new_message = parse_message_event(event_item_json, room_data);
-            if(new_message)
+            if(new_message) {
+                new_message->cache = sync_is_cache;
                 new_messages.push_back(std::move(new_message));
+            }
         }
 
         if(new_messages.empty())
-            return;
+            return 0;
 
+        size_t num_new_messages = 0;
         if(message_dir == MessageDirection::BEFORE) {
-            room_data->prepend_messages_reverse(new_messages);
+            num_new_messages = room_data->prepend_messages_reverse(new_messages);
         } else if(message_dir == MessageDirection::AFTER) {
-            room_data->append_messages(new_messages);
+            num_new_messages = room_data->append_messages(new_messages);
         }
 
         if(has_unread_notifications) {
@@ -1670,6 +1688,8 @@ namespace QuickMedia {
 
         if(delegate)
             delegate->room_add_new_messages(room_data, new_messages, next_batch.empty(), sync_is_cache, message_dir);
+
+        return num_new_messages;
     }
 
     std::shared_ptr<Message> Matrix::parse_message_event(const rapidjson::Value &event_item_json, RoomData *room_data) {
@@ -2278,19 +2298,18 @@ namespace QuickMedia {
         }
     }
 
-    PluginResult Matrix::get_previous_room_messages(RoomData *room_data, bool latest_messages) {
+    PluginResult Matrix::get_previous_room_messages(RoomData *room_data, bool latest_messages, size_t &num_new_messages) {
+        num_new_messages = 0;
         std::string from = room_data->get_prev_batch();
         if(from.empty()) {
-            #if 0
             fprintf(stderr, "Info: missing previous batch for room: %s, using /sync next batch\n", room_data->id.c_str());
             // TODO: When caching /sync, remember to add lock around getting next_batch!
             from = get_next_batch();
             if(from.empty()) {
-                fprintf(stderr, "Error: missing next batch!\n");
-                return PluginResult::OK;
+                fprintf(stderr, "Warning: missing next batch, using END\n");
+                from = "END";
+                //return PluginResult::OK;
             }
-            #endif
-            from = "END";
         }
 
         rapidjson::Document request_data(rapidjson::kObjectType);
@@ -2321,7 +2340,7 @@ namespace QuickMedia {
         //events_set_room_name(state_json, room_data);
 
         const rapidjson::Value &chunk_json = GetMember(json_root, "chunk");
-        events_add_messages(chunk_json, room_data, MessageDirection::BEFORE, false);
+        num_new_messages = events_add_messages(chunk_json, room_data, MessageDirection::BEFORE, false);
 
         const rapidjson::Value &end_json = GetMember(json_root, "end");
         if(!end_json.IsString()) {
@@ -3302,7 +3321,7 @@ namespace QuickMedia {
             room->clear_data();
         }
         // We intentionally dont clear |rooms| here because we want the objects inside it to still be valid. TODO: Clear |rooms| here
-        room_data_by_id.clear();
+        //room_data_by_id.clear();
         invites.clear();
         delegate->clear_data();
     }
diff --git a/src/plugins/NyaaSi.cpp b/src/plugins/NyaaSi.cpp
index 082aaa8..2ed5f59 100644
--- a/src/plugins/NyaaSi.cpp
+++ b/src/plugins/NyaaSi.cpp
@@ -184,7 +184,7 @@ namespace QuickMedia {
 
         auto body = create_body();
         body->items = std::move(result_items);
-        result_tabs.push_back(Tab{std::move(body), std::make_unique<NyaaSiSearchPage>(program, strip(title), url), create_search_bar("Search...", 200)});
+        result_tabs.push_back(Tab{std::move(body), std::make_unique<NyaaSiSearchPage>(program, strip(title), url), create_search_bar("Search...", 300)});
         return PluginResult::OK;
     }