Matrix: fetch additional messages in the background for all rooms, filter room list when updated after sync

7 files changed, 144 insertions, 37 deletions

diff --git a/TODO b/TODO
index 35c463b..bd5869e 100644
--- a/TODO
+++ b/TODO
@@ -115,7 +115,6 @@ Room list in matrix ignores edited messages, which it should for unread messages
 For messages that mention us we only want a notification for the last edited version to show as a notification.
 Fetch replies/pinned message using multiple threads.
 Show in room tags list when there is a message in any of the rooms in the tag.
-Apply current search filter when adding new rooms to the room list.
 Cancel video download when pressing escape or closing window (important in matrix).
 Support webp.
 Show images while they download by showing them as scanlines starting from the top. Needed for slow websites such as 4chan.
@@ -134,6 +133,5 @@ Get user displayname, avatar, room name, etc updates from /sync and update them
 Test if glScissor doesn't break loading of embedded body item.
 Handle matrix token being invalidated while running.
 Update upload limit if its updated on the server (can it be updated while the server is running?).
-Apply search filter when updating rooms (including when switching from cache to server response sync data).
 Editing a reply removes reply formatting (both in body and formatted_body). Element also does this when you edit a reply twice. This breaks element mobile that is unable to display replied-to messages without correct formatting (doesn't fetch the replied-to message).
 Implement m.room.tombstone.
\ No newline at end of file
diff --git a/plugins/Fourchan.hpp b/plugins/Fourchan.hpp
index 3ee07dd..b9e5b74 100644
--- a/plugins/Fourchan.hpp
+++ b/plugins/Fourchan.hpp
@@ -8,6 +8,7 @@ namespace QuickMedia {
         FourchanBoardsPage(Program *program, std::string resources_root) : Page(program), resources_root(std::move(resources_root)) {}
         const char* get_title() const override { return "Select board"; }
         PluginResult submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) override;
+        bool clear_search_after_submit() override { return true; }
         void get_boards(BodyItems &result_items);
 
         const std::string resources_root;
diff --git a/plugins/Matrix.hpp b/plugins/Matrix.hpp
index 78725a5..812c1a0 100644
--- a/plugins/Matrix.hpp
+++ b/plugins/Matrix.hpp
@@ -259,6 +259,7 @@ namespace QuickMedia {
 
         const char* get_title() const override { return title.c_str(); }
         PluginResult submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) override;
+        bool clear_search_after_submit() override { return true; }
 
         void on_navigate_to_page(Body *body) override;
 
@@ -274,6 +275,7 @@ namespace QuickMedia {
         void sort_rooms();
 
         MatrixQuickMedia *matrix_delegate = nullptr;
+        bool filter_on_update = false;
     private:
         std::mutex mutex;
         std::vector<std::shared_ptr<BodyItem>> room_body_items;
@@ -289,9 +291,10 @@ namespace QuickMedia {
 
     class MatrixRoomTagsPage : public Page {
     public:
-        MatrixRoomTagsPage(Program *program, Body *body) : Page(program), body(body) {}
+        MatrixRoomTagsPage(Program *program, Body *body, SearchBar *search_bar) : Page(program), body(body), search_bar(search_bar) {}
         const char* get_title() const override { return "Tags"; }
         PluginResult submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) override;
+        bool clear_search_after_submit() override { return true; }
 
         void update() override;
         void add_room_body_item_to_tag(std::shared_ptr<BodyItem> body_item, const std::string &tag);
@@ -306,6 +309,7 @@ namespace QuickMedia {
         void sort_rooms();
 
         MatrixQuickMedia *matrix_delegate = nullptr;
+        bool filter_on_update = false;
     private:
         struct TagData {
             std::shared_ptr<BodyItem> tag_item;
@@ -319,20 +323,23 @@ namespace QuickMedia {
         std::map<std::string, std::vector<std::shared_ptr<BodyItem>>> remove_room_body_items_by_tags;
         MatrixRoomsPage *current_rooms_page = nullptr;
         bool clear_data_on_update = false;
+        SearchBar *search_bar = nullptr;
     };
 
     class MatrixInvitesPage : public Page {
     public:
-        MatrixInvitesPage(Program *program, Matrix *matrix, Body *body);
+        MatrixInvitesPage(Program *program, Matrix *matrix, Body *body, SearchBar *search_bar);
 
         const char* get_title() const override { return title.c_str(); }
         PluginResult submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) override;
+        bool clear_search_after_submit() override { return true; }
 
         void update() override;
         void add_body_item(std::shared_ptr<BodyItem> body_item);
         void remove_body_item_by_room_id(const std::string &room_id);
 
         void clear_data();
+        bool filter_on_update = false;
     private:
         Matrix *matrix;
         std::mutex mutex;
@@ -342,6 +349,7 @@ namespace QuickMedia {
         std::string title = "Invites (0)";
         size_t prev_invite_count = 0;
         bool clear_data_on_update = false;
+        SearchBar *search_bar = nullptr;
     };
 
     class MatrixInviteDetailsPage : public Page {
@@ -446,10 +454,10 @@ namespace QuickMedia {
 
         bool use_tor = false;
     private:
-        PluginResult parse_sync_response(const rapidjson::Document &root);
+        PluginResult parse_sync_response(const rapidjson::Document &root, bool is_additional_messages_sync);
         PluginResult parse_notifications(const rapidjson::Value &notifications_json);
         PluginResult parse_sync_account_data(const rapidjson::Value &account_data_json, std::optional<std::set<std::string>> &dm_rooms);
-        PluginResult parse_sync_room_data(const rapidjson::Value &rooms_json);
+        PluginResult parse_sync_room_data(const rapidjson::Value &rooms_json, bool is_additional_messages_sync);
         PluginResult get_previous_room_messages(RoomData *room_data, bool latest_messages);
         void events_add_user_info(const rapidjson::Value &events_json, RoomData *room_data);
         std::shared_ptr<UserInfo> parse_user_info(const rapidjson::Value &json, const std::string &user_id, RoomData *room_data);
@@ -465,7 +473,8 @@ namespace QuickMedia {
         PluginResult upload_file(RoomData *room, const std::string &filepath, UploadInfo &file_info, UploadInfo &thumbnail_info, std::string &err_msg);
         void add_room(std::unique_ptr<RoomData> room);
         void remove_room(const std::string &room_id);
-        void set_invite(const std::string &room_id, Invite invite);
+        // Returns false if an invite to the room already exists
+        bool set_invite(const std::string &room_id, Invite invite);
         // Returns true if an invite for |room_id| exists
         bool remove_invite(const std::string &room_id);
         void set_next_batch(std::string new_next_batch);
@@ -489,6 +498,7 @@ namespace QuickMedia {
         std::mutex invite_mutex;
 
         std::thread sync_thread;
+        std::thread sync_additional_messages_thread;
         std::thread notification_thread;
         bool sync_running = false;
         bool sync_failed = false;
diff --git a/plugins/NyaaSi.hpp b/plugins/NyaaSi.hpp
index bad5863..13cc437 100644
--- a/plugins/NyaaSi.hpp
+++ b/plugins/NyaaSi.hpp
@@ -8,6 +8,7 @@ namespace QuickMedia {
         NyaaSiCategoryPage(Program *program) : Page(program) {}
         const char* get_title() const override { return "Select category"; }
         PluginResult submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) override;
+        bool clear_search_after_submit() override { return true; }
         void get_categories(BodyItems &result_items);
     };
 
diff --git a/plugins/Page.hpp b/plugins/Page.hpp
index 28c2bd6..54e7383 100644
--- a/plugins/Page.hpp
+++ b/plugins/Page.hpp
@@ -35,6 +35,7 @@ namespace QuickMedia {
             (void)result_tabs;
             return PluginResult::ERR;
         }
+        virtual bool clear_search_after_submit() { return false; }
         // Note: If pagination is done by fetching the next page until we get to |page|, then the "current page" should be reset everytime |search| is called.
         // Note: the first page is 0
         virtual PluginResult get_page(const std::string &str, int page, BodyItems &result_items) { (void)str; (void)page; (void)result_items; return PluginResult::OK; }
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 865f6fe..5d980f3 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -981,6 +981,11 @@ namespace QuickMedia {
             if(after_submit_handler)
                 after_submit_handler();
 
+            if(tabs[selected_tab].page->clear_search_after_submit() && tabs[selected_tab].search_bar) {
+                tabs[selected_tab].search_bar->clear();
+                tabs[selected_tab].search_bar->onTextUpdateCallback("");
+            }
+
             if(tabs[selected_tab].page->is_single_page()) {
                 tabs[selected_tab].search_bar->clear();
                 if(new_tabs.size() == 1)
@@ -3224,16 +3229,27 @@ namespace QuickMedia {
             }
         };
 
-        auto process_pinned_events = [&tabs](const std::optional<std::vector<std::string>> &pinned_events) {
-            if(!pinned_events || pinned_events->empty())
+        auto pinned_body_items_contains_event = [&tabs](const std::string &event_id) {
+            for(auto &body_item : tabs[PINNED_TAB_INDEX].body->items) {
+                if(static_cast<PinnedEventData*>(body_item->userdata)->event_id == event_id)
+                    return true;
+            }
+            return false;
+        };
+
+        auto process_pinned_events = [&tabs, &pinned_body_items_contains_event](const std::optional<std::vector<std::string>> &pinned_events) {
+            if(!pinned_events)
                 return;
 
             bool empty_before = tabs[PINNED_TAB_INDEX].body->items.empty();
             int selected_before = tabs[PINNED_TAB_INDEX].body->get_selected_item();
-            tabs[PINNED_TAB_INDEX].body->items.clear();
+            auto prev_pinned_body_items = tabs[PINNED_TAB_INDEX].body->items;
 
             // TODO: Add message to rooms messages when there are new pinned events
             for(const std::string &event : pinned_events.value()) {
+                if(pinned_body_items_contains_event(event))
+                    continue;
+
                 auto body = BodyItem::create("");
                 body->set_description("Loading message...");
                 PinnedEventData *event_data = new PinnedEventData();
@@ -3243,6 +3259,11 @@ namespace QuickMedia {
                 tabs[PINNED_TAB_INDEX].body->items.push_back(std::move(body));
             }
 
+            for(auto &prev_body_item : prev_pinned_body_items) {
+                if(!pinned_body_items_contains_event(static_cast<PinnedEventData*>(prev_body_item->userdata)->event_id))
+                    delete (PinnedEventData*)prev_body_item->userdata;
+            }
+
             if(empty_before)
                 tabs[PINNED_TAB_INDEX].body->select_last_item();
             else
@@ -3610,7 +3631,7 @@ namespace QuickMedia {
             return false;
         };
 
-        auto cleanup_tasks = [&set_read_marker_future, &fetch_message_future, &typing_state_queue, &typing_state_thread, &post_task_queue, &post_thread, &unreferenced_event_by_room, &tabs]() {
+        auto cleanup_tasks = [&set_read_marker_future, &fetch_message_future, &typing_state_queue, &typing_state_thread, &post_task_queue, &post_thread, &tabs]() {
             set_read_marker_future.cancel();
             fetch_message_future.cancel();
             typing_state_queue.close();
@@ -3630,6 +3651,7 @@ namespace QuickMedia {
                 for(auto &body_item : tabs[PINNED_TAB_INDEX].body->items) {
                     delete (PinnedEventData*)body_item->userdata;
                 }
+                tabs[PINNED_TAB_INDEX].body->clear_items();
             }
 
             //tabs.clear();
@@ -4281,7 +4303,6 @@ namespace QuickMedia {
                     std::vector<std::string> pinned_events;
                     matrix->get_all_pinned_events(current_room, pinned_events);
                     process_pinned_events(std::move(pinned_events));
-                    tabs[PINNED_TAB_INDEX].body->select_last_item();
                     typing_state_queue.restart();
                     typing_state_thread = std::thread(typing_state_handler);
                     post_task_queue.restart();
@@ -4314,11 +4335,13 @@ namespace QuickMedia {
 
         auto rooms_tags_body = create_body();
         rooms_tags_body->thumbnail_mask_shader = &circle_mask_shader;
-        auto matrix_rooms_tag_page = std::make_unique<MatrixRoomTagsPage>(this, rooms_tags_body.get());
+        auto matrix_rooms_tage_page_search_bar = create_search_bar("Search...", SEARCH_DELAY_FILTER);
+        auto matrix_rooms_tag_page = std::make_unique<MatrixRoomTagsPage>(this, rooms_tags_body.get(), matrix_rooms_tage_page_search_bar.get());
 
         auto invites_body = create_body();
         invites_body->thumbnail_mask_shader = &circle_mask_shader;
-        auto matrix_invites_page = std::make_unique<MatrixInvitesPage>(this, matrix, invites_body.get());
+        auto matrix_invites_page_search_bar = create_search_bar("Search...", SEARCH_DELAY_FILTER);
+        auto matrix_invites_page = std::make_unique<MatrixInvitesPage>(this, matrix, invites_body.get(), matrix_invites_page_search_bar.get());
 
         MatrixQuickMedia matrix_handler(this, matrix, matrix_rooms_page.get(), matrix_rooms_tag_page.get(), matrix_invites_page.get());
         bool sync_cached = false;
@@ -4327,8 +4350,8 @@ namespace QuickMedia {
 
         std::vector<Tab> tabs;
         tabs.push_back(Tab{std::move(rooms_body), std::move(matrix_rooms_page), std::move(matrix_rooms_page_search_bar)});
-        tabs.push_back(Tab{std::move(rooms_tags_body), std::move(matrix_rooms_tag_page), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
-        tabs.push_back(Tab{std::move(invites_body), std::move(matrix_invites_page), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
+        tabs.push_back(Tab{std::move(rooms_tags_body), std::move(matrix_rooms_tag_page), std::move(matrix_rooms_tage_page_search_bar)});
+        tabs.push_back(Tab{std::move(invites_body), std::move(matrix_invites_page), std::move(matrix_invites_page_search_bar)});
 
         while(window.isOpen()) {
             page_loop(tabs);
diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp
index 9cfc1bb..32700a7 100644
--- a/src/plugins/Matrix.cpp
+++ b/src/plugins/Matrix.cpp
@@ -21,7 +21,8 @@
 static const char* SERVICE_NAME = "matrix";
 static const char* OTHERS_ROOM_TAG = "tld.name.others";
 // Filter without account data
-static const char* FILTER = "{\"presence\":{\"limit\":0,\"types\":[\"\"]},\"account_data\":{\"limit\":0,\"types\":[\"\"]},\"room\":{\"state\":{\"not_types\":[\"m.room.related_groups\",\"m.room.power_levels\",\"m.room.join_rules\",\"m.room.history_visibility\"],\"lazy_load_members\":true},\"timeline\":{\"limit\":1,\"lazy_load_members\":true},\"ephemeral\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true},\"account_data\":{\"limit\":1,\"types\":[\"m.fully_read\",\"m.tag\"],\"lazy_load_members\":true}}}";
+static const char* INITIAL_FILTER = "{\"presence\":{\"limit\":0,\"types\":[\"\"]},\"account_data\":{\"limit\":0,\"types\":[\"\"]},\"room\":{\"state\":{\"not_types\":[\"m.room.related_groups\",\"m.room.power_levels\",\"m.room.join_rules\",\"m.room.history_visibility\"],\"lazy_load_members\":true},\"timeline\":{\"limit\":1,\"lazy_load_members\":true},\"ephemeral\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true},\"account_data\":{\"limit\":1,\"types\":[\"m.fully_read\",\"m.tag\"],\"lazy_load_members\":true}}}";
+static const char* CONTINUE_FILTER = "{\"presence\":{\"limit\":0,\"types\":[\"\"]},\"account_data\":{\"limit\":0,\"types\":[\"\"]},\"room\":{\"state\":{\"not_types\":[\"m.room.related_groups\",\"m.room.power_levels\",\"m.room.join_rules\",\"m.room.history_visibility\"],\"lazy_load_members\":true},\"timeline\":{\"limit\":20,\"lazy_load_members\":true},\"ephemeral\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true},\"account_data\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true}}}";
 
 static rapidjson::Value nullValue(rapidjson::kNullType);
 static const rapidjson::Value& GetMember(const rapidjson::Value &obj, const char *key) {
@@ -217,7 +218,7 @@ namespace QuickMedia {
 
     void RoomData::clear_data() {
         std::lock_guard<std::recursive_mutex> lock(room_mutex);
-        fetched_messages_by_event_id.clear();
+        //fetched_messages_by_event_id.clear();
         user_info_by_user_id.clear();
         messages.clear();
         messages_read_index = 0;
@@ -469,6 +470,9 @@ namespace QuickMedia {
                 body->clear_items();
             }
 
+            if(!pending_remove_body_items.empty() || !room_body_items.empty())
+                filter_on_update = true;
+
             for(const std::string &room_id : pending_remove_body_items) {
                 remove_body_item_by_url(body->items, room_id);
                 // TODO: There can be a race condition where current_chat_page is set after entering a room and then we will enter a room we left
@@ -488,6 +492,11 @@ namespace QuickMedia {
             sort_room_body_items(body->items);
         }
         matrix_delegate->update(MatrixPageType::ROOM_LIST);
+        if(filter_on_update) {
+            filter_on_update = false;
+            if(search_bar)
+                body->filter_search_fuzzy(search_bar->get_text());
+        }
     }
 
     void MatrixRoomsPage::add_body_item(std::shared_ptr<BodyItem> body_item) {
@@ -578,8 +587,10 @@ namespace QuickMedia {
 
                 if(tag_body_it->second.room_body_items.empty()) {
                     auto room_body_item_it = std::find(body->items.begin(), body->items.end(), tag_body_it->second.tag_item);
-                    if(room_body_item_it != body->items.end())
+                    if(room_body_item_it != body->items.end()) {
                         body->items.erase(room_body_item_it);
+                        filter_on_update = true;
+                    }
                     tag_body_items_by_name.erase(tag_body_it);
                 }
             }
@@ -598,6 +609,7 @@ namespace QuickMedia {
                         body->items.push_back(tag_body_item);
                         tag_data = &tag_body_items_by_name[it.first];
                         tag_data->tag_item = tag_body_item;
+                        filter_on_update = true;
                     }
                 } else {
                     tag_data = &tag_body_it->second;
@@ -610,6 +622,11 @@ namespace QuickMedia {
             add_room_body_items_by_tags.clear();
         }
         matrix_delegate->update(MatrixPageType::ROOM_LIST);
+        if(filter_on_update) {
+            filter_on_update = false;
+            if(search_bar)
+                body->filter_search_fuzzy(search_bar->get_text());
+        }
     }
 
     void MatrixRoomTagsPage::add_room_body_item_to_tag(std::shared_ptr<BodyItem> body_item, const std::string &tag) {
@@ -662,7 +679,7 @@ namespace QuickMedia {
             current_rooms_page->sort_rooms();
     }
 
-    MatrixInvitesPage::MatrixInvitesPage(Program *program, Matrix *matrix, Body *body) : Page(program), matrix(matrix), body(body) {
+    MatrixInvitesPage::MatrixInvitesPage(Program *program, Matrix *matrix, Body *body, SearchBar *search_bar) : Page(program), matrix(matrix), body(body), search_bar(search_bar) {
 
     }
 
@@ -706,6 +723,9 @@ namespace QuickMedia {
             body->clear_items();
         }
 
+        if(!pending_remove_body_items.empty() || !body_items.empty())
+            filter_on_update = true;
+
         for(const std::string &room_id : pending_remove_body_items) {
             remove_body_item_by_url(body->items, room_id);
         }
@@ -718,6 +738,12 @@ namespace QuickMedia {
             prev_invite_count = body->items.size();
             title = "Invites (" + std::to_string(body->items.size()) + ")";
         }
+
+        if(filter_on_update) {
+            filter_on_update = false;
+            if(search_bar)
+                body->filter_search_fuzzy(search_bar->get_text());
+        }
     }
 
     void MatrixInvitesPage::add_body_item(std::shared_ptr<BodyItem> body_item) {
@@ -847,7 +873,7 @@ namespace QuickMedia {
                     rapidjson::ParseResult parse_result = doc.ParseStream<rapidjson::kParseStopWhenDoneFlag>(is);
                     if(parse_result.IsError())
                         break;
-                    if(parse_sync_response(doc) != PluginResult::OK)
+                    if(parse_sync_response(doc, false) != PluginResult::OK)
                         fprintf(stderr, "Failed to parse cached sync response\n");
                 }
                 fclose(sync_cache_file);
@@ -857,6 +883,7 @@ namespace QuickMedia {
             // Filter with account data
             // {"presence":{"limit":0,"types":[""]},"account_data":{"not_types":["im.vector.setting.breadcrumbs","m.push_rules","im.vector.setting.allowed_widgets","io.element.recent_emoji"]},"room":{"state":{"limit":1,"not_types":["m.room.related_groups","m.room.power_levels","m.room.join_rules","m.room.history_visibility"],"lazy_load_members":true},"timeline":{"limit":3,"lazy_load_members":true},"ephemeral":{"limit":0,"types":[""],"lazy_load_members":true},"account_data":{"limit":1,"types":["m.fully_read"],"lazy_load_members":true}}}
 
+#if 0
             bool filter_cached = false;
             Path filter_path = get_storage_dir().join("matrix").join("filter");
             if(get_file_type(filter_path) == FileType::REGULAR)
@@ -869,13 +896,39 @@ namespace QuickMedia {
                 filter = get_filter_cached();
             else
                 filter = FILTER;
-            std::string filter_encoded = url_param_encode(filter);
+#endif
+            std::string filter_encoded = url_param_encode(INITIAL_FILTER);
 
             std::vector<CommandArg> additional_args = {
                 { "-H", "Authorization: Bearer " + access_token },
                 { "-m", "35" }
             };
 
+            sync_additional_messages_thread = std::thread([this]() {
+                std::vector<CommandArg> additional_args = {
+                    { "-H", "Authorization: Bearer " + access_token },
+                    { "-m", "35" }
+                };
+
+                char url[1024];
+                std::string filter_encoded = url_param_encode(CONTINUE_FILTER);
+                snprintf(url, sizeof(url), "%s/_matrix/client/r0/sync?filter=%s&timeout=0", homeserver.c_str(), filter_encoded.c_str());
+
+                rapidjson::Document json_root;
+                std::string err_msg;
+                DownloadResult download_result = download_json(json_root, url, additional_args, true, &err_msg);
+                if(download_result != DownloadResult::OK) {
+                    fprintf(stderr, "/sync for additional messages failed\n");
+                    return;
+                }
+
+                // TODO: Test?
+                //if(next_batch.empty())
+                //    clear_sync_cache_for_new_sync();
+
+                parse_sync_response(json_root, true);
+            });
+
             const rapidjson::Value *next_batch_json;
             PluginResult result;
             bool initial_sync = true;
@@ -916,7 +969,7 @@ namespace QuickMedia {
                 if(next_batch.empty())
                     clear_sync_cache_for_new_sync();
 
-                result = parse_sync_response(json_root);
+                result = parse_sync_response(json_root, false);
                 if(result != PluginResult::OK) {
                     fprintf(stderr, "Failed to parse sync response\n");
                     goto sync_end;
@@ -952,12 +1005,16 @@ namespace QuickMedia {
                         const rapidjson::Value &notification_json = GetMember(json_root, "notifications");
                         parse_notifications(notification_json);
                     });
+
+                    filter_encoded = url_param_encode(CONTINUE_FILTER);
                 }
 
+#if 0
                 if(!filter_cached) {
                     filter_cached = true;
                     filter_encoded = url_param_encode(get_filter_cached());
                 }
+#endif
 
                 sync_end:
                 if(sync_running)
@@ -987,6 +1044,11 @@ namespace QuickMedia {
             program_kill_in_thread(sync_thread.get_id());
             sync_thread.join();
         }
+
+        if(sync_additional_messages_thread.joinable()) {
+            program_kill_in_thread(sync_additional_messages_thread.get_id());
+            sync_additional_messages_thread.join();
+        }
         
         if(notification_thread.joinable()) {
             program_kill_in_thread(notification_thread.get_id());
@@ -1064,7 +1126,7 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    PluginResult Matrix::parse_sync_response(const rapidjson::Document &root) {
+    PluginResult Matrix::parse_sync_response(const rapidjson::Document &root, bool is_additional_messages_sync) {
         if(!root.IsObject())
             return PluginResult::ERR;
 
@@ -1074,7 +1136,7 @@ namespace QuickMedia {
         // TODO: Include "Direct messages" as a tag using |dm_rooms| above
 
         const rapidjson::Value &rooms_json = GetMember(root, "rooms");
-        parse_sync_room_data(rooms_json);
+        parse_sync_room_data(rooms_json, is_additional_messages_sync);
 
         return PluginResult::OK;
     }
@@ -1173,7 +1235,7 @@ namespace QuickMedia {
         return PluginResult::OK;
     }
 
-    PluginResult Matrix::parse_sync_room_data(const rapidjson::Value &rooms_json) {
+    PluginResult Matrix::parse_sync_room_data(const rapidjson::Value &rooms_json, bool is_additional_messages_sync) {
         if(!rooms_json.IsObject())
             return PluginResult::OK;
 
@@ -1204,14 +1266,15 @@ namespace QuickMedia {
                     const rapidjson::Value &events_json = GetMember(state_json, "events");
                     events_add_user_info(events_json, room);
                     events_set_room_name(events_json, room);
-                    events_add_pinned_events(events_json, room);
+                    if(!is_additional_messages_sync)
+                        events_add_pinned_events(events_json, room);
                 }
 
                 const rapidjson::Value &account_data_json = GetMember(it.value, "account_data");
 
                 const rapidjson::Value &timeline_json = GetMember(it.value, "timeline");
                 if(timeline_json.IsObject()) {
-                    if(!room->has_prev_batch()) {
+                    if(is_additional_messages_sync && !room->has_prev_batch()) {
                         // This may be non-existent if this is the first event in the room
                         const rapidjson::Value &prev_batch_json = GetMember(timeline_json, "prev_batch");
                         if(prev_batch_json.IsString())
@@ -1243,7 +1306,8 @@ namespace QuickMedia {
                         delegate->join_room(room);
 
                     events_add_messages(events_json, room, MessageDirection::AFTER, has_unread_notifications);
-                    events_add_pinned_events(events_json, room);
+                    if(!is_additional_messages_sync)
+                        events_add_pinned_events(events_json, room);
                 } else {
                     if(account_data_json.IsObject()) {
                         const rapidjson::Value &events_json = GetMember(account_data_json, "events");
@@ -1277,11 +1341,13 @@ namespace QuickMedia {
             }
         }
 
-        const rapidjson::Value &leave_json = GetMember(rooms_json, "leave");
-        remove_rooms(leave_json);
+        if(!is_additional_messages_sync) {
+            const rapidjson::Value &leave_json = GetMember(rooms_json, "leave");
+            remove_rooms(leave_json);
 
-        const rapidjson::Value &invite_json = GetMember(rooms_json, "invite");
-        add_invites(invite_json);
+            const rapidjson::Value &invite_json = GetMember(rooms_json, "invite");
+            add_invites(invite_json);
+        }
 
         return PluginResult::OK;
     }
@@ -1881,6 +1947,7 @@ namespace QuickMedia {
         bool has_room_name = room_data->has_name();
         bool has_room_avatar_url = room_data->has_avatar_url();
 
+        // TODO: Fix. This can be incorrect because this method can be called before we have added all users to the room
         std::vector<std::shared_ptr<UserInfo>> users_excluding_me;
         if(!has_room_name || !has_room_avatar_url)
             users_excluding_me = room_data->get_users_excluding_me(user_id); // TODO: What about thread safety with user_id? its reset in /logout
@@ -2083,8 +2150,8 @@ namespace QuickMedia {
                     invite.new_invite = !next_batch.empty();
 
                     std::string room_id_str(room_id.GetString(), room_id.GetStringLength());
-                    delegate->add_invite(room_id_str, invite);
-                    set_invite(room_id_str, std::move(invite));
+                    if(set_invite(room_id_str, invite))
+                        delegate->add_invite(room_id_str, std::move(invite));
                     break;
                 }
             }
@@ -3149,9 +3216,10 @@ namespace QuickMedia {
         room_data_by_id.erase(room_it);
     }
 
-    void Matrix::set_invite(const std::string &room_id, Invite invite) {
+    bool Matrix::set_invite(const std::string &room_id, Invite invite) {
         std::lock_guard<std::mutex> lock(invite_mutex);
-        invites[room_id] = std::move(invite);
+        auto res = invites.insert(std::make_pair(room_id, std::move(invite)));
+        return res.second;
     }
 
     bool Matrix::remove_invite(const std::string &room_id) {
@@ -3219,6 +3287,7 @@ namespace QuickMedia {
 
     // TODO: GET the filter to check if its valid?
     std::string Matrix::get_filter_cached() {
+        #if 0
         if(filter_cached)
             return filter_cached.value();
 
@@ -3255,6 +3324,10 @@ namespace QuickMedia {
             file_overwrite_atomic(filter_path, filter_cached.value());
         }
         return filter_cached.value();
+        #else
+        assert(false);
+        return INITIAL_FILTER;
+        #endif
     }
 
     DownloadResult Matrix::download_json(rapidjson::Document &result, const std::string &url, std::vector<CommandArg> additional_args, bool use_browser_useragent, std::string *err_msg) const {