Use matrix global account data for qm read marker to bypass synapse issue with /sync

3 files changed, 30 insertions, 44 deletions

diff --git a/TODO b/TODO
index beea673..87df967 100644
--- a/TODO
+++ b/TODO
@@ -244,4 +244,5 @@ Nicer custom emoji upload options - Selecting name, size, cropping.
 Add option to copy somebody else custom emoji in matrix by pressing enter to bring up message option and in that a list of every emoji in the message should be added with "Add (emoji)..." text to add the emoji.
 Detect invidious urls too, even the ones that dont have watch?v=.. this could be done by downloading the webpage (maybe only HEAD?) to check if it's invidious.
 Atomic file operations should use a random generated name instead of .tmp, because multiple instances of quickmedia may be running and they may try to write to the same file at the same time. In such cases they can also write to the same temporary file at the same time.
-TODO: https://github.com/matrix-org/synapse/issues/14444.
\ No newline at end of file
+TODO: https://github.com/matrix-org/synapse/issues/14444.
+Use matrix /sync "since" param. Its beneficial even to quickmedia because synapse is written in such a way that using "since" is faster.
\ No newline at end of file
diff --git a/plugins/Matrix.hpp b/plugins/Matrix.hpp
index 671f718..e5c86e8 100644
--- a/plugins/Matrix.hpp
+++ b/plugins/Matrix.hpp
@@ -722,8 +722,7 @@ namespace QuickMedia {
         PluginResult set_pinned_events(RoomData *room, const std::vector<std::string> &pinned_events, bool is_add);
 
         PluginResult set_qm_last_read_message_timestamp(RoomData *room, int64_t timestamp);
-        void load_qm_read_markers_from_cache();
-        void update_room_qm_read_markers_in_cache(const std::string &room_id, int64_t timestamp);
+        void load_qm_read_markers_from_account_data();
 
         PluginResult parse_sync_response(const rapidjson::Document &root, bool is_additional_messages_sync, bool initial_sync);
         PluginResult parse_notifications(const rapidjson::Value &notifications_json, std::function<void(const MatrixNotification&)> callback_func);
@@ -734,7 +733,7 @@ namespace QuickMedia {
         PluginResult get_previous_room_messages(RoomData *room_data, bool latest_messages, size_t &num_new_messages, bool *reached_end = nullptr);
         void events_add_user_info(const rapidjson::Value &events_json, RoomData *room_data, int64_t timestamp);
         std::shared_ptr<UserInfo> parse_user_info(const rapidjson::Value &json, const std::string &user_id, RoomData *room_data, int64_t timestamp);
-        void events_set_user_read_marker(const rapidjson::Value &events_json, RoomData *room_data, std::shared_ptr<UserInfo> &me);
+        void events_set_user_read_marker(const rapidjson::Value &events_json, RoomData *room_data, std::shared_ptr<UserInfo> &me, bool is_additional_messages_sync);
         // Returns the number of messages added
         size_t events_add_messages(const rapidjson::Value &events_json, RoomData *room_data, MessageDirection message_dir, bool has_unread_notifications);
         void events_set_room_info(const rapidjson::Value &events_json, RoomData *room_data, int64_t timestamp);
diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp
index d3c6f81..ec883df 100644
--- a/src/plugins/Matrix.cpp
+++ b/src/plugins/Matrix.cpp
@@ -1550,7 +1550,6 @@ namespace QuickMedia {
 
         load_silenced_invites();
         load_custom_emoji_from_cache();
-        load_qm_read_markers_from_cache();
 
         sync_thread = std::thread([this, matrix_cache_dir]() {
             sync_is_cache = true;
@@ -1570,6 +1569,8 @@ namespace QuickMedia {
             }
             sync_is_cache = false;
 
+            load_qm_read_markers_from_account_data(); // TODO: Remove when https://github.com/matrix-org/synapse/issues/14444 is fixed, if ever.
+
             // Filter with account data
             // {"presence":{"limit":0,"types":[""]},"account_data":{"not_types":["im.vector.setting.breadcrumbs","m.push_rules","im.vector.setting.allowed_widgets","io.element.recent_emoji"]},"room":{"state":{"limit":1,"not_types":["m.room.related_groups","m.room.power_levels","m.room.join_rules","m.room.history_visibility"],"lazy_load_members":true},"timeline":{"limit":3,"lazy_load_members":true},"ephemeral":{"limit":0,"types":[""],"lazy_load_members":true},"account_data":{"limit":1,"types":["m.fully_read"],"lazy_load_members":true}}}
 
@@ -2174,7 +2175,7 @@ namespace QuickMedia {
                     if(account_data_json.IsObject()) {
                         const rapidjson::Value &events_json = GetMember(account_data_json, "events");
                         auto me = get_me(room);
-                        events_set_user_read_marker(events_json, room, me);
+                        events_set_user_read_marker(events_json, room, me, is_additional_messages_sync);
                     }
 
                     if(is_new_room)
@@ -2187,7 +2188,7 @@ namespace QuickMedia {
                     if(account_data_json.IsObject()) {
                         const rapidjson::Value &events_json = GetMember(account_data_json, "events");
                         auto me = get_me(room);
-                        events_set_user_read_marker(events_json, room, me);
+                        events_set_user_read_marker(events_json, room, me, is_additional_messages_sync);
                     }
 
                     if(is_new_room)
@@ -2355,7 +2356,7 @@ namespace QuickMedia {
         return user_info;
     }
 
-    void Matrix::events_set_user_read_marker(const rapidjson::Value &events_json, RoomData *room_data, std::shared_ptr<UserInfo> &me) {
+    void Matrix::events_set_user_read_marker(const rapidjson::Value &events_json, RoomData *room_data, std::shared_ptr<UserInfo> &me, bool is_additional_messages_sync) {
         assert(me); // TODO: Remove read marker from user and set it for the room instead. We need that in the matrix pages also
         if(!events_json.IsArray() || !me)
             return;
@@ -2376,9 +2377,10 @@ namespace QuickMedia {
                 const rapidjson::Value &event_id_json = GetMember(content_json, "event_id");
                 if(!event_id_json.IsString())
                     continue;
-
-                room_data->set_user_read_marker(me, std::string(event_id_json.GetString(), event_id_json.GetStringLength()));
-            } else if(strcmp(type_json.GetString(), "qm.last_read_message_timestamp") == 0) {
+                
+                if(!sync_is_cache && !is_additional_messages_sync)
+                    room_data->set_user_read_marker(me, std::string(event_id_json.GetString(), event_id_json.GetStringLength()));
+            } else if(strcmp(type_json.GetString(), "qm.last_read_message_timestamp") == 0) { // TODO: Remove qm.last_read_message_timestamp in room level eventually when everybody has data in global level
                 const rapidjson::Value &content_json = GetMember(event_json, "content");
                 if(!content_json.IsObject())
                     continue;
@@ -4812,8 +4814,6 @@ namespace QuickMedia {
         Path custom_emoji_path = get_cache_dir().join("matrix").join("custom_emoji.json");
         remove(custom_emoji_path.data.c_str());
 
-        Path read_markers_path = get_cache_dir().join("matrix").join("read_markers.json");
-        remove(read_markers_path.data.c_str());
         //Path filter_cache_path = get_storage_dir().join("matrix").join("filter");
         //remove(filter_cache_path.data.c_str());
         for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath, FileType) {
@@ -5189,7 +5189,10 @@ namespace QuickMedia {
 
     PluginResult Matrix::set_qm_last_read_message_timestamp(RoomData *room, int64_t timestamp) {
         rapidjson::Document request_data(rapidjson::kObjectType);
-        request_data.AddMember("timestamp", timestamp, request_data.GetAllocator());
+        qm_read_markers_by_room_cache[room->id] = timestamp;
+        for(const auto &[key, val] : qm_read_markers_by_room_cache) {
+            request_data.AddMember(rapidjson::Value(key.c_str(), request_data.GetAllocator()).Move(), val, request_data.GetAllocator());
+        }
 
         rapidjson::StringBuffer buffer;
         rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
@@ -5203,33 +5206,32 @@ namespace QuickMedia {
         };
 
         std::string server_response;
-        DownloadResult download_result = download_to_string(homeserver + "/_matrix/client/r0/user/" + my_user_id + "/rooms/" + room->id + "/account_data/qm.last_read_message_timestamp", server_response, std::move(additional_args), true);
+        DownloadResult download_result = download_to_string(homeserver + "/_matrix/client/r0/user/" + my_user_id + "/account_data/qm.last_read_message_timestamp", server_response, std::move(additional_args), true);
         if(download_result != DownloadResult::OK)
             return download_result_to_plugin_result(download_result);
 
         room->read_marker_event_timestamp = timestamp;
-        update_room_qm_read_markers_in_cache(room->id, timestamp);
+        qm_read_markers_by_room_cache[room->id] = timestamp;
         return PluginResult::OK;
     }
 
-    // TODO: Separate file for each room?
-    void Matrix::load_qm_read_markers_from_cache() {
-        std::string file_content;
-        if(file_get_content(get_cache_dir().join("matrix").join("read_markers.json"), file_content) != 0)
-            return;
+    void Matrix::load_qm_read_markers_from_account_data() {
+        std::vector<CommandArg> additional_args = {
+            { "-H", "content-type: application/json" },
+            { "-H", "Authorization: Bearer " + access_token }
+        };
 
         rapidjson::Document json_root;
-        rapidjson::ParseResult parse_result = json_root.Parse(file_content.c_str(), file_content.size());
-        if(parse_result.IsError()) {
-            fprintf(stderr, "Warning: failed to parse read_markers.json, error: %d\n", parse_result.Code());
+        std::string err_msg;
+        DownloadResult download_result = download_json(json_root, homeserver + "/_matrix/client/r0/user/" + my_user_id + "/account_data/qm.last_read_message_timestamp", std::move(additional_args), true, &err_msg);
+        if(download_result != DownloadResult::OK) {
+            fprintf(stderr, "Warning: failed to get account qm.last_read_message_timestamp\n");
             return;
         }
 
-        if(!json_root.IsObject()) {
-            fprintf(stderr, "Warning: failed to parse read_markers.json\n");
+        if(!json_root.IsObject())
             return;
-        }
-        
+
         qm_read_markers_by_room_cache.clear();
         std::lock_guard<std::recursive_mutex> lock(room_data_mutex);
         for(auto const &obj : json_root.GetObject()) {
@@ -5244,22 +5246,6 @@ namespace QuickMedia {
         }
     }
 
-    void Matrix::update_room_qm_read_markers_in_cache(const std::string &room_id, int64_t timestamp) {
-        load_qm_read_markers_from_cache(); // TODO: Remove this?
-        qm_read_markers_by_room_cache[room_id] = timestamp;
-
-        rapidjson::Document request_data(rapidjson::kObjectType);
-        for(const auto &[key, val] : qm_read_markers_by_room_cache) {
-            request_data.AddMember(rapidjson::Value(key.c_str(), request_data.GetAllocator()).Move(), val, request_data.GetAllocator());
-        }
-
-        rapidjson::StringBuffer buffer;
-        rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
-        request_data.Accept(writer);
-
-        file_overwrite_atomic(get_cache_dir().join("matrix").join("read_markers.json"), std::string(buffer.GetString(), buffer.GetSize()));
-    }
-
     PluginResult Matrix::join_room(const std::string &room_id_or_name) {
         assert(delegate);
         std::vector<CommandArg> additional_args = {