From 3dd837611eef939a1fd54621c523cff4ab48e136 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Wed, 4 Nov 2020 21:45:53 +0100 Subject: Matrix: fetch additional messages in the background for all rooms, filter room list when updated after sync --- src/plugins/Matrix.cpp | 115 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 94 insertions(+), 21 deletions(-) (limited to 'src/plugins') diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp index 9cfc1bb..32700a7 100644 --- a/src/plugins/Matrix.cpp +++ b/src/plugins/Matrix.cpp @@ -21,7 +21,8 @@ static const char* SERVICE_NAME = "matrix"; static const char* OTHERS_ROOM_TAG = "tld.name.others"; // Filter without account data -static const char* FILTER = "{\"presence\":{\"limit\":0,\"types\":[\"\"]},\"account_data\":{\"limit\":0,\"types\":[\"\"]},\"room\":{\"state\":{\"not_types\":[\"m.room.related_groups\",\"m.room.power_levels\",\"m.room.join_rules\",\"m.room.history_visibility\"],\"lazy_load_members\":true},\"timeline\":{\"limit\":1,\"lazy_load_members\":true},\"ephemeral\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true},\"account_data\":{\"limit\":1,\"types\":[\"m.fully_read\",\"m.tag\"],\"lazy_load_members\":true}}}"; +static const char* INITIAL_FILTER = "{\"presence\":{\"limit\":0,\"types\":[\"\"]},\"account_data\":{\"limit\":0,\"types\":[\"\"]},\"room\":{\"state\":{\"not_types\":[\"m.room.related_groups\",\"m.room.power_levels\",\"m.room.join_rules\",\"m.room.history_visibility\"],\"lazy_load_members\":true},\"timeline\":{\"limit\":1,\"lazy_load_members\":true},\"ephemeral\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true},\"account_data\":{\"limit\":1,\"types\":[\"m.fully_read\",\"m.tag\"],\"lazy_load_members\":true}}}"; +static const char* CONTINUE_FILTER = "{\"presence\":{\"limit\":0,\"types\":[\"\"]},\"account_data\":{\"limit\":0,\"types\":[\"\"]},\"room\":{\"state\":{\"not_types\":[\"m.room.related_groups\",\"m.room.power_levels\",\"m.room.join_rules\",\"m.room.history_visibility\"],\"lazy_load_members\":true},\"timeline\":{\"limit\":20,\"lazy_load_members\":true},\"ephemeral\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true},\"account_data\":{\"limit\":0,\"types\":[\"\"],\"lazy_load_members\":true}}}"; static rapidjson::Value nullValue(rapidjson::kNullType); static const rapidjson::Value& GetMember(const rapidjson::Value &obj, const char *key) { @@ -217,7 +218,7 @@ namespace QuickMedia { void RoomData::clear_data() { std::lock_guard lock(room_mutex); - fetched_messages_by_event_id.clear(); + //fetched_messages_by_event_id.clear(); user_info_by_user_id.clear(); messages.clear(); messages_read_index = 0; @@ -469,6 +470,9 @@ namespace QuickMedia { body->clear_items(); } + if(!pending_remove_body_items.empty() || !room_body_items.empty()) + filter_on_update = true; + for(const std::string &room_id : pending_remove_body_items) { remove_body_item_by_url(body->items, room_id); // TODO: There can be a race condition where current_chat_page is set after entering a room and then we will enter a room we left @@ -488,6 +492,11 @@ namespace QuickMedia { sort_room_body_items(body->items); } matrix_delegate->update(MatrixPageType::ROOM_LIST); + if(filter_on_update) { + filter_on_update = false; + if(search_bar) + body->filter_search_fuzzy(search_bar->get_text()); + } } void MatrixRoomsPage::add_body_item(std::shared_ptr body_item) { @@ -578,8 +587,10 @@ namespace QuickMedia { if(tag_body_it->second.room_body_items.empty()) { auto room_body_item_it = std::find(body->items.begin(), body->items.end(), tag_body_it->second.tag_item); - if(room_body_item_it != body->items.end()) + if(room_body_item_it != body->items.end()) { body->items.erase(room_body_item_it); + filter_on_update = true; + } tag_body_items_by_name.erase(tag_body_it); } } @@ -598,6 +609,7 @@ namespace QuickMedia { body->items.push_back(tag_body_item); tag_data = &tag_body_items_by_name[it.first]; tag_data->tag_item = tag_body_item; + filter_on_update = true; } } else { tag_data = &tag_body_it->second; @@ -610,6 +622,11 @@ namespace QuickMedia { add_room_body_items_by_tags.clear(); } matrix_delegate->update(MatrixPageType::ROOM_LIST); + if(filter_on_update) { + filter_on_update = false; + if(search_bar) + body->filter_search_fuzzy(search_bar->get_text()); + } } void MatrixRoomTagsPage::add_room_body_item_to_tag(std::shared_ptr body_item, const std::string &tag) { @@ -662,7 +679,7 @@ namespace QuickMedia { current_rooms_page->sort_rooms(); } - MatrixInvitesPage::MatrixInvitesPage(Program *program, Matrix *matrix, Body *body) : Page(program), matrix(matrix), body(body) { + MatrixInvitesPage::MatrixInvitesPage(Program *program, Matrix *matrix, Body *body, SearchBar *search_bar) : Page(program), matrix(matrix), body(body), search_bar(search_bar) { } @@ -706,6 +723,9 @@ namespace QuickMedia { body->clear_items(); } + if(!pending_remove_body_items.empty() || !body_items.empty()) + filter_on_update = true; + for(const std::string &room_id : pending_remove_body_items) { remove_body_item_by_url(body->items, room_id); } @@ -718,6 +738,12 @@ namespace QuickMedia { prev_invite_count = body->items.size(); title = "Invites (" + std::to_string(body->items.size()) + ")"; } + + if(filter_on_update) { + filter_on_update = false; + if(search_bar) + body->filter_search_fuzzy(search_bar->get_text()); + } } void MatrixInvitesPage::add_body_item(std::shared_ptr body_item) { @@ -847,7 +873,7 @@ namespace QuickMedia { rapidjson::ParseResult parse_result = doc.ParseStream(is); if(parse_result.IsError()) break; - if(parse_sync_response(doc) != PluginResult::OK) + if(parse_sync_response(doc, false) != PluginResult::OK) fprintf(stderr, "Failed to parse cached sync response\n"); } fclose(sync_cache_file); @@ -857,6 +883,7 @@ namespace QuickMedia { // Filter with account data // {"presence":{"limit":0,"types":[""]},"account_data":{"not_types":["im.vector.setting.breadcrumbs","m.push_rules","im.vector.setting.allowed_widgets","io.element.recent_emoji"]},"room":{"state":{"limit":1,"not_types":["m.room.related_groups","m.room.power_levels","m.room.join_rules","m.room.history_visibility"],"lazy_load_members":true},"timeline":{"limit":3,"lazy_load_members":true},"ephemeral":{"limit":0,"types":[""],"lazy_load_members":true},"account_data":{"limit":1,"types":["m.fully_read"],"lazy_load_members":true}}} +#if 0 bool filter_cached = false; Path filter_path = get_storage_dir().join("matrix").join("filter"); if(get_file_type(filter_path) == FileType::REGULAR) @@ -869,13 +896,39 @@ namespace QuickMedia { filter = get_filter_cached(); else filter = FILTER; - std::string filter_encoded = url_param_encode(filter); +#endif + std::string filter_encoded = url_param_encode(INITIAL_FILTER); std::vector additional_args = { { "-H", "Authorization: Bearer " + access_token }, { "-m", "35" } }; + sync_additional_messages_thread = std::thread([this]() { + std::vector additional_args = { + { "-H", "Authorization: Bearer " + access_token }, + { "-m", "35" } + }; + + char url[1024]; + std::string filter_encoded = url_param_encode(CONTINUE_FILTER); + snprintf(url, sizeof(url), "%s/_matrix/client/r0/sync?filter=%s&timeout=0", homeserver.c_str(), filter_encoded.c_str()); + + rapidjson::Document json_root; + std::string err_msg; + DownloadResult download_result = download_json(json_root, url, additional_args, true, &err_msg); + if(download_result != DownloadResult::OK) { + fprintf(stderr, "/sync for additional messages failed\n"); + return; + } + + // TODO: Test? + //if(next_batch.empty()) + // clear_sync_cache_for_new_sync(); + + parse_sync_response(json_root, true); + }); + const rapidjson::Value *next_batch_json; PluginResult result; bool initial_sync = true; @@ -916,7 +969,7 @@ namespace QuickMedia { if(next_batch.empty()) clear_sync_cache_for_new_sync(); - result = parse_sync_response(json_root); + result = parse_sync_response(json_root, false); if(result != PluginResult::OK) { fprintf(stderr, "Failed to parse sync response\n"); goto sync_end; @@ -952,12 +1005,16 @@ namespace QuickMedia { const rapidjson::Value ¬ification_json = GetMember(json_root, "notifications"); parse_notifications(notification_json); }); + + filter_encoded = url_param_encode(CONTINUE_FILTER); } +#if 0 if(!filter_cached) { filter_cached = true; filter_encoded = url_param_encode(get_filter_cached()); } +#endif sync_end: if(sync_running) @@ -987,6 +1044,11 @@ namespace QuickMedia { program_kill_in_thread(sync_thread.get_id()); sync_thread.join(); } + + if(sync_additional_messages_thread.joinable()) { + program_kill_in_thread(sync_additional_messages_thread.get_id()); + sync_additional_messages_thread.join(); + } if(notification_thread.joinable()) { program_kill_in_thread(notification_thread.get_id()); @@ -1064,7 +1126,7 @@ namespace QuickMedia { return PluginResult::OK; } - PluginResult Matrix::parse_sync_response(const rapidjson::Document &root) { + PluginResult Matrix::parse_sync_response(const rapidjson::Document &root, bool is_additional_messages_sync) { if(!root.IsObject()) return PluginResult::ERR; @@ -1074,7 +1136,7 @@ namespace QuickMedia { // TODO: Include "Direct messages" as a tag using |dm_rooms| above const rapidjson::Value &rooms_json = GetMember(root, "rooms"); - parse_sync_room_data(rooms_json); + parse_sync_room_data(rooms_json, is_additional_messages_sync); return PluginResult::OK; } @@ -1173,7 +1235,7 @@ namespace QuickMedia { return PluginResult::OK; } - PluginResult Matrix::parse_sync_room_data(const rapidjson::Value &rooms_json) { + PluginResult Matrix::parse_sync_room_data(const rapidjson::Value &rooms_json, bool is_additional_messages_sync) { if(!rooms_json.IsObject()) return PluginResult::OK; @@ -1204,14 +1266,15 @@ namespace QuickMedia { const rapidjson::Value &events_json = GetMember(state_json, "events"); events_add_user_info(events_json, room); events_set_room_name(events_json, room); - events_add_pinned_events(events_json, room); + if(!is_additional_messages_sync) + events_add_pinned_events(events_json, room); } const rapidjson::Value &account_data_json = GetMember(it.value, "account_data"); const rapidjson::Value &timeline_json = GetMember(it.value, "timeline"); if(timeline_json.IsObject()) { - if(!room->has_prev_batch()) { + if(is_additional_messages_sync && !room->has_prev_batch()) { // This may be non-existent if this is the first event in the room const rapidjson::Value &prev_batch_json = GetMember(timeline_json, "prev_batch"); if(prev_batch_json.IsString()) @@ -1243,7 +1306,8 @@ namespace QuickMedia { delegate->join_room(room); events_add_messages(events_json, room, MessageDirection::AFTER, has_unread_notifications); - events_add_pinned_events(events_json, room); + if(!is_additional_messages_sync) + events_add_pinned_events(events_json, room); } else { if(account_data_json.IsObject()) { const rapidjson::Value &events_json = GetMember(account_data_json, "events"); @@ -1277,11 +1341,13 @@ namespace QuickMedia { } } - const rapidjson::Value &leave_json = GetMember(rooms_json, "leave"); - remove_rooms(leave_json); + if(!is_additional_messages_sync) { + const rapidjson::Value &leave_json = GetMember(rooms_json, "leave"); + remove_rooms(leave_json); - const rapidjson::Value &invite_json = GetMember(rooms_json, "invite"); - add_invites(invite_json); + const rapidjson::Value &invite_json = GetMember(rooms_json, "invite"); + add_invites(invite_json); + } return PluginResult::OK; } @@ -1881,6 +1947,7 @@ namespace QuickMedia { bool has_room_name = room_data->has_name(); bool has_room_avatar_url = room_data->has_avatar_url(); + // TODO: Fix. This can be incorrect because this method can be called before we have added all users to the room std::vector> users_excluding_me; if(!has_room_name || !has_room_avatar_url) users_excluding_me = room_data->get_users_excluding_me(user_id); // TODO: What about thread safety with user_id? its reset in /logout @@ -2083,8 +2150,8 @@ namespace QuickMedia { invite.new_invite = !next_batch.empty(); std::string room_id_str(room_id.GetString(), room_id.GetStringLength()); - delegate->add_invite(room_id_str, invite); - set_invite(room_id_str, std::move(invite)); + if(set_invite(room_id_str, invite)) + delegate->add_invite(room_id_str, std::move(invite)); break; } } @@ -3149,9 +3216,10 @@ namespace QuickMedia { room_data_by_id.erase(room_it); } - void Matrix::set_invite(const std::string &room_id, Invite invite) { + bool Matrix::set_invite(const std::string &room_id, Invite invite) { std::lock_guard lock(invite_mutex); - invites[room_id] = std::move(invite); + auto res = invites.insert(std::make_pair(room_id, std::move(invite))); + return res.second; } bool Matrix::remove_invite(const std::string &room_id) { @@ -3219,6 +3287,7 @@ namespace QuickMedia { // TODO: GET the filter to check if its valid? std::string Matrix::get_filter_cached() { + #if 0 if(filter_cached) return filter_cached.value(); @@ -3255,6 +3324,10 @@ namespace QuickMedia { file_overwrite_atomic(filter_path, filter_cached.value()); } return filter_cached.value(); + #else + assert(false); + return INITIAL_FILTER; + #endif } DownloadResult Matrix::download_json(rapidjson::Document &result, const std::string &url, std::vector additional_args, bool use_browser_useragent, std::string *err_msg) const { -- cgit v1.2.3