From f30b6fdf5c40a703b5a8d6fb2a21002377ec9067 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Tue, 6 Aug 2024 20:41:53 +0200 Subject: Reduce local-anime/local-manga disk access for speed up on mechanical harddrives, add local_anime.recursive option to make local-anime much faster when local-anime directory is a downloads directory --- src/Config.cpp | 1 + src/QuickMedia.cpp | 8 +++---- src/Storage.cpp | 32 +++++++++++---------------- src/plugins/LocalAnime.cpp | 55 ++++++++++++++++++++++++++++++---------------- src/plugins/LocalManga.cpp | 19 +++++++++++----- src/plugins/Matrix.cpp | 2 +- 6 files changed, 68 insertions(+), 49 deletions(-) (limited to 'src') diff --git a/src/Config.cpp b/src/Config.cpp index 5d16083..ecc1fce 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -275,6 +275,7 @@ namespace QuickMedia { get_json_value(local_anime_json, "sort_by_name", config->local_anime.sort_by_name); get_json_value(local_anime_json, "sort_episodes_by_name", config->local_anime.sort_episodes_by_name); get_json_value(local_anime_json, "auto_group_episodes", config->local_anime.auto_group_episodes); + get_json_value(local_anime_json, "recursive", config->local_anime.recursive); } const Json::Value &youtube_json = json_root["youtube"]; diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 332308b..d18ad4e 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -1790,7 +1790,7 @@ namespace QuickMedia { // TODO: Remove this once manga history file has been in use for a few months and is filled with history time_t now = time(NULL); - for_files_in_dir_sort_last_modified(content_storage_dir, [&](const Path &filepath, FileType) { + for_files_in_dir_sort_last_modified(content_storage_dir, [&](const Path &filepath, FileType, time_t last_modified_seconds) { // This can happen when QuickMedia crashes/is killed while writing to storage. // In that case, the storage wont be corrupt but there will be .tmp files. // TODO: Remove these .tmp files if they exist during startup @@ -1812,12 +1812,12 @@ namespace QuickMedia { if(!manga_name.isString()) return true; - time_t last_modified_time = 0; - file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_time); + if(last_modified_seconds == 0) + file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds); // TODO: Add thumbnail auto body_item = BodyItem::create(manga_name.asString()); - body_item->set_description("Last read " + seconds_to_relative_time_str(now - last_modified_time)); + body_item->set_description("Last read " + seconds_to_relative_time_str(now - last_modified_seconds)); body_item->set_description_color(get_theme().faded_text_color); auto thumbnail_it = manga_id_to_thumbnail_url_map.find(manga_id); diff --git a/src/Storage.cpp b/src/Storage.cpp index dc6b6a2..7bb3be6 100644 --- a/src/Storage.cpp +++ b/src/Storage.cpp @@ -249,7 +249,7 @@ namespace QuickMedia { for(auto &p : std::filesystem::directory_iterator(path.data)) { std::error_code ec; const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR; - if(!callback(p.path().string(), file_type)) + if(!callback(p.path().string(), file_type, 0)) break; } } catch(const std::filesystem::filesystem_error &err) { @@ -258,19 +258,13 @@ namespace QuickMedia { } } - static std::filesystem::file_time_type file_get_filetime_or(const std::filesystem::directory_entry &path, std::filesystem::file_time_type default_value) { - try { - return path.last_write_time(); - } catch(const std::filesystem::filesystem_error &err) { - return default_value; - } - } - void for_files_in_dir_sort_last_modified(const Path &path, FileIteratorCallback callback, FileSortDirection sort_dir) { - std::vector paths; + std::vector> paths_with_last_modified; try { for(auto &p : std::filesystem::directory_iterator(path.data)) { - paths.push_back(p); + time_t last_modified = 0; + file_get_last_modified_time_seconds(p.path().c_str(), &last_modified); + paths_with_last_modified.push_back(std::make_pair(p, last_modified)); } } catch(const std::filesystem::filesystem_error &err) { fprintf(stderr, "Failed to list files in directory %s, error: %s\n", path.data.c_str(), err.what()); @@ -278,19 +272,19 @@ namespace QuickMedia { } if(sort_dir == FileSortDirection::ASC) { - std::sort(paths.begin(), paths.end(), [](const std::filesystem::directory_entry &path1, std::filesystem::directory_entry &path2) { - return file_get_filetime_or(path1, std::filesystem::file_time_type::min()) > file_get_filetime_or(path2, std::filesystem::file_time_type::min()); + std::sort(paths_with_last_modified.begin(), paths_with_last_modified.end(), [](const auto &path1, const auto &path2) { + return path1.second > path2.second; }); } else { - std::sort(paths.begin(), paths.end(), [](const std::filesystem::directory_entry &path1, std::filesystem::directory_entry &path2) { - return file_get_filetime_or(path1, std::filesystem::file_time_type::min()) < file_get_filetime_or(path2, std::filesystem::file_time_type::min()); + std::sort(paths_with_last_modified.begin(), paths_with_last_modified.end(), [](const auto &path1, const auto &path2) { + return path1.second < path2.second; }); } - for(auto &p : paths) { + for(auto &p : paths_with_last_modified) { std::error_code ec; - const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR; - if(!callback(p.path().string(), file_type)) + const FileType file_type = p.first.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR; + if(!callback(p.first.path().string(), file_type, p.second)) break; } } @@ -319,7 +313,7 @@ namespace QuickMedia { for(auto &p : paths) { std::error_code ec; const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR; - if(!callback(p.path().string(), file_type)) + if(!callback(p.path().string(), file_type, 0)) break; } } diff --git a/src/plugins/LocalAnime.cpp b/src/plugins/LocalAnime.cpp index c906b9b..59e52ad 100644 --- a/src/plugins/LocalAnime.cpp +++ b/src/plugins/LocalAnime.cpp @@ -190,7 +190,7 @@ namespace QuickMedia { static std::vector get_episodes_in_directory(const Path &directory) { std::vector episodes; - for_files_in_dir_sort_name(directory, [&episodes](const Path &filepath, FileType file_type) -> bool { + for_files_in_dir_sort_name(directory, [&episodes](const Path &filepath, FileType file_type, time_t) -> bool { if(file_type != FileType::REGULAR || !is_video_ext(filepath.ext())) return true; @@ -206,21 +206,28 @@ namespace QuickMedia { static std::vector get_episodes_or_seasons_in_directory(const Path &directory) { std::vector anime_items; - auto callback = [&](const Path &filepath, FileType file_type) -> bool { - time_t modified_time_seconds; - if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &modified_time_seconds)) - return true; - + auto callback = [&](const Path &filepath, FileType file_type, time_t last_modified_seconds) -> bool { if(file_type == FileType::REGULAR) { - if(is_video_ext(filepath.ext())) - anime_items.push_back(LocalAnimeEpisode{ filepath, modified_time_seconds }); + if(is_video_ext(filepath.ext())) { + if(last_modified_seconds == 0) { + if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds)) + return true; + } + + anime_items.push_back(LocalAnimeEpisode{ filepath, last_modified_seconds }); + } return true; } + if(last_modified_seconds == 0) { + if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds)) + return true; + } + LocalAnimeSeason season; season.name = filepath.filename(); season.episodes = get_episodes_in_directory(filepath); - season.modified_time_seconds = modified_time_seconds; + season.modified_time_seconds = last_modified_seconds; if(season.episodes.empty()) return true; @@ -236,23 +243,33 @@ namespace QuickMedia { return anime_items; } - std::vector get_anime_in_directory(const Path &directory) { + std::vector get_anime_in_directory(const Path &directory, bool recursive) { std::vector anime_items; - auto callback = [&anime_items](const Path &filepath, FileType file_type) -> bool { - time_t modified_time_seconds; - if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &modified_time_seconds)) - return true; - + auto callback = [&anime_items, recursive](const Path &filepath, FileType file_type, time_t last_modified_seconds) -> bool { if(file_type == FileType::REGULAR) { - if(is_video_ext(filepath.ext())) - anime_items.push_back(LocalAnimeEpisode{ filepath, modified_time_seconds }); + if(is_video_ext(filepath.ext())) { + if(last_modified_seconds == 0) { + if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds)) + return true; + } + + anime_items.push_back(LocalAnimeEpisode{ filepath, last_modified_seconds }); + } return true; } + + if(!recursive) + return true; + if(last_modified_seconds == 0) { + if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds)) + return true; + } + LocalAnime anime; anime.name = filepath.filename(); anime.items = get_episodes_or_seasons_in_directory(filepath); - anime.modified_time_seconds = modified_time_seconds; + anime.modified_time_seconds = last_modified_seconds; if(anime.items.empty()) return true; @@ -352,7 +369,7 @@ namespace QuickMedia { PluginResult LocalAnimeSearchPage::lazy_fetch(BodyItems &result_items) { if(fetch_home_page) { fetch_home_page = false; - anime_items = get_anime_in_directory(get_config().local_anime.directory); + anime_items = get_anime_in_directory(get_config().local_anime.directory, get_config().local_anime.recursive); } std::unordered_map watch_progress = get_watch_progress_for_plugin("local-anime"); diff --git a/src/plugins/LocalManga.cpp b/src/plugins/LocalManga.cpp index fa0df27..21e8780 100644 --- a/src/plugins/LocalManga.cpp +++ b/src/plugins/LocalManga.cpp @@ -83,7 +83,7 @@ namespace QuickMedia { // Pages are sorted from 1.png to n.png static std::vector get_images_in_manga(const Path &directory) { std::vector page_list; - for_files_in_dir(directory, [&page_list](const Path &filepath, FileType file_type) -> bool { + for_files_in_dir(directory, [&page_list](const Path &filepath, FileType file_type, time_t) -> bool { if(file_type != FileType::REGULAR) return true; @@ -107,7 +107,7 @@ namespace QuickMedia { static std::vector get_chapters_in_manga(std::string manga_name, const Path &directory, bool only_include_latest, bool include_pages, bool only_get_coverpage = false) { std::vector chapter_list; - auto callback = [&chapter_list, &manga_name, only_include_latest, include_pages, only_get_coverpage](const Path &filepath, FileType file_type) -> bool { + auto callback = [&chapter_list, &manga_name, only_include_latest, include_pages, only_get_coverpage](const Path &filepath, FileType file_type, time_t last_modified_seconds) -> bool { if(file_type != FileType::DIRECTORY) return true; @@ -130,8 +130,11 @@ namespace QuickMedia { } } - if(!only_get_coverpage) - file_get_last_modified_time_seconds(filepath.data.c_str(), &local_manga_chapter.modified_time_seconds); + if(!only_get_coverpage) { + if(last_modified_seconds == 0) + file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds); + local_manga_chapter.modified_time_seconds = last_modified_seconds; + } chapter_list.push_back(std::move(local_manga_chapter)); return only_include_latest ? false : true; @@ -147,16 +150,20 @@ namespace QuickMedia { static std::vector get_manga_in_directory(const Path &directory, bool only_get_coverpage) { std::vector manga_list; - auto callback = [&manga_list, only_get_coverpage](const Path &filepath, FileType file_type) -> bool { + auto callback = [&manga_list, only_get_coverpage](const Path &filepath, FileType file_type, time_t last_modified_seconds) -> bool { if(file_type != FileType::DIRECTORY) return true; LocalManga local_manga; local_manga.name = filepath.filename(); local_manga.chapters = get_chapters_in_manga(local_manga.name, filepath, true, false, only_get_coverpage); - if(local_manga.chapters.empty() || !file_get_last_modified_time_seconds(filepath.data.c_str(), &local_manga.modified_time_seconds)) + if(local_manga.chapters.empty()) return true; + if(last_modified_seconds == 0) + file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds); + local_manga.modified_time_seconds = last_modified_seconds; + manga_list.push_back(std::move(local_manga)); return true; }; diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp index c4aa076..1b43b2d 100644 --- a/src/plugins/Matrix.cpp +++ b/src/plugins/Matrix.cpp @@ -5102,7 +5102,7 @@ namespace QuickMedia { //Path filter_cache_path = get_storage_dir().join("matrix").join("filter"); //remove(filter_cache_path.data.c_str()); - for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath, FileType) { + for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath, FileType, time_t) { remove(filepath.data.c_str()); return true; }); -- cgit v1.2.3