Reduce local-anime/local-manga disk access for speed up on mechanical harddrives, add local_anime.recursive option to make local-anime much faster when local-anime directory is a downloads directory

6 files changed, 68 insertions, 49 deletions

diff --git a/src/Config.cpp b/src/Config.cpp
index 5d16083..ecc1fce 100644
--- a/src/Config.cpp
+++ b/src/Config.cpp
@@ -275,6 +275,7 @@ namespace QuickMedia {
             get_json_value(local_anime_json, "sort_by_name", config->local_anime.sort_by_name);
             get_json_value(local_anime_json, "sort_episodes_by_name", config->local_anime.sort_episodes_by_name);
             get_json_value(local_anime_json, "auto_group_episodes", config->local_anime.auto_group_episodes);
+            get_json_value(local_anime_json, "recursive", config->local_anime.recursive);
         }
 
         const Json::Value &youtube_json = json_root["youtube"];
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 332308b..d18ad4e 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -1790,7 +1790,7 @@ namespace QuickMedia {
 
         // TODO: Remove this once manga history file has been in use for a few months and is filled with history
         time_t now = time(NULL);
-        for_files_in_dir_sort_last_modified(content_storage_dir, [&](const Path &filepath, FileType) {
+        for_files_in_dir_sort_last_modified(content_storage_dir, [&](const Path &filepath, FileType, time_t last_modified_seconds) {
             // This can happen when QuickMedia crashes/is killed while writing to storage.
             // In that case, the storage wont be corrupt but there will be .tmp files.
             // TODO: Remove these .tmp files if they exist during startup
@@ -1812,12 +1812,12 @@ namespace QuickMedia {
             if(!manga_name.isString())
                 return true;
 
-            time_t last_modified_time = 0;
-            file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_time);
+            if(last_modified_seconds == 0)
+                file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds);
 
             // TODO: Add thumbnail
             auto body_item = BodyItem::create(manga_name.asString());
-            body_item->set_description("Last read " + seconds_to_relative_time_str(now - last_modified_time));
+            body_item->set_description("Last read " + seconds_to_relative_time_str(now - last_modified_seconds));
             body_item->set_description_color(get_theme().faded_text_color);
 
             auto thumbnail_it = manga_id_to_thumbnail_url_map.find(manga_id);
diff --git a/src/Storage.cpp b/src/Storage.cpp
index dc6b6a2..7bb3be6 100644
--- a/src/Storage.cpp
+++ b/src/Storage.cpp
@@ -249,7 +249,7 @@ namespace QuickMedia {
             for(auto &p : std::filesystem::directory_iterator(path.data)) {
                 std::error_code ec;
                 const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR;
-                if(!callback(p.path().string(), file_type))
+                if(!callback(p.path().string(), file_type, 0))
                     break;
             }
         } catch(const std::filesystem::filesystem_error &err) {
@@ -258,19 +258,13 @@ namespace QuickMedia {
         }
     }
 
-    static std::filesystem::file_time_type file_get_filetime_or(const std::filesystem::directory_entry &path, std::filesystem::file_time_type default_value) {
-        try {
-            return path.last_write_time();
-        } catch(const std::filesystem::filesystem_error &err) {
-            return default_value;
-        }
-    }
-
     void for_files_in_dir_sort_last_modified(const Path &path, FileIteratorCallback callback, FileSortDirection sort_dir) {
-        std::vector<std::filesystem::directory_entry> paths;
+        std::vector<std::pair<std::filesystem::directory_entry, time_t>> paths_with_last_modified;
         try {
             for(auto &p : std::filesystem::directory_iterator(path.data)) {
-                paths.push_back(p);
+                time_t last_modified = 0;
+                file_get_last_modified_time_seconds(p.path().c_str(), &last_modified);
+                paths_with_last_modified.push_back(std::make_pair(p, last_modified));
             }
         } catch(const std::filesystem::filesystem_error &err) {
             fprintf(stderr, "Failed to list files in directory %s, error: %s\n", path.data.c_str(), err.what());
@@ -278,19 +272,19 @@ namespace QuickMedia {
         }
 
         if(sort_dir == FileSortDirection::ASC) {
-            std::sort(paths.begin(), paths.end(), [](const std::filesystem::directory_entry &path1, std::filesystem::directory_entry &path2) {
-                return file_get_filetime_or(path1, std::filesystem::file_time_type::min()) > file_get_filetime_or(path2, std::filesystem::file_time_type::min());
+            std::sort(paths_with_last_modified.begin(), paths_with_last_modified.end(), [](const auto &path1, const auto &path2) {
+                return path1.second > path2.second;
             });
         } else {
-            std::sort(paths.begin(), paths.end(), [](const std::filesystem::directory_entry &path1, std::filesystem::directory_entry &path2) {
-                return file_get_filetime_or(path1, std::filesystem::file_time_type::min()) < file_get_filetime_or(path2, std::filesystem::file_time_type::min());
+            std::sort(paths_with_last_modified.begin(), paths_with_last_modified.end(), [](const auto &path1, const auto &path2) {
+                return path1.second < path2.second;
             });
         }
 
-        for(auto &p : paths) {
+        for(auto &p : paths_with_last_modified) {
             std::error_code ec;
-            const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR;
-            if(!callback(p.path().string(), file_type))
+            const FileType file_type = p.first.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR;
+            if(!callback(p.first.path().string(), file_type, p.second))
                 break;
         }
     }
@@ -319,7 +313,7 @@ namespace QuickMedia {
         for(auto &p : paths) {
             std::error_code ec;
             const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR;
-            if(!callback(p.path().string(), file_type))
+            if(!callback(p.path().string(), file_type, 0))
                 break;
         }
     }
diff --git a/src/plugins/LocalAnime.cpp b/src/plugins/LocalAnime.cpp
index c906b9b..59e52ad 100644
--- a/src/plugins/LocalAnime.cpp
+++ b/src/plugins/LocalAnime.cpp
@@ -190,7 +190,7 @@ namespace QuickMedia {
 
     static std::vector<LocalAnimeItem> get_episodes_in_directory(const Path &directory) {
         std::vector<LocalAnimeItem> episodes;
-        for_files_in_dir_sort_name(directory, [&episodes](const Path &filepath, FileType file_type) -> bool {
+        for_files_in_dir_sort_name(directory, [&episodes](const Path &filepath, FileType file_type, time_t) -> bool {
             if(file_type != FileType::REGULAR || !is_video_ext(filepath.ext()))
                 return true;
 
@@ -206,21 +206,28 @@ namespace QuickMedia {
 
     static std::vector<LocalAnimeItem> get_episodes_or_seasons_in_directory(const Path &directory) {
         std::vector<LocalAnimeItem> anime_items;
-        auto callback = [&](const Path &filepath, FileType file_type) -> bool {
-            time_t modified_time_seconds;
-            if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &modified_time_seconds))
-                return true;
-
+        auto callback = [&](const Path &filepath, FileType file_type, time_t last_modified_seconds) -> bool {
             if(file_type == FileType::REGULAR) {
-                if(is_video_ext(filepath.ext()))
-                    anime_items.push_back(LocalAnimeEpisode{ filepath, modified_time_seconds });
+                if(is_video_ext(filepath.ext())) {
+                    if(last_modified_seconds == 0) {
+                        if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds))
+                            return true;
+                    }
+
+                    anime_items.push_back(LocalAnimeEpisode{ filepath, last_modified_seconds });
+                }
                 return true;
             }
 
+            if(last_modified_seconds == 0) {
+                if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds))
+                    return true;
+            }
+
             LocalAnimeSeason season;
             season.name = filepath.filename();
             season.episodes = get_episodes_in_directory(filepath);
-            season.modified_time_seconds = modified_time_seconds;
+            season.modified_time_seconds = last_modified_seconds;
             if(season.episodes.empty())
                 return true;
 
@@ -236,23 +243,33 @@ namespace QuickMedia {
         return anime_items;
     }
 
-    std::vector<LocalAnimeItem> get_anime_in_directory(const Path &directory) {
+    std::vector<LocalAnimeItem> get_anime_in_directory(const Path &directory, bool recursive) {
         std::vector<LocalAnimeItem> anime_items;
-        auto callback = [&anime_items](const Path &filepath, FileType file_type) -> bool {
-            time_t modified_time_seconds;
-            if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &modified_time_seconds))
-                return true;
-
+        auto callback = [&anime_items, recursive](const Path &filepath, FileType file_type, time_t last_modified_seconds) -> bool {
             if(file_type == FileType::REGULAR) {
-                if(is_video_ext(filepath.ext()))
-                    anime_items.push_back(LocalAnimeEpisode{ filepath, modified_time_seconds });
+                if(is_video_ext(filepath.ext())) {
+                    if(last_modified_seconds == 0) {
+                        if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds))
+                            return true;
+                    }
+
+                    anime_items.push_back(LocalAnimeEpisode{ filepath, last_modified_seconds });
+                }
                 return true;
             }
+
+            if(!recursive)
+                return true;
             
+            if(last_modified_seconds == 0) {
+                if(!file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds))
+                    return true;
+            }
+
             LocalAnime anime;
             anime.name = filepath.filename();
             anime.items = get_episodes_or_seasons_in_directory(filepath);
-            anime.modified_time_seconds = modified_time_seconds;
+            anime.modified_time_seconds = last_modified_seconds;
             if(anime.items.empty())
                 return true;
 
@@ -352,7 +369,7 @@ namespace QuickMedia {
     PluginResult LocalAnimeSearchPage::lazy_fetch(BodyItems &result_items) {
         if(fetch_home_page) {
             fetch_home_page = false;
-            anime_items = get_anime_in_directory(get_config().local_anime.directory);
+            anime_items = get_anime_in_directory(get_config().local_anime.directory, get_config().local_anime.recursive);
         }
         
         std::unordered_map<std::string, WatchProgress> watch_progress = get_watch_progress_for_plugin("local-anime");
diff --git a/src/plugins/LocalManga.cpp b/src/plugins/LocalManga.cpp
index fa0df27..21e8780 100644
--- a/src/plugins/LocalManga.cpp
+++ b/src/plugins/LocalManga.cpp
@@ -83,7 +83,7 @@ namespace QuickMedia {
     // Pages are sorted from 1.png to n.png
     static std::vector<LocalMangaPage> get_images_in_manga(const Path &directory) {
         std::vector<LocalMangaPage> page_list;
-        for_files_in_dir(directory, [&page_list](const Path &filepath, FileType file_type) -> bool {
+        for_files_in_dir(directory, [&page_list](const Path &filepath, FileType file_type, time_t) -> bool {
             if(file_type != FileType::REGULAR)
                 return true;
 
@@ -107,7 +107,7 @@ namespace QuickMedia {
 
     static std::vector<LocalMangaChapter> get_chapters_in_manga(std::string manga_name, const Path &directory, bool only_include_latest, bool include_pages, bool only_get_coverpage = false) {
         std::vector<LocalMangaChapter> chapter_list;
-        auto callback = [&chapter_list, &manga_name, only_include_latest, include_pages, only_get_coverpage](const Path &filepath, FileType file_type) -> bool {
+        auto callback = [&chapter_list, &manga_name, only_include_latest, include_pages, only_get_coverpage](const Path &filepath, FileType file_type, time_t last_modified_seconds) -> bool {
             if(file_type != FileType::DIRECTORY)
                 return true;
 
@@ -130,8 +130,11 @@ namespace QuickMedia {
                 }
             }
 
-            if(!only_get_coverpage)
-                file_get_last_modified_time_seconds(filepath.data.c_str(), &local_manga_chapter.modified_time_seconds);
+            if(!only_get_coverpage) {
+                if(last_modified_seconds == 0)
+                    file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds);
+                local_manga_chapter.modified_time_seconds = last_modified_seconds;
+            }
 
             chapter_list.push_back(std::move(local_manga_chapter));
             return only_include_latest ? false : true;
@@ -147,16 +150,20 @@ namespace QuickMedia {
 
     static std::vector<LocalManga> get_manga_in_directory(const Path &directory, bool only_get_coverpage) {
         std::vector<LocalManga> manga_list;
-        auto callback = [&manga_list, only_get_coverpage](const Path &filepath, FileType file_type) -> bool {
+        auto callback = [&manga_list, only_get_coverpage](const Path &filepath, FileType file_type, time_t last_modified_seconds) -> bool {
             if(file_type != FileType::DIRECTORY)
                 return true;
 
             LocalManga local_manga;
             local_manga.name = filepath.filename();
             local_manga.chapters = get_chapters_in_manga(local_manga.name, filepath, true, false, only_get_coverpage);
-            if(local_manga.chapters.empty() || !file_get_last_modified_time_seconds(filepath.data.c_str(), &local_manga.modified_time_seconds))
+            if(local_manga.chapters.empty())
                 return true;
 
+            if(last_modified_seconds == 0)
+                file_get_last_modified_time_seconds(filepath.data.c_str(), &last_modified_seconds);
+            local_manga.modified_time_seconds = last_modified_seconds;
+
             manga_list.push_back(std::move(local_manga));
             return true;
         };
diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp
index c4aa076..1b43b2d 100644
--- a/src/plugins/Matrix.cpp
+++ b/src/plugins/Matrix.cpp
@@ -5102,7 +5102,7 @@ namespace QuickMedia {
 
         //Path filter_cache_path = get_storage_dir().join("matrix").join("filter");
         //remove(filter_cache_path.data.c_str());
-        for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath, FileType) {
+        for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath, FileType, time_t) {
             remove(filepath.data.c_str());
             return true;
         });