Local-manga: improve loading of page when using slow medium

Especially when using NFS.
Only get the latest chapter when needed and cache link to the cover
page.

7 files changed, 126 insertions, 31 deletions

diff --git a/TODO b/TODO
index 9599d31..3fa13f9 100644
--- a/TODO
+++ b/TODO
@@ -216,4 +216,5 @@ Allow changing manga sorting in gui (especially for local manga).
 Allow using ~ and $HOME in config file.
 Save manga "I" mode to file and load it on startup.
 Show in bookmarks and history page if a manga has been read (local-manga).
-Add "finished reading" to online manga as well, for the manga sites that publish latest chapter in the search page.
\ No newline at end of file
+Add "finished reading" to online manga as well, for the manga sites that publish latest chapter in the search page.
+Async load visible body item content. This is needed for local-manga if the manga is stored on NFS where recursively reading all manga directories is slow. We only want to read recursively for the manga that is visible on the screen.
\ No newline at end of file
diff --git a/include/Storage.hpp b/include/Storage.hpp
index 1e38906..c187261 100644
--- a/include/Storage.hpp
+++ b/include/Storage.hpp
@@ -9,9 +9,6 @@ namespace Json {
 }
 
 namespace QuickMedia {
-    // Return false to stop the iterator
-    using FileIteratorCallback = std::function<bool(const Path &filepath)>;
-
     enum class FileType {
         FILE_NOT_FOUND,
         REGULAR,
@@ -23,6 +20,9 @@ namespace QuickMedia {
         DESC
     };
 
+    // Return false to stop the iterator
+    using FileIteratorCallback = std::function<bool(const Path &filepath, FileType file_type)>;
+
     Path get_home_dir();
     Path get_storage_dir();
     Path get_cache_dir();
diff --git a/plugins/LocalManga.hpp b/plugins/LocalManga.hpp
index e85e08b..0ab3f62 100644
--- a/plugins/LocalManga.hpp
+++ b/plugins/LocalManga.hpp
@@ -12,7 +12,6 @@ namespace QuickMedia {
     struct LocalMangaChapter {
         std::string name;
         std::vector<LocalMangaPage> pages;
-        time_t modified_time_seconds;
     };
 
     struct LocalManga {
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 8174680..edb1db1 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -51,6 +51,7 @@
 #include <unistd.h>
 #include <libgen.h>
 #include <limits.h>
+#include <sys/stat.h>
 
 #include <mglpp/graphics/Rectangle.hpp>
 #include <mglpp/graphics/Sprite.hpp>
@@ -957,7 +958,7 @@ namespace QuickMedia {
         show_notification("QuickMedia", "Upgrading mangadex ids", Urgency::LOW);
 
         std::vector<int> legacy_manga_ids;
-        for_files_in_dir_sort_last_modified(content_storage_dir, [&legacy_manga_ids](const Path &filepath) {
+        for_files_in_dir_sort_last_modified(content_storage_dir, [&legacy_manga_ids](const Path &filepath, FileType) {
             if(strcmp(filepath.ext(), ".tmp") == 0)
                 return true;
 
@@ -1460,7 +1461,7 @@ namespace QuickMedia {
 
         // TODO: Remove this once manga history file has been in use for a few months and is filled with history
         time_t now = time(NULL);
-        for_files_in_dir_sort_last_modified(content_storage_dir, [&history_items, plugin_name, &manga_id_to_thumbnail_url_map, now, local_thumbnail](const Path &filepath) {
+        for_files_in_dir_sort_last_modified(content_storage_dir, [&](const Path &filepath, FileType) {
             // This can happen when QuickMedia crashes/is killed while writing to storage.
             // In that case, the storage wont be corrupt but there will be .tmp files.
             // TODO: Remove these .tmp files if they exist during startup
@@ -3387,6 +3388,12 @@ namespace QuickMedia {
         return PageType::EXIT;
     }
 
+    // TODO: Do the same for thumbnails?
+    static bool is_symlink_valid(const char *filepath) {
+        struct stat buf;
+        return lstat(filepath, &buf) != -1;
+    }
+
     // TODO: Optimize this somehow. One image alone uses more than 20mb ram! Total ram usage for viewing one image
     // becomes 40mb (private memory, almost 100mb in total!) Unacceptable!
     Program::LoadImageResult Program::load_image_by_index(int image_index, mgl::Texture &image_texture, std::string &error_message) {
@@ -3401,7 +3408,7 @@ namespace QuickMedia {
                 upscaled_ok = false;
         }
 
-        if(get_file_type(image_path) == FileType::REGULAR && upscaled_ok) {
+        if(get_file_type(image_path) == FileType::REGULAR && is_symlink_valid(image_path.data.c_str()) && upscaled_ok) {
             if(image_texture.load_from_file(image_path.data.c_str())) {
                 return LoadImageResult::OK;
             } else {
@@ -3463,7 +3470,7 @@ namespace QuickMedia {
                         upscaled_ok = false;
                 }
 
-                if(get_file_type(image_filepath) != FileType::FILE_NOT_FOUND && upscaled_ok)
+                if(get_file_type(image_filepath) != FileType::FILE_NOT_FOUND && is_symlink_valid(image_filepath.data.c_str()) && upscaled_ok)
                     return true;
 
                 std::vector<CommandArg> extra_args;
diff --git a/src/Storage.cpp b/src/Storage.cpp
index f4732b8..effa70b 100644
--- a/src/Storage.cpp
+++ b/src/Storage.cpp
@@ -246,7 +246,9 @@ namespace QuickMedia {
     void for_files_in_dir(const Path &path, FileIteratorCallback callback) {
         try {
             for(auto &p : std::filesystem::directory_iterator(path.data)) {
-                if(!callback(p.path().string()))
+                std::error_code ec;
+                const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR;
+                if(!callback(p.path().string(), file_type))
                     break;
             }
         } catch(const std::filesystem::filesystem_error &err) {
@@ -285,7 +287,9 @@ namespace QuickMedia {
         }
 
         for(auto &p : paths) {
-            if(!callback(p.path().string()))
+            std::error_code ec;
+            const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR;
+            if(!callback(p.path().string(), file_type))
                 break;
         }
     }
@@ -312,7 +316,9 @@ namespace QuickMedia {
         }
 
         for(auto &p : paths) {
-            if(!callback(p.path().string()))
+            std::error_code ec;
+            const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR;
+            if(!callback(p.path().string(), file_type))
                 break;
         }
     }
diff --git a/src/plugins/LocalManga.cpp b/src/plugins/LocalManga.cpp
index 8344527..dc98639 100644
--- a/src/plugins/LocalManga.cpp
+++ b/src/plugins/LocalManga.cpp
@@ -7,15 +7,75 @@
 #include "../../external/cppcodec/base64_url.hpp"
 #include "../../include/QuickMedia.hpp"
 #include <json/value.h>
+#include <dirent.h>
 
 namespace QuickMedia {
+    // This is needed because the manga may be stored on NFS.
+    // TODO: Remove once body items can async load when visible on screen
+    class CoverPageLinkCache {
+    public:
+        static CoverPageLinkCache& get_instance() {
+            static CoverPageLinkCache instance;
+            Path dir = get_cache_dir().join("thumbnail-link");
+            if(create_directory_recursive(dir) != 0) {
+                show_notification("QuickMedia", "Failed to create directory: " + dir.data, Urgency::CRITICAL);
+                abort();
+            }
+
+            instance.cache_filepath = dir.join("local-manga");
+            std::string file_content;
+            if(get_file_type(instance.cache_filepath) == FileType::REGULAR && file_get_content(instance.cache_filepath, file_content) != 0) {
+                show_notification("QuickMedia", "Failed to load local manga thumbnail link cache", Urgency::CRITICAL);
+                abort();
+            }
+
+            std::unordered_map<std::string, std::string> manga_map;
+            string_split(file_content, '\n', [&manga_map](const char *str_part, size_t size) {
+                const void *space_p = memchr(str_part, ' ', size);
+                if(!space_p)
+                    return true;
+
+                std::string manga_name_base64(str_part, (const char*)space_p - str_part);
+                std::string cover_filepath((const char*)space_p + 1, str_part + size - ((const char*)space_p + 1));
+                manga_map[std::move(manga_name_base64)] = std::move(cover_filepath);
+                return true;
+            });
+
+            instance.manga_name_base64_to_cover_filepath_map = std::move(manga_map);
+            return instance;
+        }
+
+        std::string get_cached_cover_page_link_for_manga(const std::string &manga_name) {
+            std::string manga_name_base64_url = cppcodec::base64_url::encode<std::string>(manga_name);
+            auto it = manga_name_base64_to_cover_filepath_map.find(manga_name_base64_url);
+            if(it == manga_name_base64_to_cover_filepath_map.end())
+                return "";
+            return it->second;
+        }
+
+        void add_manga_to_thumbnail_link_cache(const std::string &manga_name, const std::string &cover_filepath) {
+            std::string manga_name_base64_url = cppcodec::base64_url::encode<std::string>(manga_name);
+            manga_name_base64_to_cover_filepath_map[manga_name_base64_url] = cover_filepath;
+
+            FILE *file = fopen(cache_filepath.data.c_str(), "ab");
+            if(file) {
+                std::string line = manga_name_base64_url + " " + cover_filepath + "\n";
+                fwrite(line.data(), 1, line.size(), file);
+                fclose(file);
+            }
+        }
+    private:
+        std::unordered_map<std::string, std::string> manga_name_base64_to_cover_filepath_map;
+        Path cache_filepath;
+    };
+
     static const mgl::Color finished_reading_color = mgl::Color(43, 255, 47);
 
     // Pages are sorted from 1.png to n.png
     static std::vector<LocalMangaPage> get_images_in_manga(const Path &directory) {
         std::vector<LocalMangaPage> page_list;
-        for_files_in_dir(directory, [&page_list](const Path &filepath) -> bool {
-            if(get_file_type(filepath) != FileType::REGULAR)
+        for_files_in_dir(directory, [&page_list](const Path &filepath, FileType file_type) -> bool {
+            if(file_type != FileType::REGULAR)
                 return true;
 
             std::string filname_no_ext = filepath.filename_no_ext();
@@ -36,20 +96,33 @@ namespace QuickMedia {
         return page_list;
     }
 
-    static std::vector<LocalMangaChapter> get_chapters_in_manga(const Path &directory) {
+    static std::vector<LocalMangaChapter> get_chapters_in_manga(std::string manga_name, const Path &directory, bool only_include_latest, bool include_pages, bool only_get_coverpage = false) {
         std::vector<LocalMangaChapter> chapter_list;
-        auto callback = [&chapter_list](const Path &filepath) -> bool {
-            if(get_file_type(filepath) != FileType::DIRECTORY)
+        auto callback = [&chapter_list, &manga_name, only_include_latest, include_pages, only_get_coverpage](const Path &filepath, FileType file_type) -> bool {
+            if(file_type != FileType::DIRECTORY)
                 return true;
 
             LocalMangaChapter local_manga_chapter;
             local_manga_chapter.name = filepath.filename();
-            local_manga_chapter.pages = get_images_in_manga(filepath);
-            if(local_manga_chapter.pages.empty() || !file_get_last_modified_time_seconds(filepath.data.c_str(), &local_manga_chapter.modified_time_seconds))
-                return true;
+            if(include_pages) {
+                local_manga_chapter.pages = get_images_in_manga(filepath);
+                if(local_manga_chapter.pages.empty())
+                    return true;
+            } else if(only_get_coverpage) {
+                std::string cover_page = CoverPageLinkCache::get_instance().get_cached_cover_page_link_for_manga(manga_name);
+                if(!cover_page.empty()) {
+                    local_manga_chapter.pages.push_back({ cover_page, 1 });
+                } else {
+                    local_manga_chapter.pages = get_images_in_manga(filepath);
+                    if(local_manga_chapter.pages.empty())
+                        return true;
+
+                    CoverPageLinkCache::get_instance().add_manga_to_thumbnail_link_cache(manga_name, local_manga_chapter.pages.front().path.data);
+                }
+            }
 
             chapter_list.push_back(std::move(local_manga_chapter));
-            return true;
+            return only_include_latest ? false : true;
         };
 
         if(get_config().local_manga_sort_chapters_by_name)
@@ -60,15 +133,15 @@ namespace QuickMedia {
         return chapter_list;
     }
 
-    static std::vector<LocalManga> get_manga_in_directory(const Path &directory) {
+    static std::vector<LocalManga> get_manga_in_directory(const Path &directory, bool only_get_coverpage) {
         std::vector<LocalManga> manga_list;
-        auto callback = [&manga_list](const Path &filepath) -> bool {
-            if(get_file_type(filepath) != FileType::DIRECTORY)
+        auto callback = [&manga_list, only_get_coverpage](const Path &filepath, FileType file_type) -> bool {
+            if(file_type != FileType::DIRECTORY)
                 return true;
 
             LocalManga local_manga;
             local_manga.name = filepath.filename();
-            local_manga.chapters = get_chapters_in_manga(filepath);
+            local_manga.chapters = get_chapters_in_manga(local_manga.name, filepath, true, false, only_get_coverpage);
             if(local_manga.chapters.empty() || !file_get_last_modified_time_seconds(filepath.data.c_str(), &local_manga.modified_time_seconds))
                 return true;
 
@@ -102,7 +175,7 @@ namespace QuickMedia {
         }
 
         Path manga_url = Path(get_config().local_manga_directory).join(manga_name);
-        std::vector<LocalMangaChapter> chapters = get_chapters_in_manga(manga_url);
+        std::vector<LocalMangaChapter> chapters = get_chapters_in_manga(manga_name, manga_url, true, true);
         if(chapters.empty() || chapters.front().pages.empty())
             return false;
 
@@ -229,7 +302,8 @@ namespace QuickMedia {
         body_item->url = local_manga.name;
         body_item->set_description("Latest chapter: " + local_manga.chapters.front().name + "\nUpdated " + seconds_to_relative_time_str(time_now - local_manga.modified_time_seconds));
         body_item->set_description_color(get_theme().faded_text_color);
-        body_item->thumbnail_url = local_manga.chapters.back().pages.front().path.data;
+        if(!local_manga.chapters.back().pages.empty())
+            body_item->thumbnail_url = local_manga.chapters.back().pages.front().path.data;
         body_item->thumbnail_is_local = true;
         body_item->thumbnail_size = {101, 141};
         return body_item;
@@ -258,7 +332,15 @@ namespace QuickMedia {
         }
 
         Path manga_url = Path(get_config().local_manga_directory).join(args.url);
-        std::vector<LocalMangaChapter> chapters = get_chapters_in_manga(manga_url);
+        std::vector<LocalMangaChapter> chapters = get_chapters_in_manga(args.url, manga_url, false, false);
+
+        auto manga_it = std::find_if(manga_list.begin(), manga_list.end(), [&args](const LocalManga &local_manga) {
+            return local_manga.name == args.url;
+        });
+        if(manga_it == manga_list.end()) {
+            show_notification("QuickMedia", "The selected manga seems to have been removed?", Urgency::CRITICAL);
+            return PluginResult::OK;
+        }
 
         const time_t time_now = time(nullptr);
         BodyItems chapters_items;
@@ -266,7 +348,7 @@ namespace QuickMedia {
         for(const LocalMangaChapter &local_manga_chapter : chapters) {
             auto body_item = BodyItem::create(local_manga_chapter.name);
             body_item->url = local_manga_chapter.name;
-            body_item->set_description("Updated " + seconds_to_relative_time_str(time_now - local_manga_chapter.modified_time_seconds));
+            body_item->set_description("Updated " + seconds_to_relative_time_str(time_now - manga_it->modified_time_seconds));
             body_item->set_description_color(get_theme().faded_text_color);
             chapters_items.push_back(std::move(body_item));
         }
@@ -293,7 +375,7 @@ namespace QuickMedia {
             return PluginResult::OK;
         }
 
-        manga_list = get_manga_in_directory(get_config().local_manga_directory);
+        manga_list = get_manga_in_directory(get_config().local_manga_directory, true);
 
         if(standalone)
             finished_reading_manga = get_manga_finished_reading(manga_list);
diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp
index a71b3a8..ba95a4c 100644
--- a/src/plugins/Matrix.cpp
+++ b/src/plugins/Matrix.cpp
@@ -3832,7 +3832,7 @@ namespace QuickMedia {
         remove(matrix_sync_data_path.data.c_str());
         //Path filter_cache_path = get_storage_dir().join("matrix").join("filter");
         //remove(filter_cache_path.data.c_str());
-        for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath) {
+        for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath, FileType) {
             remove(filepath.data.c_str());
             return true;
         });