From 2f9ae9e9462a5a366461f20b4d0c2f4b80ef1b68 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 12 Feb 2022 15:42:16 +0100 Subject: Local-manga: improve loading of page when using slow medium Especially when using NFS. Only get the latest chapter when needed and cache link to the cover page. --- TODO | 3 +- include/Storage.hpp | 6 +-- plugins/LocalManga.hpp | 1 - src/QuickMedia.cpp | 15 ++++-- src/Storage.cpp | 12 +++-- src/plugins/LocalManga.cpp | 118 ++++++++++++++++++++++++++++++++++++++------- src/plugins/Matrix.cpp | 2 +- 7 files changed, 126 insertions(+), 31 deletions(-) diff --git a/TODO b/TODO index 9599d31..3fa13f9 100644 --- a/TODO +++ b/TODO @@ -216,4 +216,5 @@ Allow changing manga sorting in gui (especially for local manga). Allow using ~ and $HOME in config file. Save manga "I" mode to file and load it on startup. Show in bookmarks and history page if a manga has been read (local-manga). -Add "finished reading" to online manga as well, for the manga sites that publish latest chapter in the search page. \ No newline at end of file +Add "finished reading" to online manga as well, for the manga sites that publish latest chapter in the search page. +Async load visible body item content. This is needed for local-manga if the manga is stored on NFS where recursively reading all manga directories is slow. We only want to read recursively for the manga that is visible on the screen. \ No newline at end of file diff --git a/include/Storage.hpp b/include/Storage.hpp index 1e38906..c187261 100644 --- a/include/Storage.hpp +++ b/include/Storage.hpp @@ -9,9 +9,6 @@ namespace Json { } namespace QuickMedia { - // Return false to stop the iterator - using FileIteratorCallback = std::function; - enum class FileType { FILE_NOT_FOUND, REGULAR, @@ -23,6 +20,9 @@ namespace QuickMedia { DESC }; + // Return false to stop the iterator + using FileIteratorCallback = std::function; + Path get_home_dir(); Path get_storage_dir(); Path get_cache_dir(); diff --git a/plugins/LocalManga.hpp b/plugins/LocalManga.hpp index e85e08b..0ab3f62 100644 --- a/plugins/LocalManga.hpp +++ b/plugins/LocalManga.hpp @@ -12,7 +12,6 @@ namespace QuickMedia { struct LocalMangaChapter { std::string name; std::vector pages; - time_t modified_time_seconds; }; struct LocalManga { diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 8174680..edb1db1 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -957,7 +958,7 @@ namespace QuickMedia { show_notification("QuickMedia", "Upgrading mangadex ids", Urgency::LOW); std::vector legacy_manga_ids; - for_files_in_dir_sort_last_modified(content_storage_dir, [&legacy_manga_ids](const Path &filepath) { + for_files_in_dir_sort_last_modified(content_storage_dir, [&legacy_manga_ids](const Path &filepath, FileType) { if(strcmp(filepath.ext(), ".tmp") == 0) return true; @@ -1460,7 +1461,7 @@ namespace QuickMedia { // TODO: Remove this once manga history file has been in use for a few months and is filled with history time_t now = time(NULL); - for_files_in_dir_sort_last_modified(content_storage_dir, [&history_items, plugin_name, &manga_id_to_thumbnail_url_map, now, local_thumbnail](const Path &filepath) { + for_files_in_dir_sort_last_modified(content_storage_dir, [&](const Path &filepath, FileType) { // This can happen when QuickMedia crashes/is killed while writing to storage. // In that case, the storage wont be corrupt but there will be .tmp files. // TODO: Remove these .tmp files if they exist during startup @@ -3387,6 +3388,12 @@ namespace QuickMedia { return PageType::EXIT; } + // TODO: Do the same for thumbnails? + static bool is_symlink_valid(const char *filepath) { + struct stat buf; + return lstat(filepath, &buf) != -1; + } + // TODO: Optimize this somehow. One image alone uses more than 20mb ram! Total ram usage for viewing one image // becomes 40mb (private memory, almost 100mb in total!) Unacceptable! Program::LoadImageResult Program::load_image_by_index(int image_index, mgl::Texture &image_texture, std::string &error_message) { @@ -3401,7 +3408,7 @@ namespace QuickMedia { upscaled_ok = false; } - if(get_file_type(image_path) == FileType::REGULAR && upscaled_ok) { + if(get_file_type(image_path) == FileType::REGULAR && is_symlink_valid(image_path.data.c_str()) && upscaled_ok) { if(image_texture.load_from_file(image_path.data.c_str())) { return LoadImageResult::OK; } else { @@ -3463,7 +3470,7 @@ namespace QuickMedia { upscaled_ok = false; } - if(get_file_type(image_filepath) != FileType::FILE_NOT_FOUND && upscaled_ok) + if(get_file_type(image_filepath) != FileType::FILE_NOT_FOUND && is_symlink_valid(image_filepath.data.c_str()) && upscaled_ok) return true; std::vector extra_args; diff --git a/src/Storage.cpp b/src/Storage.cpp index f4732b8..effa70b 100644 --- a/src/Storage.cpp +++ b/src/Storage.cpp @@ -246,7 +246,9 @@ namespace QuickMedia { void for_files_in_dir(const Path &path, FileIteratorCallback callback) { try { for(auto &p : std::filesystem::directory_iterator(path.data)) { - if(!callback(p.path().string())) + std::error_code ec; + const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR; + if(!callback(p.path().string(), file_type)) break; } } catch(const std::filesystem::filesystem_error &err) { @@ -285,7 +287,9 @@ namespace QuickMedia { } for(auto &p : paths) { - if(!callback(p.path().string())) + std::error_code ec; + const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR; + if(!callback(p.path().string(), file_type)) break; } } @@ -312,7 +316,9 @@ namespace QuickMedia { } for(auto &p : paths) { - if(!callback(p.path().string())) + std::error_code ec; + const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR; + if(!callback(p.path().string(), file_type)) break; } } diff --git a/src/plugins/LocalManga.cpp b/src/plugins/LocalManga.cpp index 8344527..dc98639 100644 --- a/src/plugins/LocalManga.cpp +++ b/src/plugins/LocalManga.cpp @@ -7,15 +7,75 @@ #include "../../external/cppcodec/base64_url.hpp" #include "../../include/QuickMedia.hpp" #include +#include namespace QuickMedia { + // This is needed because the manga may be stored on NFS. + // TODO: Remove once body items can async load when visible on screen + class CoverPageLinkCache { + public: + static CoverPageLinkCache& get_instance() { + static CoverPageLinkCache instance; + Path dir = get_cache_dir().join("thumbnail-link"); + if(create_directory_recursive(dir) != 0) { + show_notification("QuickMedia", "Failed to create directory: " + dir.data, Urgency::CRITICAL); + abort(); + } + + instance.cache_filepath = dir.join("local-manga"); + std::string file_content; + if(get_file_type(instance.cache_filepath) == FileType::REGULAR && file_get_content(instance.cache_filepath, file_content) != 0) { + show_notification("QuickMedia", "Failed to load local manga thumbnail link cache", Urgency::CRITICAL); + abort(); + } + + std::unordered_map manga_map; + string_split(file_content, '\n', [&manga_map](const char *str_part, size_t size) { + const void *space_p = memchr(str_part, ' ', size); + if(!space_p) + return true; + + std::string manga_name_base64(str_part, (const char*)space_p - str_part); + std::string cover_filepath((const char*)space_p + 1, str_part + size - ((const char*)space_p + 1)); + manga_map[std::move(manga_name_base64)] = std::move(cover_filepath); + return true; + }); + + instance.manga_name_base64_to_cover_filepath_map = std::move(manga_map); + return instance; + } + + std::string get_cached_cover_page_link_for_manga(const std::string &manga_name) { + std::string manga_name_base64_url = cppcodec::base64_url::encode(manga_name); + auto it = manga_name_base64_to_cover_filepath_map.find(manga_name_base64_url); + if(it == manga_name_base64_to_cover_filepath_map.end()) + return ""; + return it->second; + } + + void add_manga_to_thumbnail_link_cache(const std::string &manga_name, const std::string &cover_filepath) { + std::string manga_name_base64_url = cppcodec::base64_url::encode(manga_name); + manga_name_base64_to_cover_filepath_map[manga_name_base64_url] = cover_filepath; + + FILE *file = fopen(cache_filepath.data.c_str(), "ab"); + if(file) { + std::string line = manga_name_base64_url + " " + cover_filepath + "\n"; + fwrite(line.data(), 1, line.size(), file); + fclose(file); + } + } + private: + std::unordered_map manga_name_base64_to_cover_filepath_map; + Path cache_filepath; + }; + static const mgl::Color finished_reading_color = mgl::Color(43, 255, 47); // Pages are sorted from 1.png to n.png static std::vector get_images_in_manga(const Path &directory) { std::vector page_list; - for_files_in_dir(directory, [&page_list](const Path &filepath) -> bool { - if(get_file_type(filepath) != FileType::REGULAR) + for_files_in_dir(directory, [&page_list](const Path &filepath, FileType file_type) -> bool { + if(file_type != FileType::REGULAR) return true; std::string filname_no_ext = filepath.filename_no_ext(); @@ -36,20 +96,33 @@ namespace QuickMedia { return page_list; } - static std::vector get_chapters_in_manga(const Path &directory) { + static std::vector get_chapters_in_manga(std::string manga_name, const Path &directory, bool only_include_latest, bool include_pages, bool only_get_coverpage = false) { std::vector chapter_list; - auto callback = [&chapter_list](const Path &filepath) -> bool { - if(get_file_type(filepath) != FileType::DIRECTORY) + auto callback = [&chapter_list, &manga_name, only_include_latest, include_pages, only_get_coverpage](const Path &filepath, FileType file_type) -> bool { + if(file_type != FileType::DIRECTORY) return true; LocalMangaChapter local_manga_chapter; local_manga_chapter.name = filepath.filename(); - local_manga_chapter.pages = get_images_in_manga(filepath); - if(local_manga_chapter.pages.empty() || !file_get_last_modified_time_seconds(filepath.data.c_str(), &local_manga_chapter.modified_time_seconds)) - return true; + if(include_pages) { + local_manga_chapter.pages = get_images_in_manga(filepath); + if(local_manga_chapter.pages.empty()) + return true; + } else if(only_get_coverpage) { + std::string cover_page = CoverPageLinkCache::get_instance().get_cached_cover_page_link_for_manga(manga_name); + if(!cover_page.empty()) { + local_manga_chapter.pages.push_back({ cover_page, 1 }); + } else { + local_manga_chapter.pages = get_images_in_manga(filepath); + if(local_manga_chapter.pages.empty()) + return true; + + CoverPageLinkCache::get_instance().add_manga_to_thumbnail_link_cache(manga_name, local_manga_chapter.pages.front().path.data); + } + } chapter_list.push_back(std::move(local_manga_chapter)); - return true; + return only_include_latest ? false : true; }; if(get_config().local_manga_sort_chapters_by_name) @@ -60,15 +133,15 @@ namespace QuickMedia { return chapter_list; } - static std::vector get_manga_in_directory(const Path &directory) { + static std::vector get_manga_in_directory(const Path &directory, bool only_get_coverpage) { std::vector manga_list; - auto callback = [&manga_list](const Path &filepath) -> bool { - if(get_file_type(filepath) != FileType::DIRECTORY) + auto callback = [&manga_list, only_get_coverpage](const Path &filepath, FileType file_type) -> bool { + if(file_type != FileType::DIRECTORY) return true; LocalManga local_manga; local_manga.name = filepath.filename(); - local_manga.chapters = get_chapters_in_manga(filepath); + local_manga.chapters = get_chapters_in_manga(local_manga.name, filepath, true, false, only_get_coverpage); if(local_manga.chapters.empty() || !file_get_last_modified_time_seconds(filepath.data.c_str(), &local_manga.modified_time_seconds)) return true; @@ -102,7 +175,7 @@ namespace QuickMedia { } Path manga_url = Path(get_config().local_manga_directory).join(manga_name); - std::vector chapters = get_chapters_in_manga(manga_url); + std::vector chapters = get_chapters_in_manga(manga_name, manga_url, true, true); if(chapters.empty() || chapters.front().pages.empty()) return false; @@ -229,7 +302,8 @@ namespace QuickMedia { body_item->url = local_manga.name; body_item->set_description("Latest chapter: " + local_manga.chapters.front().name + "\nUpdated " + seconds_to_relative_time_str(time_now - local_manga.modified_time_seconds)); body_item->set_description_color(get_theme().faded_text_color); - body_item->thumbnail_url = local_manga.chapters.back().pages.front().path.data; + if(!local_manga.chapters.back().pages.empty()) + body_item->thumbnail_url = local_manga.chapters.back().pages.front().path.data; body_item->thumbnail_is_local = true; body_item->thumbnail_size = {101, 141}; return body_item; @@ -258,7 +332,15 @@ namespace QuickMedia { } Path manga_url = Path(get_config().local_manga_directory).join(args.url); - std::vector chapters = get_chapters_in_manga(manga_url); + std::vector chapters = get_chapters_in_manga(args.url, manga_url, false, false); + + auto manga_it = std::find_if(manga_list.begin(), manga_list.end(), [&args](const LocalManga &local_manga) { + return local_manga.name == args.url; + }); + if(manga_it == manga_list.end()) { + show_notification("QuickMedia", "The selected manga seems to have been removed?", Urgency::CRITICAL); + return PluginResult::OK; + } const time_t time_now = time(nullptr); BodyItems chapters_items; @@ -266,7 +348,7 @@ namespace QuickMedia { for(const LocalMangaChapter &local_manga_chapter : chapters) { auto body_item = BodyItem::create(local_manga_chapter.name); body_item->url = local_manga_chapter.name; - body_item->set_description("Updated " + seconds_to_relative_time_str(time_now - local_manga_chapter.modified_time_seconds)); + body_item->set_description("Updated " + seconds_to_relative_time_str(time_now - manga_it->modified_time_seconds)); body_item->set_description_color(get_theme().faded_text_color); chapters_items.push_back(std::move(body_item)); } @@ -293,7 +375,7 @@ namespace QuickMedia { return PluginResult::OK; } - manga_list = get_manga_in_directory(get_config().local_manga_directory); + manga_list = get_manga_in_directory(get_config().local_manga_directory, true); if(standalone) finished_reading_manga = get_manga_finished_reading(manga_list); diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp index a71b3a8..ba95a4c 100644 --- a/src/plugins/Matrix.cpp +++ b/src/plugins/Matrix.cpp @@ -3832,7 +3832,7 @@ namespace QuickMedia { remove(matrix_sync_data_path.data.c_str()); //Path filter_cache_path = get_storage_dir().join("matrix").join("filter"); //remove(filter_cache_path.data.c_str()); - for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath) { + for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath, FileType) { remove(filepath.data.c_str()); return true; }); -- cgit v1.2.3