From 2f9ae9e9462a5a366461f20b4d0c2f4b80ef1b68 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 12 Feb 2022 15:42:16 +0100 Subject: Local-manga: improve loading of page when using slow medium Especially when using NFS. Only get the latest chapter when needed and cache link to the cover page. --- src/QuickMedia.cpp | 15 ++++-- src/Storage.cpp | 12 +++-- src/plugins/LocalManga.cpp | 118 ++++++++++++++++++++++++++++++++++++++------- src/plugins/Matrix.cpp | 2 +- 4 files changed, 121 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 8174680..edb1db1 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -957,7 +958,7 @@ namespace QuickMedia { show_notification("QuickMedia", "Upgrading mangadex ids", Urgency::LOW); std::vector legacy_manga_ids; - for_files_in_dir_sort_last_modified(content_storage_dir, [&legacy_manga_ids](const Path &filepath) { + for_files_in_dir_sort_last_modified(content_storage_dir, [&legacy_manga_ids](const Path &filepath, FileType) { if(strcmp(filepath.ext(), ".tmp") == 0) return true; @@ -1460,7 +1461,7 @@ namespace QuickMedia { // TODO: Remove this once manga history file has been in use for a few months and is filled with history time_t now = time(NULL); - for_files_in_dir_sort_last_modified(content_storage_dir, [&history_items, plugin_name, &manga_id_to_thumbnail_url_map, now, local_thumbnail](const Path &filepath) { + for_files_in_dir_sort_last_modified(content_storage_dir, [&](const Path &filepath, FileType) { // This can happen when QuickMedia crashes/is killed while writing to storage. // In that case, the storage wont be corrupt but there will be .tmp files. // TODO: Remove these .tmp files if they exist during startup @@ -3387,6 +3388,12 @@ namespace QuickMedia { return PageType::EXIT; } + // TODO: Do the same for thumbnails? + static bool is_symlink_valid(const char *filepath) { + struct stat buf; + return lstat(filepath, &buf) != -1; + } + // TODO: Optimize this somehow. One image alone uses more than 20mb ram! Total ram usage for viewing one image // becomes 40mb (private memory, almost 100mb in total!) Unacceptable! Program::LoadImageResult Program::load_image_by_index(int image_index, mgl::Texture &image_texture, std::string &error_message) { @@ -3401,7 +3408,7 @@ namespace QuickMedia { upscaled_ok = false; } - if(get_file_type(image_path) == FileType::REGULAR && upscaled_ok) { + if(get_file_type(image_path) == FileType::REGULAR && is_symlink_valid(image_path.data.c_str()) && upscaled_ok) { if(image_texture.load_from_file(image_path.data.c_str())) { return LoadImageResult::OK; } else { @@ -3463,7 +3470,7 @@ namespace QuickMedia { upscaled_ok = false; } - if(get_file_type(image_filepath) != FileType::FILE_NOT_FOUND && upscaled_ok) + if(get_file_type(image_filepath) != FileType::FILE_NOT_FOUND && is_symlink_valid(image_filepath.data.c_str()) && upscaled_ok) return true; std::vector extra_args; diff --git a/src/Storage.cpp b/src/Storage.cpp index f4732b8..effa70b 100644 --- a/src/Storage.cpp +++ b/src/Storage.cpp @@ -246,7 +246,9 @@ namespace QuickMedia { void for_files_in_dir(const Path &path, FileIteratorCallback callback) { try { for(auto &p : std::filesystem::directory_iterator(path.data)) { - if(!callback(p.path().string())) + std::error_code ec; + const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR; + if(!callback(p.path().string(), file_type)) break; } } catch(const std::filesystem::filesystem_error &err) { @@ -285,7 +287,9 @@ namespace QuickMedia { } for(auto &p : paths) { - if(!callback(p.path().string())) + std::error_code ec; + const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR; + if(!callback(p.path().string(), file_type)) break; } } @@ -312,7 +316,9 @@ namespace QuickMedia { } for(auto &p : paths) { - if(!callback(p.path().string())) + std::error_code ec; + const FileType file_type = p.is_directory(ec) ? FileType::DIRECTORY : FileType::REGULAR; + if(!callback(p.path().string(), file_type)) break; } } diff --git a/src/plugins/LocalManga.cpp b/src/plugins/LocalManga.cpp index 8344527..dc98639 100644 --- a/src/plugins/LocalManga.cpp +++ b/src/plugins/LocalManga.cpp @@ -7,15 +7,75 @@ #include "../../external/cppcodec/base64_url.hpp" #include "../../include/QuickMedia.hpp" #include +#include namespace QuickMedia { + // This is needed because the manga may be stored on NFS. + // TODO: Remove once body items can async load when visible on screen + class CoverPageLinkCache { + public: + static CoverPageLinkCache& get_instance() { + static CoverPageLinkCache instance; + Path dir = get_cache_dir().join("thumbnail-link"); + if(create_directory_recursive(dir) != 0) { + show_notification("QuickMedia", "Failed to create directory: " + dir.data, Urgency::CRITICAL); + abort(); + } + + instance.cache_filepath = dir.join("local-manga"); + std::string file_content; + if(get_file_type(instance.cache_filepath) == FileType::REGULAR && file_get_content(instance.cache_filepath, file_content) != 0) { + show_notification("QuickMedia", "Failed to load local manga thumbnail link cache", Urgency::CRITICAL); + abort(); + } + + std::unordered_map manga_map; + string_split(file_content, '\n', [&manga_map](const char *str_part, size_t size) { + const void *space_p = memchr(str_part, ' ', size); + if(!space_p) + return true; + + std::string manga_name_base64(str_part, (const char*)space_p - str_part); + std::string cover_filepath((const char*)space_p + 1, str_part + size - ((const char*)space_p + 1)); + manga_map[std::move(manga_name_base64)] = std::move(cover_filepath); + return true; + }); + + instance.manga_name_base64_to_cover_filepath_map = std::move(manga_map); + return instance; + } + + std::string get_cached_cover_page_link_for_manga(const std::string &manga_name) { + std::string manga_name_base64_url = cppcodec::base64_url::encode(manga_name); + auto it = manga_name_base64_to_cover_filepath_map.find(manga_name_base64_url); + if(it == manga_name_base64_to_cover_filepath_map.end()) + return ""; + return it->second; + } + + void add_manga_to_thumbnail_link_cache(const std::string &manga_name, const std::string &cover_filepath) { + std::string manga_name_base64_url = cppcodec::base64_url::encode(manga_name); + manga_name_base64_to_cover_filepath_map[manga_name_base64_url] = cover_filepath; + + FILE *file = fopen(cache_filepath.data.c_str(), "ab"); + if(file) { + std::string line = manga_name_base64_url + " " + cover_filepath + "\n"; + fwrite(line.data(), 1, line.size(), file); + fclose(file); + } + } + private: + std::unordered_map manga_name_base64_to_cover_filepath_map; + Path cache_filepath; + }; + static const mgl::Color finished_reading_color = mgl::Color(43, 255, 47); // Pages are sorted from 1.png to n.png static std::vector get_images_in_manga(const Path &directory) { std::vector page_list; - for_files_in_dir(directory, [&page_list](const Path &filepath) -> bool { - if(get_file_type(filepath) != FileType::REGULAR) + for_files_in_dir(directory, [&page_list](const Path &filepath, FileType file_type) -> bool { + if(file_type != FileType::REGULAR) return true; std::string filname_no_ext = filepath.filename_no_ext(); @@ -36,20 +96,33 @@ namespace QuickMedia { return page_list; } - static std::vector get_chapters_in_manga(const Path &directory) { + static std::vector get_chapters_in_manga(std::string manga_name, const Path &directory, bool only_include_latest, bool include_pages, bool only_get_coverpage = false) { std::vector chapter_list; - auto callback = [&chapter_list](const Path &filepath) -> bool { - if(get_file_type(filepath) != FileType::DIRECTORY) + auto callback = [&chapter_list, &manga_name, only_include_latest, include_pages, only_get_coverpage](const Path &filepath, FileType file_type) -> bool { + if(file_type != FileType::DIRECTORY) return true; LocalMangaChapter local_manga_chapter; local_manga_chapter.name = filepath.filename(); - local_manga_chapter.pages = get_images_in_manga(filepath); - if(local_manga_chapter.pages.empty() || !file_get_last_modified_time_seconds(filepath.data.c_str(), &local_manga_chapter.modified_time_seconds)) - return true; + if(include_pages) { + local_manga_chapter.pages = get_images_in_manga(filepath); + if(local_manga_chapter.pages.empty()) + return true; + } else if(only_get_coverpage) { + std::string cover_page = CoverPageLinkCache::get_instance().get_cached_cover_page_link_for_manga(manga_name); + if(!cover_page.empty()) { + local_manga_chapter.pages.push_back({ cover_page, 1 }); + } else { + local_manga_chapter.pages = get_images_in_manga(filepath); + if(local_manga_chapter.pages.empty()) + return true; + + CoverPageLinkCache::get_instance().add_manga_to_thumbnail_link_cache(manga_name, local_manga_chapter.pages.front().path.data); + } + } chapter_list.push_back(std::move(local_manga_chapter)); - return true; + return only_include_latest ? false : true; }; if(get_config().local_manga_sort_chapters_by_name) @@ -60,15 +133,15 @@ namespace QuickMedia { return chapter_list; } - static std::vector get_manga_in_directory(const Path &directory) { + static std::vector get_manga_in_directory(const Path &directory, bool only_get_coverpage) { std::vector manga_list; - auto callback = [&manga_list](const Path &filepath) -> bool { - if(get_file_type(filepath) != FileType::DIRECTORY) + auto callback = [&manga_list, only_get_coverpage](const Path &filepath, FileType file_type) -> bool { + if(file_type != FileType::DIRECTORY) return true; LocalManga local_manga; local_manga.name = filepath.filename(); - local_manga.chapters = get_chapters_in_manga(filepath); + local_manga.chapters = get_chapters_in_manga(local_manga.name, filepath, true, false, only_get_coverpage); if(local_manga.chapters.empty() || !file_get_last_modified_time_seconds(filepath.data.c_str(), &local_manga.modified_time_seconds)) return true; @@ -102,7 +175,7 @@ namespace QuickMedia { } Path manga_url = Path(get_config().local_manga_directory).join(manga_name); - std::vector chapters = get_chapters_in_manga(manga_url); + std::vector chapters = get_chapters_in_manga(manga_name, manga_url, true, true); if(chapters.empty() || chapters.front().pages.empty()) return false; @@ -229,7 +302,8 @@ namespace QuickMedia { body_item->url = local_manga.name; body_item->set_description("Latest chapter: " + local_manga.chapters.front().name + "\nUpdated " + seconds_to_relative_time_str(time_now - local_manga.modified_time_seconds)); body_item->set_description_color(get_theme().faded_text_color); - body_item->thumbnail_url = local_manga.chapters.back().pages.front().path.data; + if(!local_manga.chapters.back().pages.empty()) + body_item->thumbnail_url = local_manga.chapters.back().pages.front().path.data; body_item->thumbnail_is_local = true; body_item->thumbnail_size = {101, 141}; return body_item; @@ -258,7 +332,15 @@ namespace QuickMedia { } Path manga_url = Path(get_config().local_manga_directory).join(args.url); - std::vector chapters = get_chapters_in_manga(manga_url); + std::vector chapters = get_chapters_in_manga(args.url, manga_url, false, false); + + auto manga_it = std::find_if(manga_list.begin(), manga_list.end(), [&args](const LocalManga &local_manga) { + return local_manga.name == args.url; + }); + if(manga_it == manga_list.end()) { + show_notification("QuickMedia", "The selected manga seems to have been removed?", Urgency::CRITICAL); + return PluginResult::OK; + } const time_t time_now = time(nullptr); BodyItems chapters_items; @@ -266,7 +348,7 @@ namespace QuickMedia { for(const LocalMangaChapter &local_manga_chapter : chapters) { auto body_item = BodyItem::create(local_manga_chapter.name); body_item->url = local_manga_chapter.name; - body_item->set_description("Updated " + seconds_to_relative_time_str(time_now - local_manga_chapter.modified_time_seconds)); + body_item->set_description("Updated " + seconds_to_relative_time_str(time_now - manga_it->modified_time_seconds)); body_item->set_description_color(get_theme().faded_text_color); chapters_items.push_back(std::move(body_item)); } @@ -293,7 +375,7 @@ namespace QuickMedia { return PluginResult::OK; } - manga_list = get_manga_in_directory(get_config().local_manga_directory); + manga_list = get_manga_in_directory(get_config().local_manga_directory, true); if(standalone) finished_reading_manga = get_manga_finished_reading(manga_list); diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp index a71b3a8..ba95a4c 100644 --- a/src/plugins/Matrix.cpp +++ b/src/plugins/Matrix.cpp @@ -3832,7 +3832,7 @@ namespace QuickMedia { remove(matrix_sync_data_path.data.c_str()); //Path filter_cache_path = get_storage_dir().join("matrix").join("filter"); //remove(filter_cache_path.data.c_str()); - for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath) { + for_files_in_dir(get_cache_dir().join("matrix").join("events"), [](const Path &filepath, FileType) { remove(filepath.data.c_str()); return true; }); -- cgit v1.2.3