From 2ba21aa9aa91b975fe0c8be630dde05d0d9b5366 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sun, 13 Oct 2019 03:32:07 +0200 Subject: Manganelo: Download all images at once, and show page after it has downloaded --- src/QuickMedia.cpp | 136 +++++++++++++++++++++++++++++++++++++++++++++- src/Storage.cpp | 26 +++++++-- src/plugins/Manganelo.cpp | 43 +++++++-------- 3 files changed, 175 insertions(+), 30 deletions(-) (limited to 'src') diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index bcd3efe..f277f46 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -150,7 +150,9 @@ namespace QuickMedia { case Page::IMAGES: { body->draw_thumbnails = false; window.setKeyRepeatEnabled(false); + window.setFramerateLimit(4); image_page(); + window.setFramerateLimit(0); window.setKeyRepeatEnabled(true); break; } @@ -309,7 +311,9 @@ namespace QuickMedia { std::string manga_id; if(!manga_extract_id_from_url(content_url, manga_id)) return false; - content_storage_file = content_storage_dir.join(base64_encode(manga_id)); + + manga_id_base64 = base64_encode(manga_id); + content_storage_file = content_storage_dir.join(manga_id_base64); content_storage_json.clear(); content_storage_json["name"] = content_title; FileType file_type = get_file_type(content_storage_file); @@ -792,6 +796,90 @@ namespace QuickMedia { } } + Program::LoadImageResult Program::load_image_by_index(int image_index, sf::Texture &image_texture, sf::String &error_message) { + Path image_path = content_cache_dir; + image_path.join(std::to_string(image_index + 1)); + + Path image_finished_path(image_path.data + ".finished"); + if(get_file_type(image_finished_path) != FileType::FILE_NOT_FOUND && get_file_type(image_path) == FileType::REGULAR) { + std::string image_data; + if(file_get_content(image_path, image_data) == 0) { + if(image_texture.loadFromMemory(image_data.data(), image_data.size())) { + image_texture.setSmooth(true); + image_texture.generateMipmap(); + return LoadImageResult::OK; + } else { + error_message = std::string("Failed to load image for page ") + std::to_string(image_index + 1); + return LoadImageResult::FAILED; + } + } else { + show_notification("Manganelo", "Failed to load image for page " + std::to_string(image_index + 1) + ". Image filepath: " + image_path.data, Urgency::CRITICAL); + error_message = std::string("Failed to load image for page ") + std::to_string(image_index + 1); + return LoadImageResult::FAILED; + } + } else { + error_message = "Downloading page " + std::to_string(image_index + 1) + "..."; + return LoadImageResult::DOWNLOAD_IN_PROGRESS; + } + } + + void Program::download_chapter_images_if_needed(Manganelo *image_plugin) { + if(downloading_chapter_url == images_url) + return; + + downloading_chapter_url = images_url; + if(image_download_future.valid()) { + image_download_cancel = true; + image_download_future.get(); + image_download_cancel = false; + } + + std::string chapter_url = images_url; + Path content_cache_dir_ = content_cache_dir; + image_download_future = std::async(std::launch::async, [chapter_url, image_plugin, content_cache_dir_, this]() { + // TODO: Download images in parallel + int page = 1; + image_plugin->for_each_page_in_chapter(chapter_url, [content_cache_dir_, &page, this](const std::string &url) { + if(image_download_cancel) + return false; + #if 0 + size_t last_index = url.find_last_of('/'); + if(last_index == std::string::npos || (int)url.size() - (int)last_index + 1 <= 0) { + show_notification("Manganelo", "Image url is in incorrect format, missing '/': " + url, Urgency::CRITICAL); + return false; + } + + std::string image_filename = url.substr(last_index + 1); + Path image_filepath = content_cache_dir_; + image_filepath.join(image_filename); + #endif + Path image_filepath = content_cache_dir_; + image_filepath.join(std::to_string(page++)); + + Path lockfile_path(image_filepath.data + ".finished"); + if(get_file_type(lockfile_path) != FileType::FILE_NOT_FOUND) + return true; + + std::string image_content; + if(download_to_string(url, image_content) != DownloadResult::OK) { + show_notification("Manganelo", "Failed to download image: " + url, Urgency::CRITICAL); + return false; + } + + if(file_overwrite(image_filepath, image_content) != 0) { + show_notification("Storage", "Failed to save image to file: " + image_filepath.data, Urgency::CRITICAL); + return false; + } + + if(create_lock_file(lockfile_path) != 0) { + show_notification("Storage", "Failed to save image finished state to file: " + lockfile_path.data, Urgency::CRITICAL); + return false; + } + return true; + }); + }); + } + void Program::image_page() { search_bar->onTextUpdateCallback = nullptr; search_bar->onTextSubmitCallback = nullptr; @@ -801,11 +889,22 @@ namespace QuickMedia { sf::Text error_message("", font, 30); error_message.setFillColor(sf::Color::White); + assert(current_plugin->name == "manganelo"); Manganelo *image_plugin = static_cast(current_plugin); std::string image_data; + bool download_in_progress = false; + + content_cache_dir = get_cache_dir().join("manga").join(manga_id_base64).join(base64_encode(chapter_title)); + if(create_directory_recursive(content_cache_dir) != 0) { + show_notification("Storage", "Failed to create directory: " + content_cache_dir.data, Urgency::CRITICAL); + current_page = Page::EPISODE_LIST; + return; + } + download_chapter_images_if_needed(image_plugin); // TODO: Optimize this somehow. One image alone uses more than 20mb ram! Total ram usage for viewing one image // becomes 40mb (private memory, almost 100mb in total!) Unacceptable! + #if 0 ImageResult image_result = image_plugin->get_image_by_index(images_url, image_index, image_data); if(image_result == ImageResult::OK) { if(image_texture.loadFromMemory(image_data.data(), image_data.size())) { @@ -822,11 +921,24 @@ namespace QuickMedia { error_message.setString(std::string("Network error, failed to get image for page ") + std::to_string(image_index + 1)); } image_data.resize(0); + #endif int num_images = 0; image_plugin->get_number_of_images(images_url, num_images); image_index = std::min(image_index, num_images); + if(image_index < num_images) { + sf::String error_msg; + LoadImageResult load_image_result = load_image_by_index(image_index, image_texture, error_msg); + if(load_image_result == LoadImageResult::OK) + image.setTexture(image_texture, true); + else if(load_image_result == LoadImageResult::DOWNLOAD_IN_PROGRESS) + download_in_progress = true; + error_message.setString(error_msg); + } else if(image_index == num_images) { + error_message.setString("End of " + chapter_title); + } + Json::Value &json_chapters = content_storage_json["chapters"]; Json::Value json_chapter; int latest_read = image_index + 1; @@ -868,9 +980,12 @@ namespace QuickMedia { texture_size_f = sf::Vector2f(texture_size.x, texture_size.y); } + sf::Clock check_downloaded_timer; + const sf::Int32 check_downloaded_timeout_ms = 1000; + // TODO: Show to user if a certain page is missing (by checking page name (number) and checking if some are skipped) while (current_page == Page::IMAGES) { - if(window.waitEvent(event)) { + while(window.pollEvent(event)) { if (event.type == sf::Event::Closed) { current_page = Page::EXIT; } else if(event.type == sf::Event::Resized) { @@ -907,6 +1022,23 @@ namespace QuickMedia { } } + if(download_in_progress && check_downloaded_timer.getElapsedTime().asMilliseconds() >= check_downloaded_timeout_ms) { + sf::String error_msg; + LoadImageResult load_image_result = load_image_by_index(image_index, image_texture, error_msg); + if(load_image_result == LoadImageResult::OK) { + image.setTexture(image_texture, true); + download_in_progress = false; + error = false; + texture_size = image.getTexture()->getSize(); + texture_size_f = sf::Vector2f(texture_size.x, texture_size.y); + } else if(load_image_result == LoadImageResult::FAILED) { + download_in_progress = false; + } + error_message.setString(error_msg); + resized = true; + check_downloaded_timer.restart(); + } + const float font_height = chapter_text.getCharacterSize() + 8.0f; const float background_height = font_height + 6.0f; diff --git a/src/Storage.cpp b/src/Storage.cpp index f75f6be..1a199d9 100644 --- a/src/Storage.cpp +++ b/src/Storage.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #endif static int makedir(const char *path) { @@ -49,12 +51,16 @@ namespace QuickMedia { return get_home_dir().join(".config").join("quickmedia"); } + Path get_cache_dir() { + return get_home_dir().join(".cache").join("quickmedia"); + } + int create_directory_recursive(const Path &path) { size_t index = 0; while(true) { index = path.data.find('/', index); - // Skips first '/' on unix-like systems + // Skips first '/', we don't want to try and create the root directory if(index == 0) { ++index; continue; @@ -100,10 +106,20 @@ namespace QuickMedia { int file_overwrite(const Path &path, const std::string &data) { FILE *file = fopen(path.data.c_str(), "wb"); if(!file) - return -errno; + return errno; - fwrite(data.data(), 1, data.size(), file); - fclose(file); - return 0; + if(fwrite(data.data(), 1, data.size(), file) != data.size()) { + fclose(file); + return -1; + } + + return fclose(file); + } + + int create_lock_file(const Path &path) { + int fd = open(path.data.c_str(), O_CREAT | O_EXCL); + if(fd == -1) + return errno; + return close(fd); } } \ No newline at end of file diff --git a/src/plugins/Manganelo.cpp b/src/plugins/Manganelo.cpp index cd22cb0..7af35a6 100644 --- a/src/plugins/Manganelo.cpp +++ b/src/plugins/Manganelo.cpp @@ -93,29 +93,8 @@ namespace QuickMedia { return SuggestionResult::OK; } - ImageResult Manganelo::get_image_by_index(const std::string &url, int index, std::string &image_data) { - ImageResult image_result = get_image_urls_for_chapter(url); - if(image_result != ImageResult::OK) - return image_result; - - int num_images = last_chapter_image_urls.size(); - if(index < 0 || index >= num_images) - return ImageResult::END; - - // TODO: Cache image in file/memory - switch(download_to_string(last_chapter_image_urls[index], image_data)) { - case DownloadResult::OK: - return ImageResult::OK; - case DownloadResult::ERR: - return ImageResult::ERR; - case DownloadResult::NET_ERR: - return ImageResult::NET_ERR; - default: - return ImageResult::ERR; - } - } - ImageResult Manganelo::get_number_of_images(const std::string &url, int &num_images) { + std::lock_guard lock(image_urls_mutex); num_images = 0; ImageResult image_result = get_image_urls_for_chapter(url); if(image_result != ImageResult::OK) @@ -147,7 +126,7 @@ namespace QuickMedia { if(src) { // TODO: If image loads too slow, try switching mirror std::string image_url = src; - string_replace_all(image_url, "s3.mkklcdnv3.com", "bu.mkklcdnbuv1.com"); + //string_replace_all(image_url, "s3.mkklcdnv3.com", "bu.mkklcdnbuv1.com"); urls->emplace_back(std::move(image_url)); } }, &last_chapter_image_urls); @@ -158,4 +137,22 @@ namespace QuickMedia { last_chapter_url = url; return result == 0 ? ImageResult::OK : ImageResult::ERR; } + + ImageResult Manganelo::for_each_page_in_chapter(const std::string &chapter_url, PageCallback callback) { + std::vector image_urls; + { + std::lock_guard lock(image_urls_mutex); + ImageResult image_result = get_image_urls_for_chapter(chapter_url); + if(image_result != ImageResult::OK) + return image_result; + + image_urls = last_chapter_image_urls; + } + + for(const std::string &url : image_urls) { + if(!callback(url)) + break; + } + return ImageResult::OK; + } } \ No newline at end of file -- cgit v1.2.3