diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | include/QuickMedia.hpp | 14 | ||||
| -rw-r--r-- | include/Storage.hpp | 2 | ||||
| -rw-r--r-- | plugins/Fourchan.hpp | 1 | ||||
| -rw-r--r-- | plugins/Manganelo.hpp | 9 | ||||
| -rw-r--r-- | plugins/Plugin.hpp | 3 | ||||
| -rw-r--r-- | plugins/Pornhub.hpp | 1 | ||||
| -rw-r--r-- | plugins/Youtube.hpp | 1 | ||||
| -rw-r--r-- | src/QuickMedia.cpp | 136 | ||||
| -rw-r--r-- | src/Storage.cpp | 26 | ||||
| -rw-r--r-- | src/plugins/Manganelo.cpp | 43 |
11 files changed, 209 insertions, 31 deletions
@@ -33,4 +33,6 @@ Add scrollbar.\ Add option to scale image to window size.\ The search should wait until there are search results before clearing the search field and selecting the search suggestion.\ Somehow deal with youtube banning ip when searching too often.\ -Optimize shadow rendering for items (Right now they fill too much space that is behind items). It should also be a blurry shadow.
\ No newline at end of file +Optimize shadow rendering for items (Right now they fill too much space that is behind items). It should also be a blurry shadow.\ +When continuing to read manga from a different page from the first and there is no cache for the chapter, +then start downloading from the current page instead of page 1.
\ No newline at end of file diff --git a/include/QuickMedia.hpp b/include/QuickMedia.hpp index 70c7001..8479d58 100644 --- a/include/QuickMedia.hpp +++ b/include/QuickMedia.hpp @@ -14,6 +14,7 @@ namespace QuickMedia { class Plugin; + class Manganelo; class Program { public: @@ -30,6 +31,14 @@ namespace QuickMedia { void content_list_page(); void content_details_page(); + enum class LoadImageResult { + OK, + FAILED, + DOWNLOAD_IN_PROGRESS + }; + + LoadImageResult load_image_by_index(int image_index, sf::Texture &image_texture, sf::String &error_message); + void download_chapter_images_if_needed(Manganelo *image_plugin); void select_episode(BodyItem *item, bool start_from_beginning); private: sf::RenderWindow window; @@ -49,8 +58,13 @@ namespace QuickMedia { std::string chapter_title; int image_index; Path content_storage_file; + Path content_cache_dir; + std::string manga_id_base64; Json::Value content_storage_json; std::unordered_set<std::string> watched_videos; std::future<BodyItems> search_suggestion_future; + std::future<void> image_download_future; + std::string downloading_chapter_url; + bool image_download_cancel; }; }
\ No newline at end of file diff --git a/include/Storage.hpp b/include/Storage.hpp index bd4283c..5b35ba7 100644 --- a/include/Storage.hpp +++ b/include/Storage.hpp @@ -11,8 +11,10 @@ namespace QuickMedia { Path get_home_dir(); Path get_storage_dir(); + Path get_cache_dir(); int create_directory_recursive(const Path &path); FileType get_file_type(const Path &path); int file_get_content(const Path &path, std::string &result); int file_overwrite(const Path &path, const std::string &data); + int create_lock_file(const Path &path); }
\ No newline at end of file diff --git a/plugins/Fourchan.hpp b/plugins/Fourchan.hpp index d6b482b..7f9ad13 100644 --- a/plugins/Fourchan.hpp +++ b/plugins/Fourchan.hpp @@ -5,6 +5,7 @@ namespace QuickMedia { class Fourchan : public Plugin { public: + Fourchan() : Plugin("4chan") {} PluginResult get_front_page(BodyItems &result_items) override; SearchResult search(const std::string &url, BodyItems &result_items) override; SuggestionResult update_search_suggestions(const std::string &text, BodyItems &result_items) override; diff --git a/plugins/Manganelo.hpp b/plugins/Manganelo.hpp index ab3bb3f..3405de2 100644 --- a/plugins/Manganelo.hpp +++ b/plugins/Manganelo.hpp @@ -1,10 +1,16 @@ #pragma once #include "Plugin.hpp" +#include <functional> +#include <mutex> namespace QuickMedia { + // Return false to stop iteration + using PageCallback = std::function<bool(const std::string &url)>; + class Manganelo : public Plugin { public: + Manganelo() : Plugin("manganelo") {} SearchResult search(const std::string &url, BodyItems &result_items) override; SuggestionResult update_search_suggestions(const std::string &text, BodyItems &result_items) override; ImageResult get_image_by_index(const std::string &url, int index, std::string &image_data); @@ -13,11 +19,14 @@ namespace QuickMedia { bool search_results_has_thumbnails() const override { return false; } int get_search_delay() const override { return 150; } Page get_page_after_search() const override { return Page::EPISODE_LIST; } + + ImageResult for_each_page_in_chapter(const std::string &chapter_url, PageCallback callback); private: // Caches url. If the same url is requested multiple times then the cache is used ImageResult get_image_urls_for_chapter(const std::string &url); private: std::string last_chapter_url; std::vector<std::string> last_chapter_image_urls; + std::mutex image_urls_mutex; }; }
\ No newline at end of file diff --git a/plugins/Plugin.hpp b/plugins/Plugin.hpp index d5802af..dffe898 100644 --- a/plugins/Plugin.hpp +++ b/plugins/Plugin.hpp @@ -50,6 +50,7 @@ namespace QuickMedia { class Plugin { public: + Plugin(const std::string &name) : name(name) {} virtual ~Plugin() = default; virtual PluginResult get_front_page(BodyItems &result_items) { @@ -74,6 +75,8 @@ namespace QuickMedia { virtual int get_search_delay() const = 0; virtual bool search_suggestion_is_search() const { return false; } virtual Page get_page_after_search() const = 0; + + const std::string name; protected: std::string url_param_encode(const std::string ¶m) const; }; diff --git a/plugins/Pornhub.hpp b/plugins/Pornhub.hpp index edbfdab..188a68e 100644 --- a/plugins/Pornhub.hpp +++ b/plugins/Pornhub.hpp @@ -5,6 +5,7 @@ namespace QuickMedia { class Pornhub : public Plugin { public: + Pornhub() : Plugin("pornhub") {} SuggestionResult update_search_suggestions(const std::string &text, BodyItems &result_items) override; BodyItems get_related_media(const std::string &url) override; bool search_suggestions_has_thumbnails() const override { return true; } diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp index 4ca7955..a676cd0 100644 --- a/plugins/Youtube.hpp +++ b/plugins/Youtube.hpp @@ -5,6 +5,7 @@ namespace QuickMedia { class Youtube : public Plugin { public: + Youtube() : Plugin("youtube") {} SuggestionResult update_search_suggestions(const std::string &text, BodyItems &result_items) override; BodyItems get_related_media(const std::string &url) override; bool search_suggestions_has_thumbnails() const override { return true; } diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index bcd3efe..f277f46 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -150,7 +150,9 @@ namespace QuickMedia { case Page::IMAGES: { body->draw_thumbnails = false; window.setKeyRepeatEnabled(false); + window.setFramerateLimit(4); image_page(); + window.setFramerateLimit(0); window.setKeyRepeatEnabled(true); break; } @@ -309,7 +311,9 @@ namespace QuickMedia { std::string manga_id; if(!manga_extract_id_from_url(content_url, manga_id)) return false; - content_storage_file = content_storage_dir.join(base64_encode(manga_id)); + + manga_id_base64 = base64_encode(manga_id); + content_storage_file = content_storage_dir.join(manga_id_base64); content_storage_json.clear(); content_storage_json["name"] = content_title; FileType file_type = get_file_type(content_storage_file); @@ -792,6 +796,90 @@ namespace QuickMedia { } } + Program::LoadImageResult Program::load_image_by_index(int image_index, sf::Texture &image_texture, sf::String &error_message) { + Path image_path = content_cache_dir; + image_path.join(std::to_string(image_index + 1)); + + Path image_finished_path(image_path.data + ".finished"); + if(get_file_type(image_finished_path) != FileType::FILE_NOT_FOUND && get_file_type(image_path) == FileType::REGULAR) { + std::string image_data; + if(file_get_content(image_path, image_data) == 0) { + if(image_texture.loadFromMemory(image_data.data(), image_data.size())) { + image_texture.setSmooth(true); + image_texture.generateMipmap(); + return LoadImageResult::OK; + } else { + error_message = std::string("Failed to load image for page ") + std::to_string(image_index + 1); + return LoadImageResult::FAILED; + } + } else { + show_notification("Manganelo", "Failed to load image for page " + std::to_string(image_index + 1) + ". Image filepath: " + image_path.data, Urgency::CRITICAL); + error_message = std::string("Failed to load image for page ") + std::to_string(image_index + 1); + return LoadImageResult::FAILED; + } + } else { + error_message = "Downloading page " + std::to_string(image_index + 1) + "..."; + return LoadImageResult::DOWNLOAD_IN_PROGRESS; + } + } + + void Program::download_chapter_images_if_needed(Manganelo *image_plugin) { + if(downloading_chapter_url == images_url) + return; + + downloading_chapter_url = images_url; + if(image_download_future.valid()) { + image_download_cancel = true; + image_download_future.get(); + image_download_cancel = false; + } + + std::string chapter_url = images_url; + Path content_cache_dir_ = content_cache_dir; + image_download_future = std::async(std::launch::async, [chapter_url, image_plugin, content_cache_dir_, this]() { + // TODO: Download images in parallel + int page = 1; + image_plugin->for_each_page_in_chapter(chapter_url, [content_cache_dir_, &page, this](const std::string &url) { + if(image_download_cancel) + return false; + #if 0 + size_t last_index = url.find_last_of('/'); + if(last_index == std::string::npos || (int)url.size() - (int)last_index + 1 <= 0) { + show_notification("Manganelo", "Image url is in incorrect format, missing '/': " + url, Urgency::CRITICAL); + return false; + } + + std::string image_filename = url.substr(last_index + 1); + Path image_filepath = content_cache_dir_; + image_filepath.join(image_filename); + #endif + Path image_filepath = content_cache_dir_; + image_filepath.join(std::to_string(page++)); + + Path lockfile_path(image_filepath.data + ".finished"); + if(get_file_type(lockfile_path) != FileType::FILE_NOT_FOUND) + return true; + + std::string image_content; + if(download_to_string(url, image_content) != DownloadResult::OK) { + show_notification("Manganelo", "Failed to download image: " + url, Urgency::CRITICAL); + return false; + } + + if(file_overwrite(image_filepath, image_content) != 0) { + show_notification("Storage", "Failed to save image to file: " + image_filepath.data, Urgency::CRITICAL); + return false; + } + + if(create_lock_file(lockfile_path) != 0) { + show_notification("Storage", "Failed to save image finished state to file: " + lockfile_path.data, Urgency::CRITICAL); + return false; + } + return true; + }); + }); + } + void Program::image_page() { search_bar->onTextUpdateCallback = nullptr; search_bar->onTextSubmitCallback = nullptr; @@ -801,11 +889,22 @@ namespace QuickMedia { sf::Text error_message("", font, 30); error_message.setFillColor(sf::Color::White); + assert(current_plugin->name == "manganelo"); Manganelo *image_plugin = static_cast<Manganelo*>(current_plugin); std::string image_data; + bool download_in_progress = false; + + content_cache_dir = get_cache_dir().join("manga").join(manga_id_base64).join(base64_encode(chapter_title)); + if(create_directory_recursive(content_cache_dir) != 0) { + show_notification("Storage", "Failed to create directory: " + content_cache_dir.data, Urgency::CRITICAL); + current_page = Page::EPISODE_LIST; + return; + } + download_chapter_images_if_needed(image_plugin); // TODO: Optimize this somehow. One image alone uses more than 20mb ram! Total ram usage for viewing one image // becomes 40mb (private memory, almost 100mb in total!) Unacceptable! + #if 0 ImageResult image_result = image_plugin->get_image_by_index(images_url, image_index, image_data); if(image_result == ImageResult::OK) { if(image_texture.loadFromMemory(image_data.data(), image_data.size())) { @@ -822,11 +921,24 @@ namespace QuickMedia { error_message.setString(std::string("Network error, failed to get image for page ") + std::to_string(image_index + 1)); } image_data.resize(0); + #endif int num_images = 0; image_plugin->get_number_of_images(images_url, num_images); image_index = std::min(image_index, num_images); + if(image_index < num_images) { + sf::String error_msg; + LoadImageResult load_image_result = load_image_by_index(image_index, image_texture, error_msg); + if(load_image_result == LoadImageResult::OK) + image.setTexture(image_texture, true); + else if(load_image_result == LoadImageResult::DOWNLOAD_IN_PROGRESS) + download_in_progress = true; + error_message.setString(error_msg); + } else if(image_index == num_images) { + error_message.setString("End of " + chapter_title); + } + Json::Value &json_chapters = content_storage_json["chapters"]; Json::Value json_chapter; int latest_read = image_index + 1; @@ -868,9 +980,12 @@ namespace QuickMedia { texture_size_f = sf::Vector2f(texture_size.x, texture_size.y); } + sf::Clock check_downloaded_timer; + const sf::Int32 check_downloaded_timeout_ms = 1000; + // TODO: Show to user if a certain page is missing (by checking page name (number) and checking if some are skipped) while (current_page == Page::IMAGES) { - if(window.waitEvent(event)) { + while(window.pollEvent(event)) { if (event.type == sf::Event::Closed) { current_page = Page::EXIT; } else if(event.type == sf::Event::Resized) { @@ -907,6 +1022,23 @@ namespace QuickMedia { } } + if(download_in_progress && check_downloaded_timer.getElapsedTime().asMilliseconds() >= check_downloaded_timeout_ms) { + sf::String error_msg; + LoadImageResult load_image_result = load_image_by_index(image_index, image_texture, error_msg); + if(load_image_result == LoadImageResult::OK) { + image.setTexture(image_texture, true); + download_in_progress = false; + error = false; + texture_size = image.getTexture()->getSize(); + texture_size_f = sf::Vector2f(texture_size.x, texture_size.y); + } else if(load_image_result == LoadImageResult::FAILED) { + download_in_progress = false; + } + error_message.setString(error_msg); + resized = true; + check_downloaded_timer.restart(); + } + const float font_height = chapter_text.getCharacterSize() + 8.0f; const float background_height = font_height + 6.0f; diff --git a/src/Storage.cpp b/src/Storage.cpp index f75f6be..1a199d9 100644 --- a/src/Storage.cpp +++ b/src/Storage.cpp @@ -6,6 +6,8 @@ #include <pwd.h> #include <unistd.h> #include <sys/stat.h> +#include <sys/types.h> +#include <fcntl.h> #endif static int makedir(const char *path) { @@ -49,12 +51,16 @@ namespace QuickMedia { return get_home_dir().join(".config").join("quickmedia"); } + Path get_cache_dir() { + return get_home_dir().join(".cache").join("quickmedia"); + } + int create_directory_recursive(const Path &path) { size_t index = 0; while(true) { index = path.data.find('/', index); - // Skips first '/' on unix-like systems + // Skips first '/', we don't want to try and create the root directory if(index == 0) { ++index; continue; @@ -100,10 +106,20 @@ namespace QuickMedia { int file_overwrite(const Path &path, const std::string &data) { FILE *file = fopen(path.data.c_str(), "wb"); if(!file) - return -errno; + return errno; - fwrite(data.data(), 1, data.size(), file); - fclose(file); - return 0; + if(fwrite(data.data(), 1, data.size(), file) != data.size()) { + fclose(file); + return -1; + } + + return fclose(file); + } + + int create_lock_file(const Path &path) { + int fd = open(path.data.c_str(), O_CREAT | O_EXCL); + if(fd == -1) + return errno; + return close(fd); } }
\ No newline at end of file diff --git a/src/plugins/Manganelo.cpp b/src/plugins/Manganelo.cpp index cd22cb0..7af35a6 100644 --- a/src/plugins/Manganelo.cpp +++ b/src/plugins/Manganelo.cpp @@ -93,29 +93,8 @@ namespace QuickMedia { return SuggestionResult::OK; } - ImageResult Manganelo::get_image_by_index(const std::string &url, int index, std::string &image_data) { - ImageResult image_result = get_image_urls_for_chapter(url); - if(image_result != ImageResult::OK) - return image_result; - - int num_images = last_chapter_image_urls.size(); - if(index < 0 || index >= num_images) - return ImageResult::END; - - // TODO: Cache image in file/memory - switch(download_to_string(last_chapter_image_urls[index], image_data)) { - case DownloadResult::OK: - return ImageResult::OK; - case DownloadResult::ERR: - return ImageResult::ERR; - case DownloadResult::NET_ERR: - return ImageResult::NET_ERR; - default: - return ImageResult::ERR; - } - } - ImageResult Manganelo::get_number_of_images(const std::string &url, int &num_images) { + std::lock_guard<std::mutex> lock(image_urls_mutex); num_images = 0; ImageResult image_result = get_image_urls_for_chapter(url); if(image_result != ImageResult::OK) @@ -147,7 +126,7 @@ namespace QuickMedia { if(src) { // TODO: If image loads too slow, try switching mirror std::string image_url = src; - string_replace_all(image_url, "s3.mkklcdnv3.com", "bu.mkklcdnbuv1.com"); + //string_replace_all(image_url, "s3.mkklcdnv3.com", "bu.mkklcdnbuv1.com"); urls->emplace_back(std::move(image_url)); } }, &last_chapter_image_urls); @@ -158,4 +137,22 @@ namespace QuickMedia { last_chapter_url = url; return result == 0 ? ImageResult::OK : ImageResult::ERR; } + + ImageResult Manganelo::for_each_page_in_chapter(const std::string &chapter_url, PageCallback callback) { + std::vector<std::string> image_urls; + { + std::lock_guard<std::mutex> lock(image_urls_mutex); + ImageResult image_result = get_image_urls_for_chapter(chapter_url); + if(image_result != ImageResult::OK) + return image_result; + + image_urls = last_chapter_image_urls; + } + + for(const std::string &url : image_urls) { + if(!callback(url)) + break; + } + return ImageResult::OK; + } }
\ No newline at end of file |