From 9866713ba916f9768edca02c61ed5ec580bd9557 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sun, 27 Sep 2020 01:08:34 +0200 Subject: Reduce scroll cpu usage from 10% to 1-2% by load image files in another thread but load the texture in the main thread --- TODO | 1 - include/Body.hpp | 20 ++++++--- include/ImageViewer.hpp | 6 ++- src/Body.cpp | 117 +++++++++++++++++++++++++++--------------------- src/ImageViewer.cpp | 40 ++++++++++------- src/QuickMedia.cpp | 47 ++++++++++++------- 6 files changed, 139 insertions(+), 92 deletions(-) diff --git a/TODO b/TODO index 2baecfc..41a5971 100644 --- a/TODO +++ b/TODO @@ -30,7 +30,6 @@ Use one special thread to load cached files. Right now if there are multiple ima Press pgup/pgdown to scroll and entire page. Press home/end to scroll to top/bottom. Scrolling past page causes the page to jump up and down very fast because the new thumbnail is loaded. Fix this somehow. In youtube this can be fixed by setting the thumbnail image fallback size to the same size and thumbnail images, but that doesn't work for matrix with different image sizes! -Find a way to decrease cpu usage when scrolling fast. Right now sf::Texture::load will use up to 10% cpu on my machine. Add setting to disable sending typing events to the server (matrix). Support emoji (mainly for matrix), by readding Text code from dchat. Also do the same but for inline images, text editing and url colors and clicking (also clicking on inline images). Also take code from dchat to support gifs (inline in text). \ No newline at end of file diff --git a/include/Body.hpp b/include/Body.hpp index 008aa70..9f7c2d8 100644 --- a/include/Body.hpp +++ b/include/Body.hpp @@ -10,6 +10,7 @@ #include "../external/RoundedRectangleShape.hpp" #include #include +#include namespace QuickMedia { class Program; @@ -82,6 +83,7 @@ namespace QuickMedia { class Body { public: Body(Program *program, sf::Font *font, sf::Font *bold_font, sf::Font *cjk_font); + ~Body(); // Select previous item, ignoring invisible items. Returns true if the item was changed. This can be used to check if the top was hit when wrap_around is set to false bool select_previous_item(); @@ -123,7 +125,6 @@ namespace QuickMedia { sf::Text progress_text; sf::Text replies_text; BodyItems items; - std::thread thumbnail_load_thread; bool draw_thumbnails; bool wrap_around; // Set to {0, 0} to disable resizing @@ -134,14 +135,22 @@ namespace QuickMedia { void draw_item(sf::RenderWindow &window, BodyItem *item, const sf::Vector2f &pos, const sf::Vector2f &size, const float item_height, const int item_index, const Json::Value &content_progress); float get_item_height(BodyItem *item); private: + enum class LoadingState { + NOT_LOADED, + LOADING, + FINISHED_LOADING, + APPLIED_TO_TEXTURE + }; + struct ThumbnailData { bool referenced = false; - std::shared_ptr texture; - bool loaded = false; + LoadingState loading_state = LoadingState::NOT_LOADED; + sf::Texture texture; + std::unique_ptr image; // Set in another thread, and then reset after loading it into |texture| }; Program *program; - std::shared_ptr load_thumbnail_from_url(const std::string &url, bool local, sf::Vector2i thumbnail_resize_target_size); - std::unordered_map item_thumbnail_textures; + void load_thumbnail_from_url(const std::string &url, bool local, sf::Vector2i thumbnail_resize_target_size, std::shared_ptr thumbnail_data); + std::unordered_map> item_thumbnail_textures; bool loading_thumbnail; int selected_item; int prev_selected_item; @@ -151,5 +160,6 @@ namespace QuickMedia { sf::RectangleShape item_background_shadow; sf::RoundedRectangleShape item_background; sf::Sprite image; + std::future load_thumbnail_future; }; } \ No newline at end of file diff --git a/include/ImageViewer.hpp b/include/ImageViewer.hpp index 93e7d7c..0ca64e9 100644 --- a/include/ImageViewer.hpp +++ b/include/ImageViewer.hpp @@ -18,13 +18,15 @@ namespace QuickMedia { WAITING, LOADING, FAILED_TO_LOAD, - LOADED + LOADED, + APPLIED_TO_TEXTURE }; struct ImageData { sf::Texture texture; sf::Sprite sprite; ImageStatus image_status; + std::unique_ptr image_data_str; bool visible_on_screen; }; @@ -47,7 +49,7 @@ namespace QuickMedia { int get_focused_page() const; int get_num_pages() const { return num_pages; } private: - void load_image_async(const Path &path, std::shared_ptr image_data, int page); + void load_image_async(const Path &path, std::shared_ptr image_data); bool render_page(sf::RenderWindow &window, int page, double offset_y); sf::Vector2 get_page_size(int page); private: diff --git a/src/Body.cpp b/src/Body.cpp index 25a7bc3..477591e 100644 --- a/src/Body.cpp +++ b/src/Body.cpp @@ -86,6 +86,11 @@ namespace QuickMedia { item_background.setFillColor(sf::Color(55, 60, 68)); } + Body::~Body() { + if(load_thumbnail_future.valid()) + load_thumbnail_future.get(); + } + bool Body::select_previous_item() { if(items.empty()) return false; @@ -292,65 +297,49 @@ namespace QuickMedia { // TODO: Do not load thumbnails for images larger than 30mb. // TODO: Load the thumbnail embedded in the file instead. - std::shared_ptr Body::load_thumbnail_from_url(const std::string &url, bool local, sf::Vector2i thumbnail_resize_target_size) { - auto result = std::make_shared(); - result->setSmooth(true); + void Body::load_thumbnail_from_url(const std::string &url, bool local, sf::Vector2i thumbnail_resize_target_size, std::shared_ptr thumbnail_data) { assert(!loading_thumbnail); loading_thumbnail = true; - thumbnail_load_thread = std::thread([this, result, url, local, thumbnail_resize_target_size]() { + + load_thumbnail_future = std::async(std::launch::async, [this, url, local, thumbnail_resize_target_size, thumbnail_data]() { // TODO: Use sha256 instead of base64_url encoding Path thumbnail_path = get_cache_dir().join("thumbnails").join(base64_url::encode(url)); - std::string texture_data; - if(file_get_content(thumbnail_path, texture_data) == 0) { + thumbnail_data->image = std::make_unique(); + if(thumbnail_data->image->loadFromFile(thumbnail_path.data)) { fprintf(stderr, "Loaded %s from thumbnail cache\n", url.c_str()); - result->loadFromMemory(texture_data.data(), texture_data.size()); - loading_thumbnail = false; + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; return; } else { if(local) { - if(file_get_content(url, texture_data) != 0) { - loading_thumbnail = false; + if(!thumbnail_data->image->loadFromFile(url)) { + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; return; } } else { - if(download_to_string_cache(url, texture_data, {}, program->get_current_plugin()->use_tor, true) != DownloadResult::OK) { - loading_thumbnail = false; + std::string texture_data; + if(download_to_string_cache(url, texture_data, {}, program->get_current_plugin()->use_tor, true) != DownloadResult::OK || !thumbnail_data->image->loadFromMemory(texture_data.data(), texture_data.size())) { + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; return; } } } if(thumbnail_resize_target_size.x != 0 && thumbnail_resize_target_size.y != 0) { - auto image = std::make_unique(); - // TODO: Load from file instead? decreases ram usage and we save to file above anyways - if(image->loadFromMemory(texture_data.data(), texture_data.size())) { - texture_data.resize(0); - sf::Vector2u new_image_size = to_vec2u(clamp_to_size(to_vec2f(image->getSize()), to_vec2f(thumbnail_resize_target_size))); - if(new_image_size.x < image->getSize().x || new_image_size.y < image->getSize().y) { - sf::Image destination_image; - copy_resize(*image, destination_image, new_image_size); - if(save_image_as_thumbnail_atomic(destination_image, thumbnail_path, get_ext(url))) { - image.reset(); - result->loadFromImage(destination_image); - } else { - result->loadFromImage(*image); - } - loading_thumbnail = false; - return; - } else { - result->loadFromImage(*image); - loading_thumbnail = false; - return; - } + sf::Vector2u new_image_size = to_vec2u(clamp_to_size(to_vec2f(thumbnail_data->image->getSize()), to_vec2f(thumbnail_resize_target_size))); + if(new_image_size.x < thumbnail_data->image->getSize().x || new_image_size.y < thumbnail_data->image->getSize().y) { + auto destination_image = std::make_unique(); + copy_resize(*thumbnail_data->image, *destination_image, new_image_size); + thumbnail_data->image = std::move(destination_image); + save_image_as_thumbnail_atomic(*thumbnail_data->image, thumbnail_path, get_ext(url)); + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + return; } } - result->loadFromMemory(texture_data.data(), texture_data.size()); - loading_thumbnail = false; + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + return; }); - thumbnail_load_thread.detach(); - return result; } void Body::draw(sf::RenderWindow &window, sf::Vector2f pos, sf::Vector2f size) { @@ -376,10 +365,15 @@ namespace QuickMedia { image_fallback.setSize(thumbnail_fallback_size); item_background_shadow.setFillColor(line_seperator_color); + if(loading_thumbnail && load_thumbnail_future.valid() && load_thumbnail_future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) { + load_thumbnail_future.get(); + loading_thumbnail = false; + } + int num_items = items.size(); if(num_items == 0 || size.y <= 0.0f) { for(auto it = item_thumbnail_textures.begin(); it != item_thumbnail_textures.end();) { - if(!it->second.referenced) + if(!it->second->referenced) it = item_thumbnail_textures.erase(it); else ++it; @@ -388,7 +382,7 @@ namespace QuickMedia { } for(auto &thumbnail_it : item_thumbnail_textures) { - thumbnail_it.second.referenced = false; + thumbnail_it.second->referenced = false; } for(auto &body_item : items) { @@ -503,7 +497,7 @@ namespace QuickMedia { glDisable(GL_SCISSOR_TEST); for(auto it = item_thumbnail_textures.begin(); it != item_thumbnail_textures.end();) { - if(!it->second.referenced) + if(!it->second->referenced) it = item_thumbnail_textures.erase(it); else ++it; @@ -512,14 +506,27 @@ namespace QuickMedia { void Body::draw_item(sf::RenderWindow &window, BodyItem *item, const sf::Vector2f &pos, const sf::Vector2f &size, const float item_height, const int item_index, const Json::Value &content_progress) { // TODO: Instead of generating a new hash everytime to access textures, cache the hash of the thumbnail url - // Intentionally create the item with the key item->thumbnail_url if it doesn't exist - item_thumbnail_textures[item->thumbnail_url].referenced = true; - auto &item_thumbnail = item_thumbnail_textures[item->thumbnail_url]; + std::shared_ptr item_thumbnail; + auto item_thumbnail_it = item_thumbnail_textures.find(item->thumbnail_url); + if(item_thumbnail_it == item_thumbnail_textures.end()) { + item_thumbnail = std::make_shared(); + item_thumbnail_textures.insert(std::make_pair(item->thumbnail_url, item_thumbnail)); + } else { + item_thumbnail = item_thumbnail_it->second; + } + item_thumbnail->referenced = true; if(draw_thumbnails) { - if(!item->thumbnail_url.empty() && !loading_thumbnail && !item_thumbnail.loaded && !item_thumbnail.texture) { - item_thumbnail.loaded = true; - item_thumbnail.texture = load_thumbnail_from_url(item->thumbnail_url, item->thumbnail_is_local, thumbnail_resize_target_size); + if(!loading_thumbnail && !item->thumbnail_url.empty() && item_thumbnail->loading_state == LoadingState::NOT_LOADED) { + item_thumbnail->loading_state = LoadingState::LOADING; + load_thumbnail_from_url(item->thumbnail_url, item->thumbnail_is_local, thumbnail_resize_target_size, item_thumbnail); + } + + if(item_thumbnail->loading_state == LoadingState::FINISHED_LOADING) { + if(!item_thumbnail->texture.loadFromImage(*item_thumbnail->image)) + fprintf(stderr, "Warning: failed to load texture from image: %s\n", item->thumbnail_url.c_str()); + item_thumbnail->image.reset(); + item_thumbnail->loading_state = LoadingState::APPLIED_TO_TEXTURE; } } @@ -541,8 +548,8 @@ namespace QuickMedia { if(draw_thumbnails) { // TODO: Verify if this is safe. The thumbnail is being modified in another thread // and it might not be fully finished before the native handle is set? - if(item_thumbnail.loaded && item_thumbnail.texture && item_thumbnail.texture->getNativeHandle() != 0) { - image.setTexture(*item_thumbnail.texture, true); + if(item_thumbnail->loading_state == LoadingState::APPLIED_TO_TEXTURE && item_thumbnail->texture.getNativeHandle() != 0) { + image.setTexture(item_thumbnail->texture, true); auto image_size = image.getTexture()->getSize(); auto height_ratio = std::min(image_max_height, (float)image_size.y) / image_size.y; auto scale = image.getScale(); @@ -628,10 +635,18 @@ namespace QuickMedia { item_height += item->description_text->getHeight() - 2.0f; } if(draw_thumbnails && !item->thumbnail_url.empty()) { - auto &item_thumbnail = item_thumbnail_textures[item->thumbnail_url]; + std::shared_ptr item_thumbnail; + auto item_thumbnail_it = item_thumbnail_textures.find(item->thumbnail_url); + if(item_thumbnail_it == item_thumbnail_textures.end()) { + item_thumbnail = std::make_shared(); + item_thumbnail_textures.insert(std::make_pair(item->thumbnail_url, item_thumbnail)); + } else { + item_thumbnail = item_thumbnail_it->second; + } + float image_height = image_fallback.getSize().y; - if(item_thumbnail.texture && item_thumbnail.texture->getNativeHandle() != 0) { - auto image_size = item_thumbnail.texture->getSize(); + if(item_thumbnail->loading_state == LoadingState::APPLIED_TO_TEXTURE && item_thumbnail->texture.getNativeHandle() != 0) { + auto image_size = item_thumbnail->texture.getSize(); image_height = std::min(image_max_height, (float)image_size.y); } item_height = std::max(item_height, image_height); diff --git a/src/ImageViewer.cpp b/src/ImageViewer.cpp index 86e7138..97ef7c9 100644 --- a/src/ImageViewer.cpp +++ b/src/ImageViewer.cpp @@ -34,22 +34,18 @@ namespace QuickMedia { has_size_vertical_cursor = size_vertical_cursor.loadFromSystem(sf::Cursor::SizeVertical); } - void ImageViewer::load_image_async(const Path &path, std::shared_ptr image_data, int page) { + void ImageViewer::load_image_async(const Path &path, std::shared_ptr image_data) { image_data->image_status = ImageStatus::LOADING; image_data->texture.setSmooth(true); assert(!loading_image); loading_image = true; - image_loader_thread = std::thread([this, image_data, path, page]() { - std::string image_data_str; - if(file_get_content(path, image_data_str) == 0) { - if(image_data->texture.loadFromMemory(image_data_str.data(), image_data_str.size())) { - image_data->sprite.setTexture(image_data->texture, true); - page_size[page].size = get_page_size(page); - page_size[page].loaded = true; - image_data->image_status = ImageStatus::LOADED; - } else { - image_data->image_status = ImageStatus::FAILED_TO_LOAD; - } + image_loader_thread = std::thread([this, image_data, path]() { + auto image_data_str = std::make_unique(); + if(file_get_content(path, *image_data_str) == 0) { + image_data->image_data_str = std::move(image_data_str); + image_data->image_status = ImageStatus::LOADED; + } else { + image_data->image_status = ImageStatus::FAILED_TO_LOAD; } loading_image = false; }); @@ -60,8 +56,20 @@ namespace QuickMedia { if(page < 0 || page >= (int)image_data.size()) return false; - const sf::Vector2 image_size = get_page_size(page); std::shared_ptr &page_image_data = image_data[page]; + if(page_image_data && page_image_data->image_status == ImageStatus::LOADED) { + if(page_image_data->texture.loadFromMemory(page_image_data->image_data_str->data(), page_image_data->image_data_str->size())) { + page_image_data->sprite.setTexture(page_image_data->texture, true); + page_size[page].size = get_page_size(page); + page_size[page].loaded = true; + page_image_data->image_status = ImageStatus::APPLIED_TO_TEXTURE; + } else { + page_image_data->image_status = ImageStatus::FAILED_TO_LOAD; + page_image_data->image_data_str.reset(); + } + } + + const sf::Vector2 image_size = get_page_size(page); sf::Vector2 render_pos(std::floor(window_size.x * 0.5 - image_size.x * 0.5), - image_size.y * 0.5 + scroll + offset_y); if(render_pos.y + image_size.y <= 0.0 || render_pos.y >= window_size.y) { if(page_image_data) @@ -80,7 +88,7 @@ namespace QuickMedia { } if(page_image_data) { - if(page_image_data->image_status == ImageStatus::LOADED) { + if(page_image_data->image_status == ImageStatus::APPLIED_TO_TEXTURE) { page_image_data->sprite.setPosition(render_pos.x, render_pos.y); window.draw(page_image_data->sprite); } else { @@ -90,7 +98,7 @@ namespace QuickMedia { if(!loading_image) { Path image_path = chapter_cache_dir; image_path.join(page_str); - load_image_async(image_path, page_image_data, page); + load_image_async(image_path, page_image_data); } msg = "Loading image for page " + page_str; } else if(page_image_data->image_status == ImageStatus::LOADING) { @@ -320,7 +328,7 @@ namespace QuickMedia { for(auto &page_data : image_data) { if(page_data && !page_data->visible_on_screen) { fprintf(stderr, "ImageViewer: Unloaded page %d\n", 1 + i); - page_data = nullptr; + page_data.reset(); } ++i; } diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 6293e52..05b65ea 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -2695,14 +2695,14 @@ namespace QuickMedia { std::future captcha_request_future; std::future captcha_post_solution_future; std::future post_comment_future; - std::future load_image_future; + std::future load_image_future; + bool downloading_image = false; sf::Texture captcha_texture; sf::Sprite captcha_sprite; std::mutex captcha_image_mutex; auto attached_image_texture = std::make_unique(); sf::Sprite attached_image_sprite; - std::mutex attachment_load_mutex; GoogleCaptchaChallengeInfo challenge_info; sf::Text challenge_description_text("", *font, 24); @@ -2726,6 +2726,8 @@ namespace QuickMedia { // TODO: Show a white image with "Loading..." text while the captcha image is downloading + // TODO: Make google captcha images load texture in the main thread, otherwise high cpu usage. I guess its fine right now because of only 1 image? + // TODO: Make this work with other sites than 4chan auto request_google_captcha_image = [this, &captcha_texture, &captcha_image_mutex, &navigation_stage, &captcha_sprite, &challenge_description_text](GoogleCaptchaChallengeInfo &challenge_info) { std::string payload_image_data; @@ -2865,26 +2867,21 @@ namespace QuickMedia { content_url = std::move(prev_content_url); redraw = true; } else { + if(downloading_image && load_image_future.valid()) + load_image_future.get(); + downloading_image = true; navigation_stage = NavigationStage::VIEWING_ATTACHED_IMAGE; - load_image_future = std::async(std::launch::async, [this, &image_board, &attached_image_texture, &attached_image_sprite, &attachment_load_mutex]() -> bool { + load_image_future = std::async(std::launch::async, [this, &image_board]() { + std::string image_data; BodyItem *selected_item = body->get_selected(); if(!selected_item || selected_item->attached_content_url.empty()) { - return false; + return image_data; } - std::string image_data; - if(download_to_string(selected_item->attached_content_url, image_data, {}, image_board->use_tor) != DownloadResult::OK) { + if(download_to_string_cache(selected_item->attached_content_url, image_data, {}, image_board->use_tor) != DownloadResult::OK) { show_notification(image_board->name, "Failed to download image: " + selected_item->attached_content_url, Urgency::CRITICAL); - return false; - } - - std::lock_guard lock(attachment_load_mutex); - if(!attached_image_texture->loadFromMemory(image_data.data(), image_data.size())) { - show_notification(image_board->name, "Failed to load image downloaded from url: " + selected_item->attached_content_url, Urgency::CRITICAL); - return false; + image_data.clear(); } - attached_image_texture->setSmooth(true); - attached_image_sprite.setTexture(*attached_image_texture, true); - return true; + return image_data; }); } } @@ -3050,8 +3047,24 @@ namespace QuickMedia { } else if(navigation_stage == NavigationStage::POSTING_COMMENT) { // TODO: Show "Posting..." when posting comment } else if(navigation_stage == NavigationStage::VIEWING_ATTACHED_IMAGE) { + std::string image_data; + if(downloading_image && load_image_future.valid() && load_image_future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) { + downloading_image = false; + image_data = load_image_future.get(); + + if(attached_image_texture->loadFromMemory(image_data.data(), image_data.size())) { + attached_image_texture->setSmooth(true); + attached_image_sprite.setTexture(*attached_image_texture, true); + } else { + BodyItem *selected_item = body->get_selected(); + std::string selected_item_attached_url; + if(selected_item) + selected_item_attached_url = selected_item->attached_content_url; + show_notification(image_board->name, "Failed to load image downloaded from url: " + selected_item->attached_content_url, Urgency::CRITICAL); + } + } + // TODO: Show a white image with the text "Downloading..." while the image is downloading and loading - std::lock_guard lock(attachment_load_mutex); if(attached_image_texture->getNativeHandle() != 0) { auto content_size = window_size; sf::Vector2u texture_size = attached_image_texture->getSize(); -- cgit v1.2.3