Reduce scroll cpu usage from 10% to 1-2% by load image files in another thread but load the texture in the main thread

6 files changed, 139 insertions, 92 deletions

diff --git a/TODO b/TODO
index 2baecfc..41a5971 100644
--- a/TODO
+++ b/TODO
@@ -30,7 +30,6 @@ Use one special thread to load cached files. Right now if there are multiple ima
 Press pgup/pgdown to scroll and entire page.
 Press home/end to scroll to top/bottom.
 Scrolling past page causes the page to jump up and down very fast because the new thumbnail is loaded. Fix this somehow. In youtube this can be fixed by setting the thumbnail image fallback size to the same size and thumbnail images, but that doesn't work for matrix with different image sizes!
-Find a way to decrease cpu usage when scrolling fast. Right now sf::Texture::load will use up to 10% cpu on my machine.
 Add setting to disable sending typing events to the server (matrix).
 Support emoji (mainly for matrix), by readding Text code from dchat. Also do the same but for inline images, text editing and url colors and clicking (also clicking on inline images).
 Also take code from dchat to support gifs (inline in text).
\ No newline at end of file
diff --git a/include/Body.hpp b/include/Body.hpp
index 008aa70..9f7c2d8 100644
--- a/include/Body.hpp
+++ b/include/Body.hpp
@@ -10,6 +10,7 @@
 #include "../external/RoundedRectangleShape.hpp"
 #include <json/value.h>
 #include <thread>
+#include <future>
 
 namespace QuickMedia {
     class Program;
@@ -82,6 +83,7 @@ namespace QuickMedia {
     class Body {
     public:
         Body(Program *program, sf::Font *font, sf::Font *bold_font, sf::Font *cjk_font);
+        ~Body();
 
         // Select previous item, ignoring invisible items. Returns true if the item was changed. This can be used to check if the top was hit when wrap_around is set to false
         bool select_previous_item();
@@ -123,7 +125,6 @@ namespace QuickMedia {
         sf::Text progress_text;
         sf::Text replies_text;
         BodyItems items;
-        std::thread thumbnail_load_thread;
         bool draw_thumbnails;
         bool wrap_around;
         // Set to {0, 0} to disable resizing
@@ -134,14 +135,22 @@ namespace QuickMedia {
         void draw_item(sf::RenderWindow &window, BodyItem *item, const sf::Vector2f &pos, const sf::Vector2f &size, const float item_height, const int item_index, const Json::Value &content_progress);
         float get_item_height(BodyItem *item);
     private:
+        enum class LoadingState {
+            NOT_LOADED,
+            LOADING,
+            FINISHED_LOADING,
+            APPLIED_TO_TEXTURE
+        };
+
         struct ThumbnailData {
             bool referenced = false;
-            std::shared_ptr<sf::Texture> texture;
-            bool loaded = false;
+            LoadingState loading_state = LoadingState::NOT_LOADED;
+            sf::Texture texture;
+            std::unique_ptr<sf::Image> image; // Set in another thread, and then reset after loading it into |texture|
         };
         Program *program;
-        std::shared_ptr<sf::Texture> load_thumbnail_from_url(const std::string &url, bool local, sf::Vector2i thumbnail_resize_target_size);
-        std::unordered_map<std::string, ThumbnailData> item_thumbnail_textures;
+        void load_thumbnail_from_url(const std::string &url, bool local, sf::Vector2i thumbnail_resize_target_size, std::shared_ptr<ThumbnailData> thumbnail_data);
+        std::unordered_map<std::string, std::shared_ptr<ThumbnailData>> item_thumbnail_textures;
         bool loading_thumbnail;
         int selected_item;
         int prev_selected_item;
@@ -151,5 +160,6 @@ namespace QuickMedia {
         sf::RectangleShape item_background_shadow;
         sf::RoundedRectangleShape item_background;
         sf::Sprite image;
+        std::future<void> load_thumbnail_future;
     };
 }
\ No newline at end of file
diff --git a/include/ImageViewer.hpp b/include/ImageViewer.hpp
index 93e7d7c..0ca64e9 100644
--- a/include/ImageViewer.hpp
+++ b/include/ImageViewer.hpp
@@ -18,13 +18,15 @@ namespace QuickMedia {
         WAITING,
         LOADING,
         FAILED_TO_LOAD,
-        LOADED
+        LOADED,
+        APPLIED_TO_TEXTURE
     };
 
     struct ImageData {
         sf::Texture texture;
         sf::Sprite sprite;
         ImageStatus image_status;
+        std::unique_ptr<std::string> image_data_str;
         bool visible_on_screen;
     };
 
@@ -47,7 +49,7 @@ namespace QuickMedia {
         int get_focused_page() const;
         int get_num_pages() const { return num_pages; }
     private:
-        void load_image_async(const Path &path, std::shared_ptr<ImageData> image_data, int page);
+        void load_image_async(const Path &path, std::shared_ptr<ImageData> image_data);
         bool render_page(sf::RenderWindow &window, int page, double offset_y);
         sf::Vector2<double> get_page_size(int page);
     private:
diff --git a/src/Body.cpp b/src/Body.cpp
index 25a7bc3..477591e 100644
--- a/src/Body.cpp
+++ b/src/Body.cpp
@@ -86,6 +86,11 @@ namespace QuickMedia {
         item_background.setFillColor(sf::Color(55, 60, 68));
     }
 
+    Body::~Body() {
+        if(load_thumbnail_future.valid())
+            load_thumbnail_future.get();
+    }
+
     bool Body::select_previous_item() {
         if(items.empty())
             return false;
@@ -292,65 +297,49 @@ namespace QuickMedia {
 
     // TODO: Do not load thumbnails for images larger than 30mb.
     // TODO: Load the thumbnail embedded in the file instead.
-    std::shared_ptr<sf::Texture> Body::load_thumbnail_from_url(const std::string &url, bool local, sf::Vector2i thumbnail_resize_target_size) {
-        auto result = std::make_shared<sf::Texture>();
-        result->setSmooth(true);
+    void Body::load_thumbnail_from_url(const std::string &url, bool local, sf::Vector2i thumbnail_resize_target_size, std::shared_ptr<ThumbnailData> thumbnail_data) {
         assert(!loading_thumbnail);
         loading_thumbnail = true;
-        thumbnail_load_thread = std::thread([this, result, url, local, thumbnail_resize_target_size]() {
+
+        load_thumbnail_future = std::async(std::launch::async, [this, url, local, thumbnail_resize_target_size, thumbnail_data]() {
             // TODO: Use sha256 instead of base64_url encoding
             Path thumbnail_path = get_cache_dir().join("thumbnails").join(base64_url::encode(url));
 
-            std::string texture_data;
-            if(file_get_content(thumbnail_path, texture_data) == 0) {
+            thumbnail_data->image = std::make_unique<sf::Image>();
+            if(thumbnail_data->image->loadFromFile(thumbnail_path.data)) {
                 fprintf(stderr, "Loaded %s from thumbnail cache\n", url.c_str());
-                result->loadFromMemory(texture_data.data(), texture_data.size());
-                loading_thumbnail = false;
+                thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
                 return;
             } else {
                 if(local) {
-                    if(file_get_content(url, texture_data) != 0) {
-                        loading_thumbnail = false;
+                    if(!thumbnail_data->image->loadFromFile(url)) {
+                        thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
                         return;
                     }
                 } else {
-                    if(download_to_string_cache(url, texture_data, {}, program->get_current_plugin()->use_tor, true) != DownloadResult::OK) {
-                        loading_thumbnail = false;
+                    std::string texture_data;
+                    if(download_to_string_cache(url, texture_data, {}, program->get_current_plugin()->use_tor, true) != DownloadResult::OK || !thumbnail_data->image->loadFromMemory(texture_data.data(), texture_data.size())) {
+                        thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
                         return;
                     }
                 }
             }
 
             if(thumbnail_resize_target_size.x != 0 && thumbnail_resize_target_size.y != 0) {
-                auto image = std::make_unique<sf::Image>();
-                // TODO: Load from file instead? decreases ram usage and we save to file above anyways
-                if(image->loadFromMemory(texture_data.data(), texture_data.size())) {
-                    texture_data.resize(0);
-                    sf::Vector2u new_image_size = to_vec2u(clamp_to_size(to_vec2f(image->getSize()), to_vec2f(thumbnail_resize_target_size)));
-                    if(new_image_size.x < image->getSize().x || new_image_size.y < image->getSize().y) {
-                        sf::Image destination_image;
-                        copy_resize(*image, destination_image, new_image_size);
-                        if(save_image_as_thumbnail_atomic(destination_image, thumbnail_path, get_ext(url))) {
-                            image.reset();
-                            result->loadFromImage(destination_image);
-                        } else {
-                            result->loadFromImage(*image);
-                        }
-                        loading_thumbnail = false;
-                        return;
-                    } else {
-                        result->loadFromImage(*image);
-                        loading_thumbnail = false;
-                        return;
-                    }
+                sf::Vector2u new_image_size = to_vec2u(clamp_to_size(to_vec2f(thumbnail_data->image->getSize()), to_vec2f(thumbnail_resize_target_size)));
+                if(new_image_size.x < thumbnail_data->image->getSize().x || new_image_size.y < thumbnail_data->image->getSize().y) {
+                    auto destination_image = std::make_unique<sf::Image>();
+                    copy_resize(*thumbnail_data->image, *destination_image, new_image_size);
+                    thumbnail_data->image = std::move(destination_image);
+                    save_image_as_thumbnail_atomic(*thumbnail_data->image, thumbnail_path, get_ext(url));
+                    thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
+                    return;
                 }
             }
 
-            result->loadFromMemory(texture_data.data(), texture_data.size());
-            loading_thumbnail = false;
+            thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
+            return;
         });
-        thumbnail_load_thread.detach();
-        return result;
     }
 
     void Body::draw(sf::RenderWindow &window, sf::Vector2f pos, sf::Vector2f size) {
@@ -376,10 +365,15 @@ namespace QuickMedia {
         image_fallback.setSize(thumbnail_fallback_size);
         item_background_shadow.setFillColor(line_seperator_color);
 
+        if(loading_thumbnail && load_thumbnail_future.valid() && load_thumbnail_future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) {
+            load_thumbnail_future.get();
+            loading_thumbnail = false;
+        }
+
         int num_items = items.size();
         if(num_items == 0 || size.y <= 0.0f) {
             for(auto it = item_thumbnail_textures.begin(); it != item_thumbnail_textures.end();) {
-                if(!it->second.referenced) 
+                if(!it->second->referenced)
                     it = item_thumbnail_textures.erase(it);
                 else
                     ++it;
@@ -388,7 +382,7 @@ namespace QuickMedia {
         }
 
         for(auto &thumbnail_it : item_thumbnail_textures) {
-            thumbnail_it.second.referenced = false;
+            thumbnail_it.second->referenced = false;
         }
 
         for(auto &body_item : items) {
@@ -503,7 +497,7 @@ namespace QuickMedia {
         glDisable(GL_SCISSOR_TEST);
 
         for(auto it = item_thumbnail_textures.begin(); it != item_thumbnail_textures.end();) {
-            if(!it->second.referenced) 
+            if(!it->second->referenced)
                 it = item_thumbnail_textures.erase(it);
             else
                 ++it;
@@ -512,14 +506,27 @@ namespace QuickMedia {
 
     void Body::draw_item(sf::RenderWindow &window, BodyItem *item, const sf::Vector2f &pos, const sf::Vector2f &size, const float item_height, const int item_index, const Json::Value &content_progress) {
         // TODO: Instead of generating a new hash everytime to access textures, cache the hash of the thumbnail url
-        // Intentionally create the item with the key item->thumbnail_url if it doesn't exist
-        item_thumbnail_textures[item->thumbnail_url].referenced = true;
-        auto &item_thumbnail = item_thumbnail_textures[item->thumbnail_url];
+        std::shared_ptr<ThumbnailData> item_thumbnail;
+        auto item_thumbnail_it = item_thumbnail_textures.find(item->thumbnail_url);
+        if(item_thumbnail_it == item_thumbnail_textures.end()) {
+            item_thumbnail = std::make_shared<ThumbnailData>();
+            item_thumbnail_textures.insert(std::make_pair(item->thumbnail_url, item_thumbnail));
+        } else {
+            item_thumbnail = item_thumbnail_it->second;
+        }
+        item_thumbnail->referenced = true;
 
         if(draw_thumbnails) {
-            if(!item->thumbnail_url.empty() && !loading_thumbnail && !item_thumbnail.loaded && !item_thumbnail.texture) {
-                item_thumbnail.loaded = true;
-                item_thumbnail.texture = load_thumbnail_from_url(item->thumbnail_url, item->thumbnail_is_local, thumbnail_resize_target_size);
+            if(!loading_thumbnail && !item->thumbnail_url.empty() && item_thumbnail->loading_state == LoadingState::NOT_LOADED) {
+                item_thumbnail->loading_state = LoadingState::LOADING;
+                load_thumbnail_from_url(item->thumbnail_url, item->thumbnail_is_local, thumbnail_resize_target_size, item_thumbnail);
+            }
+
+            if(item_thumbnail->loading_state == LoadingState::FINISHED_LOADING) {
+                if(!item_thumbnail->texture.loadFromImage(*item_thumbnail->image))
+                    fprintf(stderr, "Warning: failed to load texture from image: %s\n", item->thumbnail_url.c_str());
+                item_thumbnail->image.reset();
+                item_thumbnail->loading_state = LoadingState::APPLIED_TO_TEXTURE;
             }
         }
 
@@ -541,8 +548,8 @@ namespace QuickMedia {
         if(draw_thumbnails) {
             // TODO: Verify if this is safe. The thumbnail is being modified in another thread
             // and it might not be fully finished before the native handle is set?
-            if(item_thumbnail.loaded && item_thumbnail.texture && item_thumbnail.texture->getNativeHandle() != 0) {
-                image.setTexture(*item_thumbnail.texture, true);
+            if(item_thumbnail->loading_state == LoadingState::APPLIED_TO_TEXTURE && item_thumbnail->texture.getNativeHandle() != 0) {
+                image.setTexture(item_thumbnail->texture, true);
                 auto image_size = image.getTexture()->getSize();
                 auto height_ratio = std::min(image_max_height, (float)image_size.y) / image_size.y;
                 auto scale = image.getScale();
@@ -628,10 +635,18 @@ namespace QuickMedia {
             item_height += item->description_text->getHeight() - 2.0f;
         }
         if(draw_thumbnails && !item->thumbnail_url.empty()) {
-            auto &item_thumbnail = item_thumbnail_textures[item->thumbnail_url];
+            std::shared_ptr<ThumbnailData> item_thumbnail;
+            auto item_thumbnail_it = item_thumbnail_textures.find(item->thumbnail_url);
+            if(item_thumbnail_it == item_thumbnail_textures.end()) {
+                item_thumbnail = std::make_shared<ThumbnailData>();
+                item_thumbnail_textures.insert(std::make_pair(item->thumbnail_url, item_thumbnail));
+            } else {
+                item_thumbnail = item_thumbnail_it->second;
+            }
+
             float image_height = image_fallback.getSize().y;
-            if(item_thumbnail.texture && item_thumbnail.texture->getNativeHandle() != 0) {
-                auto image_size = item_thumbnail.texture->getSize();
+            if(item_thumbnail->loading_state == LoadingState::APPLIED_TO_TEXTURE && item_thumbnail->texture.getNativeHandle() != 0) {
+                auto image_size = item_thumbnail->texture.getSize();
                 image_height = std::min(image_max_height, (float)image_size.y);
             }
             item_height = std::max(item_height, image_height);
diff --git a/src/ImageViewer.cpp b/src/ImageViewer.cpp
index 86e7138..97ef7c9 100644
--- a/src/ImageViewer.cpp
+++ b/src/ImageViewer.cpp
@@ -34,22 +34,18 @@ namespace QuickMedia {
         has_size_vertical_cursor = size_vertical_cursor.loadFromSystem(sf::Cursor::SizeVertical);
     }
 
-    void ImageViewer::load_image_async(const Path &path, std::shared_ptr<ImageData> image_data, int page) {
+    void ImageViewer::load_image_async(const Path &path, std::shared_ptr<ImageData> image_data) {
         image_data->image_status = ImageStatus::LOADING;
         image_data->texture.setSmooth(true);
         assert(!loading_image);
         loading_image = true;
-        image_loader_thread = std::thread([this, image_data, path, page]() {
-            std::string image_data_str;
-            if(file_get_content(path, image_data_str) == 0) {
-                if(image_data->texture.loadFromMemory(image_data_str.data(), image_data_str.size())) {
-                    image_data->sprite.setTexture(image_data->texture, true);
-                    page_size[page].size = get_page_size(page);
-                    page_size[page].loaded = true;
-                    image_data->image_status = ImageStatus::LOADED;
-                } else {
-                    image_data->image_status = ImageStatus::FAILED_TO_LOAD;
-                }
+        image_loader_thread = std::thread([this, image_data, path]() {
+            auto image_data_str = std::make_unique<std::string>();
+            if(file_get_content(path, *image_data_str) == 0) {
+                image_data->image_data_str = std::move(image_data_str);
+                image_data->image_status = ImageStatus::LOADED;
+            } else {
+                image_data->image_status = ImageStatus::FAILED_TO_LOAD;
             }
             loading_image = false;
         });
@@ -60,8 +56,20 @@ namespace QuickMedia {
         if(page < 0 || page >= (int)image_data.size())
             return false;
 
-        const sf::Vector2<double> image_size = get_page_size(page);
         std::shared_ptr<ImageData> &page_image_data = image_data[page];
+        if(page_image_data && page_image_data->image_status == ImageStatus::LOADED) {
+            if(page_image_data->texture.loadFromMemory(page_image_data->image_data_str->data(), page_image_data->image_data_str->size())) {
+                page_image_data->sprite.setTexture(page_image_data->texture, true);
+                page_size[page].size = get_page_size(page);
+                page_size[page].loaded = true;
+                page_image_data->image_status = ImageStatus::APPLIED_TO_TEXTURE;
+            } else {
+                page_image_data->image_status = ImageStatus::FAILED_TO_LOAD;
+                page_image_data->image_data_str.reset();
+            }
+        }
+
+        const sf::Vector2<double> image_size = get_page_size(page);
         sf::Vector2<double> render_pos(std::floor(window_size.x * 0.5 - image_size.x * 0.5), - image_size.y * 0.5 + scroll + offset_y);
         if(render_pos.y + image_size.y <= 0.0 || render_pos.y >= window_size.y) {
             if(page_image_data)
@@ -80,7 +88,7 @@ namespace QuickMedia {
         }
 
         if(page_image_data) {
-            if(page_image_data->image_status == ImageStatus::LOADED) {
+            if(page_image_data->image_status == ImageStatus::APPLIED_TO_TEXTURE) {
                 page_image_data->sprite.setPosition(render_pos.x, render_pos.y);
                 window.draw(page_image_data->sprite);
             } else {
@@ -90,7 +98,7 @@ namespace QuickMedia {
                     if(!loading_image) {
                         Path image_path = chapter_cache_dir;
                         image_path.join(page_str);
-                        load_image_async(image_path, page_image_data, page);
+                        load_image_async(image_path, page_image_data);
                     }
                     msg = "Loading image for page " + page_str;
                 } else if(page_image_data->image_status == ImageStatus::LOADING) {
@@ -320,7 +328,7 @@ namespace QuickMedia {
         for(auto &page_data : image_data) {
             if(page_data && !page_data->visible_on_screen) {
                 fprintf(stderr, "ImageViewer: Unloaded page %d\n", 1 + i);
-                page_data = nullptr;
+                page_data.reset();
             }
             ++i;
         }
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 6293e52..05b65ea 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -2695,14 +2695,14 @@ namespace QuickMedia {
         std::future<bool> captcha_request_future;
         std::future<bool> captcha_post_solution_future;
         std::future<bool> post_comment_future;
-        std::future<bool> load_image_future;
+        std::future<std::string> load_image_future;
+        bool downloading_image = false;
         sf::Texture captcha_texture;
         sf::Sprite captcha_sprite;
         std::mutex captcha_image_mutex;
 
         auto attached_image_texture = std::make_unique<sf::Texture>();
         sf::Sprite attached_image_sprite;
-        std::mutex attachment_load_mutex;
 
         GoogleCaptchaChallengeInfo challenge_info;
         sf::Text challenge_description_text("", *font, 24);
@@ -2726,6 +2726,8 @@ namespace QuickMedia {
 
         // TODO: Show a white image with "Loading..." text while the captcha image is downloading
 
+        // TODO: Make google captcha images load texture in the main thread, otherwise high cpu usage. I guess its fine right now because of only 1 image?
+
         // TODO: Make this work with other sites than 4chan
         auto request_google_captcha_image = [this, &captcha_texture, &captcha_image_mutex, &navigation_stage, &captcha_sprite, &challenge_description_text](GoogleCaptchaChallengeInfo &challenge_info) {
             std::string payload_image_data;
@@ -2865,26 +2867,21 @@ namespace QuickMedia {
                                 content_url = std::move(prev_content_url);
                                 redraw = true;
                             } else {
+                                if(downloading_image && load_image_future.valid())
+                                    load_image_future.get();
+                                downloading_image = true;
                                 navigation_stage = NavigationStage::VIEWING_ATTACHED_IMAGE;
-                                load_image_future = std::async(std::launch::async, [this, &image_board, &attached_image_texture, &attached_image_sprite, &attachment_load_mutex]() -> bool {
+                                load_image_future = std::async(std::launch::async, [this, &image_board]() {
+                                    std::string image_data;
                                     BodyItem *selected_item = body->get_selected();
                                     if(!selected_item || selected_item->attached_content_url.empty()) {
-                                        return false;
+                                        return image_data;
                                     }
-                                    std::string image_data;
-                                    if(download_to_string(selected_item->attached_content_url, image_data, {}, image_board->use_tor) != DownloadResult::OK) {
+                                    if(download_to_string_cache(selected_item->attached_content_url, image_data, {}, image_board->use_tor) != DownloadResult::OK) {
                                         show_notification(image_board->name, "Failed to download image: " + selected_item->attached_content_url, Urgency::CRITICAL);
-                                        return false;
-                                    }
-
-                                    std::lock_guard<std::mutex> lock(attachment_load_mutex);
-                                    if(!attached_image_texture->loadFromMemory(image_data.data(), image_data.size())) {
-                                        show_notification(image_board->name, "Failed to load image downloaded from url: " + selected_item->attached_content_url, Urgency::CRITICAL);
-                                        return false;
+                                        image_data.clear();
                                     }
-                                    attached_image_texture->setSmooth(true);
-                                    attached_image_sprite.setTexture(*attached_image_texture, true);
-                                    return true;
+                                    return image_data;
                                 });
                             }
                         }
@@ -3050,8 +3047,24 @@ namespace QuickMedia {
             } else if(navigation_stage == NavigationStage::POSTING_COMMENT) {
                 // TODO: Show "Posting..." when posting comment
             } else if(navigation_stage == NavigationStage::VIEWING_ATTACHED_IMAGE) {
+                std::string image_data;
+                if(downloading_image && load_image_future.valid() && load_image_future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) {
+                    downloading_image = false;
+                    image_data = load_image_future.get();
+                    
+                    if(attached_image_texture->loadFromMemory(image_data.data(), image_data.size())) {
+                        attached_image_texture->setSmooth(true);
+                        attached_image_sprite.setTexture(*attached_image_texture, true);
+                    } else {
+                        BodyItem *selected_item = body->get_selected();
+                        std::string selected_item_attached_url;
+                        if(selected_item)
+                            selected_item_attached_url = selected_item->attached_content_url;
+                        show_notification(image_board->name, "Failed to load image downloaded from url: " + selected_item->attached_content_url, Urgency::CRITICAL);
+                    }
+                }
+                
                 // TODO: Show a white image with the text "Downloading..." while the image is downloading and loading
-                std::lock_guard<std::mutex> lock(attachment_load_mutex);
                 if(attached_image_texture->getNativeHandle() != 0) {
                     auto content_size = window_size;
                     sf::Vector2u texture_size = attached_image_texture->getSize();