Use imagemagick to create thumbnails instead of doing it ourselves. Better result and less memory usage because out of process memory reclaimed on exit

6 files changed, 101 insertions, 122 deletions

diff --git a/README.md b/README.md
index 771d1d4..b0b14ad 100644
--- a/README.md
+++ b/README.md
@@ -83,6 +83,7 @@ See project.conf \[dependencies].
 ## Runtime
 ### Required
 `curl` is required for network requests.\
+`imagemagick` is required for thumbnails.\
 `noto-fonts` and `noto-fonts-cjk` is required for latin and japanese characters.
 ### Optional
 `mpv` needs to be installed to play videos.\
diff --git a/include/AsyncImageLoader.hpp b/include/AsyncImageLoader.hpp
index 5de9215..d485e3a 100644
--- a/include/AsyncImageLoader.hpp
+++ b/include/AsyncImageLoader.hpp
@@ -25,6 +25,8 @@ namespace QuickMedia {
         sf::Clock texture_applied_time;
     };
 
+    bool create_thumbnail(const Path &thumbnail_path, const Path &thumbnail_path_resized, sf::Vector2i resize_target_size);
+
     constexpr int NUM_IMAGE_LOAD_THREADS = 4;
 
     class AsyncImageLoader {
diff --git a/plugins/Matrix.hpp b/plugins/Matrix.hpp
index 26ad926..da55f81 100644
--- a/plugins/Matrix.hpp
+++ b/plugins/Matrix.hpp
@@ -524,7 +524,7 @@ namespace QuickMedia {
         void add_invites(const rapidjson::Value &invite_json);
         void remove_rooms(const rapidjson::Value &leave_json);
         std::shared_ptr<Message> parse_message_event(const rapidjson::Value &event_item_json, RoomData *room_data);
-        PluginResult upload_file(RoomData *room, const std::string &filepath, UploadInfo &file_info, UploadInfo &thumbnail_info, std::string &err_msg);
+        PluginResult upload_file(RoomData *room, const std::string &filepath, UploadInfo &file_info, UploadInfo &thumbnail_info, std::string &err_msg, bool upload_thumbnail = true);
         void add_room(std::unique_ptr<RoomData> room);
         void remove_room(const std::string &room_id);
         // Returns false if an invite to the room already exists
diff --git a/src/AsyncImageLoader.cpp b/src/AsyncImageLoader.cpp
index 9792781..c096e31 100644
--- a/src/AsyncImageLoader.cpp
+++ b/src/AsyncImageLoader.cpp
@@ -8,90 +8,34 @@
 #include <assert.h>
 
 namespace QuickMedia {
-    // Linear interpolation
-    // TODO: Is this implementation ok? it always uses +1 offset for interpolation but if we were to resize an image with near 1x1 scaling
-    // then it would be slightly blurry
-    static void copy_resize(const sf::Image &source, sf::Image &destination, sf::Vector2u destination_size) {
-        const sf::Vector2u source_size = source.getSize();
-        if(source_size.x == 0 || source_size.y == 0 || destination_size.x == 0 || destination_size.y == 0)
-            return;
+    bool create_thumbnail(const Path &thumbnail_path, const Path &thumbnail_path_resized, sf::Vector2i resize_target_size) {
+        // > is to only shrink image if smaller than the target size
+        std::string new_size = std::to_string(resize_target_size.x) + "x" + std::to_string(resize_target_size.y) + ">";
 
-        //float width_ratio = (float)source_size.x / (float)destination_size.x;
-        //float height_ratio = (float)source_size.y / (float)destination_size.y;
-
-        const sf::Uint8 *source_pixels = source.getPixelsPtr();
-        // TODO: Remove this somehow. Right now we need to allocate this and also allocate the same array in the destination image
-        sf::Uint32 *destination_pixels = new sf::Uint32[destination_size.x * destination_size.y];
-        sf::Uint32 *destination_pixel = destination_pixels;
-        for(unsigned int y = 0; y < destination_size.y; ++y) {
-            for(unsigned int x = 0; x < destination_size.x; ++x) {
-                int scaled_x_start = ((float)x / (float)destination_size.x) * source_size.x;
-                int scaled_y_start = ((float)y / (float)destination_size.y) * source_size.y;
-                int scaled_x_end = ((float)(x + 1) / (float)destination_size.x) * source_size.x;
-                int scaled_y_end = ((float)(y + 1) / (float)destination_size.y) * source_size.y;
-                if(scaled_x_end > (int)source_size.x - 1) scaled_x_end = source_size.x - 1;
-                if(scaled_y_end > (int)source_size.y - 1) scaled_y_end = source_size.y - 1;
-                //float scaled_x = x * width_ratio;
-                //float scaled_y = y * height_ratio;
-
-                //sf::Uint32 *source_pixel = (sf::Uint32*)(source_pixels + (int)(scaled_x + scaled_y * source_size.x) * 4);
-                sf::Uint32 sum_red = 0;
-                sf::Uint32 sum_green = 0;
-                sf::Uint32 sum_blue = 0;
-                sf::Uint32 sum_alpha = 0;
-                sf::Uint32 num_colors = (scaled_x_end - scaled_x_start) * (scaled_y_end - scaled_y_start);
-                for(int yy = scaled_y_start; yy < scaled_y_end; ++yy) {
-                    for(int xx = scaled_x_start; xx < scaled_x_end; ++xx) {
-                        sf::Uint32 *source_pixel = (sf::Uint32*)(source_pixels + (xx + yy * source_size.x) * 4);
-                        sum_red += (*source_pixel >> 24);
-                        sum_green += ((*source_pixel >> 16) & 0xFF);
-                        sum_blue += ((*source_pixel >> 8) & 0xFF);
-                        sum_alpha += (*source_pixel & 0xFF);
-                    }
-                }
-                sum_red /= num_colors;
-                sum_green /= num_colors;
-                sum_blue /= num_colors;
-                sum_alpha /= num_colors;
-                *destination_pixel = (sum_red << 24) | (sum_green << 16) | (sum_blue << 8) | sum_alpha;
-                ++destination_pixel;
-            }
-        }
-        destination.create(destination_size.x, destination_size.y, (sf::Uint8*)destination_pixels);
-        delete []destination_pixels;
-    }
-    
-    static bool save_image_as_thumbnail_atomic(const sf::Image &image, const Path &thumbnail_path, const char *ext) {
-        Path tmp_path = thumbnail_path;
-        tmp_path.append(".tmp");
-        const char *thumbnail_path_ext = thumbnail_path.ext();
-        if(is_image_ext(ext))
-            tmp_path.append(ext);
-        else if(is_image_ext(thumbnail_path_ext))
-            tmp_path.append(thumbnail_path_ext);
-        else
-            tmp_path.append(".png");
-        return image.saveToFile(tmp_path.data) && (rename(tmp_path.data.c_str(), thumbnail_path.data.c_str()) == 0);
-    }
+        // We only want the first frame if its a gif
+        Path thumbnail_path_first_frame = thumbnail_path;
+        thumbnail_path_first_frame.append("[0]");
+
+        Path result_path_tmp = thumbnail_path_resized;
+        result_path_tmp.append(".tmp");
 
-    // Returns empty string if no extension
-    static const char* get_ext(const std::string &path) {
-        size_t slash_index = path.rfind('/');
-        size_t index = path.rfind('.');
-        if(index != std::string::npos && (slash_index == std::string::npos || index > slash_index))
-            return path.c_str() + index;
-        return "";
+        const char *args[] = { "convert", thumbnail_path_first_frame.data.c_str(),  "-thumbnail", new_size.c_str(), result_path_tmp.data.c_str(), nullptr};
+        int convert_res = exec_program(args, nullptr, nullptr);
+        if(convert_res == 0 && rename(result_path_tmp.data.c_str(), thumbnail_path_resized.data.c_str()) == 0)
+            return true;
+        else
+            return false;
     }
 
-    static void create_thumbnail_if_thumbnail_smaller_than_image(const std::string &original_url, const Path &thumbnail_path, ThumbnailData *thumbnail_data, sf::Vector2i resize_target_size) {
-        sf::Vector2u new_image_size = clamp_to_size(thumbnail_data->image->getSize(), sf::Vector2u(resize_target_size.x, resize_target_size.y));
-        if(new_image_size.x < thumbnail_data->image->getSize().x || new_image_size.y < thumbnail_data->image->getSize().y) {
-            auto destination_image = std::make_unique<sf::Image>();
-            copy_resize(*thumbnail_data->image, *destination_image, new_image_size);
-            thumbnail_data->image = std::move(destination_image);
-            save_image_as_thumbnail_atomic(*thumbnail_data->image, thumbnail_path, get_ext(original_url));
-            thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
+    // Create thumbnail and load it. On failure load the original image
+    static void create_thumbnail(const Path &thumbnail_path, const Path &thumbnail_path_resized, ThumbnailData *thumbnail_data, sf::Vector2i resize_target_size) {
+        if(create_thumbnail(thumbnail_path, thumbnail_path_resized, resize_target_size)) {
+            load_image_from_file(*thumbnail_data->image, thumbnail_path_resized.data);
+        } else {
+            load_image_from_file(*thumbnail_data->image, thumbnail_path.data);
+            fprintf(stderr, "Failed to convert %s to a thumbnail, using the original image\n", thumbnail_path.data.c_str());
         }
+        thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
     }
 
     AsyncImageLoader& AsyncImageLoader::get_instance() {
@@ -106,7 +50,7 @@ namespace QuickMedia {
             loading_image[i] = false;
         }
 
-        load_image_thread = std::thread([this]{
+        load_image_thread = std::thread([this]() mutable {
             std::optional<ThumbnailLoadData> thumbnail_load_data_opt;
             while(true) {
                 thumbnail_load_data_opt = image_load_queue.pop_wait();
@@ -114,24 +58,31 @@ namespace QuickMedia {
                     break;
 
                 ThumbnailLoadData &thumbnail_load_data = thumbnail_load_data_opt.value();
-
                 thumbnail_load_data.thumbnail_data->image = std::make_unique<sf::Image>();
-                if(load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_load_data.thumbnail_path.data)) {
-                    fprintf(stderr, "Loaded %s from thumbnail cache\n", thumbnail_load_data.path.data.c_str());
+
+                Path thumbnail_path_resized = thumbnail_load_data.thumbnail_path;
+                if(thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0)
+                    thumbnail_path_resized.append("_" + std::to_string(thumbnail_load_data.resize_target_size.x) + "x" + std::to_string(thumbnail_load_data.resize_target_size.y));
+
+                if(get_file_type(thumbnail_path_resized) == FileType::REGULAR) {
+                    load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_path_resized.data);
                     thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
+                    fprintf(stderr, "Loaded %s from thumbnail cache\n", thumbnail_path_resized.data.c_str());
                     continue;
                 }
 
-                if(thumbnail_load_data.local) {
-                    if(load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_load_data.path.data)
-                        && thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0)
-                    {
-                        create_thumbnail_if_thumbnail_smaller_than_image(thumbnail_load_data.path.data, thumbnail_load_data.thumbnail_path, thumbnail_load_data.thumbnail_data.get(), thumbnail_load_data.resize_target_size);
-                    }
-                    thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
-                } else {
-                    thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
-                }
+                Path thumbnail_original_path;
+                if(thumbnail_load_data.local)
+                    thumbnail_original_path = thumbnail_load_data.path;
+                else
+                    thumbnail_original_path = thumbnail_load_data.thumbnail_path;
+
+                if(thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0)
+                    create_thumbnail(thumbnail_original_path, thumbnail_path_resized, thumbnail_load_data.thumbnail_data.get(), thumbnail_load_data.resize_target_size);
+                else
+                    load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_original_path.data);
+
+                thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
             }
         });
     }
@@ -164,15 +115,13 @@ namespace QuickMedia {
         SHA256 sha256;
         sha256.add(url.data(), url.size());
         Path thumbnail_path = get_cache_dir().join("thumbnails").join(sha256.getHash());
-        if(resize_target_size.x != 0 && resize_target_size.y != 0)
-            thumbnail_path.append("_" + std::to_string(resize_target_size.x) + "x" + std::to_string(resize_target_size.y));
-        if(get_file_type(thumbnail_path) == FileType::REGULAR) {
+        if(local && get_file_type(url) == FileType::REGULAR) {
             thumbnail_data->loading_state = LoadingState::LOADING;
-            image_load_queue.push({ url, thumbnail_path, local, thumbnail_data, resize_target_size });
+            image_load_queue.push({ url, thumbnail_path, true, thumbnail_data, resize_target_size });
             return;
-        } else if(local && get_file_type(url) == FileType::REGULAR) {
+        } else if(get_file_type(thumbnail_path) == FileType::REGULAR) {
             thumbnail_data->loading_state = LoadingState::LOADING;
-            image_load_queue.push({ url, thumbnail_path, true, thumbnail_data, resize_target_size });
+            image_load_queue.push({ url, thumbnail_path, false, thumbnail_data, resize_target_size });
             return;
         }
 
@@ -188,30 +137,34 @@ namespace QuickMedia {
         // TODO: Keep the thread running and use conditional variable instead to sleep until a new image should be loaded. Same in ImageViewer.
         download_image_thread[free_index] = std::thread([this, free_index, thumbnail_path, url, local, resize_target_size, thumbnail_data, use_tor]() mutable {
             thumbnail_data->image = std::make_unique<sf::Image>();
-            if(load_image_from_file(*thumbnail_data->image, thumbnail_path.data)) {
-                fprintf(stderr, "Loaded %s from thumbnail cache\n", url.c_str());
+
+            Path thumbnail_path_resized = thumbnail_path;
+            if(resize_target_size.x != 0 && resize_target_size.y != 0)
+                thumbnail_path_resized.append("_" + std::to_string(resize_target_size.x) + "x" + std::to_string(resize_target_size.y));
+
+            if(get_file_type(thumbnail_path_resized) == FileType::REGULAR) {
+                load_image_from_file(*thumbnail_data->image, thumbnail_path_resized.data);
+                thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
+                fprintf(stderr, "Loaded %s from thumbnail cache\n", thumbnail_path_resized.data.c_str());
+                return;
+            }
+
+            if(!local && get_file_type(thumbnail_path.data) != FileType::REGULAR && download_to_file(url, thumbnail_path.data, {}, use_tor, true) != DownloadResult::OK) {
                 thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
                 loading_image[free_index] = false;
                 return;
-            } else {
-                if(local) {
-                    if(!load_image_from_file(*thumbnail_data->image, url)) {
-                        thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
-                        loading_image[free_index] = false;
-                        return;
-                    }
-                } else {
-                    // Use the same path as the thumbnail path, since it wont be overwritten if the image is smaller than the thumbnail target size
-                    if(download_to_file(url, thumbnail_path.data, {}, use_tor, true) != DownloadResult::OK || !load_image_from_file(*thumbnail_data->image, thumbnail_path.data)) {
-                        thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
-                        loading_image[free_index] = false;
-                        return;
-                    }
-                }
             }
 
+            Path thumbnail_original_path;
+            if(local)
+                thumbnail_original_path = url;
+            else
+                thumbnail_original_path = thumbnail_path;
+
             if(resize_target_size.x != 0 && resize_target_size.y != 0)
-                create_thumbnail_if_thumbnail_smaller_than_image(url, thumbnail_path, thumbnail_data.get(), resize_target_size);
+                create_thumbnail(thumbnail_original_path, thumbnail_path_resized, thumbnail_data.get(), resize_target_size);
+            else
+                load_image_from_file(*thumbnail_data->image, thumbnail_original_path.data);
 
             thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
             loading_image[free_index] = false;
diff --git a/src/ImageUtils.cpp b/src/ImageUtils.cpp
index a0b2c1e..25a6680 100644
--- a/src/ImageUtils.cpp
+++ b/src/ImageUtils.cpp
@@ -124,6 +124,6 @@ namespace QuickMedia {
     }
 
     bool is_image_ext(const char *ext) {
-        return strcasecmp(ext, ".jpg") == 0 || strcasecmp(ext, ".jpeg") == 0 || strcasecmp(ext, ".png") == 0 || strcasecmp(ext, ".gif") == 0;
+        return strcasecmp(ext, ".jpg") == 0 || strcasecmp(ext, ".jpeg") == 0 || strcasecmp(ext, ".png") == 0 || strcasecmp(ext, ".gif") == 0 || strcasecmp(ext, ".webp") == 0;
     }
 }
\ No newline at end of file
diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp
index e0a9a4b..21459d0 100644
--- a/src/plugins/Matrix.cpp
+++ b/src/plugins/Matrix.cpp
@@ -6,6 +6,7 @@
 #include "../../include/Program.hpp"
 #include "../../include/base64_url.hpp"
 #include "../../include/Json.hpp"
+#include "../../include/AsyncImageLoader.hpp"
 #include "../../include/Utils.hpp"
 #include <rapidjson/document.h>
 #include <rapidjson/writer.h>
@@ -3133,7 +3134,7 @@ namespace QuickMedia {
         return post_message(room, filename, event_id_response, file_info_opt, thumbnail_info_opt);
     }
 
-    PluginResult Matrix::upload_file(RoomData *room, const std::string &filepath, UploadInfo &file_info, UploadInfo &thumbnail_info, std::string &err_msg) {
+    PluginResult Matrix::upload_file(RoomData *room, const std::string &filepath, UploadInfo &file_info, UploadInfo &thumbnail_info, std::string &err_msg, bool upload_thumbnail) {
         FileAnalyzer file_analyzer;
         if(!file_analyzer.load_file(filepath.c_str())) {
             err_msg = "Failed to load " + filepath;
@@ -3163,8 +3164,7 @@ namespace QuickMedia {
             return PluginResult::ERR;
         }
 
-        if(is_content_type_video(file_analyzer.get_content_type())) {
-            // TODO: Also upload thumbnail for images. Take into consideration below upload_file, we dont want to upload thumbnail of thumbnail
+        if(upload_thumbnail && is_content_type_video(file_analyzer.get_content_type())) {
             char tmp_filename[] = "/tmp/quickmedia_video_frame_XXXXXX";
             int tmp_file = mkstemp(tmp_filename);
             if(tmp_file != -1) {
@@ -3184,6 +3184,29 @@ namespace QuickMedia {
             } else {
                 fprintf(stderr, "Failed to create temporary file for video thumbnail, ignoring thumbnail...\n");
             }
+        } else if(upload_thumbnail && is_content_type_image(file_analyzer.get_content_type())) {
+            char tmp_filename[] = "/tmp/quickmedia_thumbnail_XXXXXX";
+            int tmp_file = mkstemp(tmp_filename);
+            if(tmp_file != -1) {
+                std::string thumbnail_path;
+                if(create_thumbnail(filepath, tmp_filename, sf::Vector2i(600, 337)))
+                    thumbnail_path = tmp_filename;
+                else
+                    thumbnail_path = filepath;
+
+                UploadInfo upload_info_ignored; // Ignore because it wont be set anyways. Thumbnails dont have thumbnails.
+                PluginResult upload_thumbnail_result = upload_file(room, thumbnail_path, thumbnail_info, upload_info_ignored, err_msg, false);
+                if(upload_thumbnail_result != PluginResult::OK) {
+                    close(tmp_file);
+                    remove(tmp_filename);
+                    return upload_thumbnail_result;
+                }
+
+                close(tmp_file);
+                remove(tmp_filename);
+            } else {
+                fprintf(stderr, "Failed to create temporary file for image thumbnail, ignoring thumbnail...\n");
+            }
         }
 
         std::vector<CommandArg> additional_args = {