From 84a0496d635cad9c49ea76117f71aa7e3ac964ed Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sun, 28 Mar 2021 14:07:34 +0200 Subject: Use imagemagick to create thumbnails instead of doing it ourselves. Better result and less memory usage because out of process memory reclaimed on exit --- README.md | 1 + include/AsyncImageLoader.hpp | 2 + plugins/Matrix.hpp | 2 +- src/AsyncImageLoader.cpp | 187 ++++++++++++++++--------------------------- src/ImageUtils.cpp | 2 +- src/plugins/Matrix.cpp | 29 ++++++- 6 files changed, 101 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 771d1d4..b0b14ad 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ See project.conf \[dependencies]. ## Runtime ### Required `curl` is required for network requests.\ +`imagemagick` is required for thumbnails.\ `noto-fonts` and `noto-fonts-cjk` is required for latin and japanese characters. ### Optional `mpv` needs to be installed to play videos.\ diff --git a/include/AsyncImageLoader.hpp b/include/AsyncImageLoader.hpp index 5de9215..d485e3a 100644 --- a/include/AsyncImageLoader.hpp +++ b/include/AsyncImageLoader.hpp @@ -25,6 +25,8 @@ namespace QuickMedia { sf::Clock texture_applied_time; }; + bool create_thumbnail(const Path &thumbnail_path, const Path &thumbnail_path_resized, sf::Vector2i resize_target_size); + constexpr int NUM_IMAGE_LOAD_THREADS = 4; class AsyncImageLoader { diff --git a/plugins/Matrix.hpp b/plugins/Matrix.hpp index 26ad926..da55f81 100644 --- a/plugins/Matrix.hpp +++ b/plugins/Matrix.hpp @@ -524,7 +524,7 @@ namespace QuickMedia { void add_invites(const rapidjson::Value &invite_json); void remove_rooms(const rapidjson::Value &leave_json); std::shared_ptr parse_message_event(const rapidjson::Value &event_item_json, RoomData *room_data); - PluginResult upload_file(RoomData *room, const std::string &filepath, UploadInfo &file_info, UploadInfo &thumbnail_info, std::string &err_msg); + PluginResult upload_file(RoomData *room, const std::string &filepath, UploadInfo &file_info, UploadInfo &thumbnail_info, std::string &err_msg, bool upload_thumbnail = true); void add_room(std::unique_ptr room); void remove_room(const std::string &room_id); // Returns false if an invite to the room already exists diff --git a/src/AsyncImageLoader.cpp b/src/AsyncImageLoader.cpp index 9792781..c096e31 100644 --- a/src/AsyncImageLoader.cpp +++ b/src/AsyncImageLoader.cpp @@ -8,90 +8,34 @@ #include namespace QuickMedia { - // Linear interpolation - // TODO: Is this implementation ok? it always uses +1 offset for interpolation but if we were to resize an image with near 1x1 scaling - // then it would be slightly blurry - static void copy_resize(const sf::Image &source, sf::Image &destination, sf::Vector2u destination_size) { - const sf::Vector2u source_size = source.getSize(); - if(source_size.x == 0 || source_size.y == 0 || destination_size.x == 0 || destination_size.y == 0) - return; + bool create_thumbnail(const Path &thumbnail_path, const Path &thumbnail_path_resized, sf::Vector2i resize_target_size) { + // > is to only shrink image if smaller than the target size + std::string new_size = std::to_string(resize_target_size.x) + "x" + std::to_string(resize_target_size.y) + ">"; - //float width_ratio = (float)source_size.x / (float)destination_size.x; - //float height_ratio = (float)source_size.y / (float)destination_size.y; - - const sf::Uint8 *source_pixels = source.getPixelsPtr(); - // TODO: Remove this somehow. Right now we need to allocate this and also allocate the same array in the destination image - sf::Uint32 *destination_pixels = new sf::Uint32[destination_size.x * destination_size.y]; - sf::Uint32 *destination_pixel = destination_pixels; - for(unsigned int y = 0; y < destination_size.y; ++y) { - for(unsigned int x = 0; x < destination_size.x; ++x) { - int scaled_x_start = ((float)x / (float)destination_size.x) * source_size.x; - int scaled_y_start = ((float)y / (float)destination_size.y) * source_size.y; - int scaled_x_end = ((float)(x + 1) / (float)destination_size.x) * source_size.x; - int scaled_y_end = ((float)(y + 1) / (float)destination_size.y) * source_size.y; - if(scaled_x_end > (int)source_size.x - 1) scaled_x_end = source_size.x - 1; - if(scaled_y_end > (int)source_size.y - 1) scaled_y_end = source_size.y - 1; - //float scaled_x = x * width_ratio; - //float scaled_y = y * height_ratio; - - //sf::Uint32 *source_pixel = (sf::Uint32*)(source_pixels + (int)(scaled_x + scaled_y * source_size.x) * 4); - sf::Uint32 sum_red = 0; - sf::Uint32 sum_green = 0; - sf::Uint32 sum_blue = 0; - sf::Uint32 sum_alpha = 0; - sf::Uint32 num_colors = (scaled_x_end - scaled_x_start) * (scaled_y_end - scaled_y_start); - for(int yy = scaled_y_start; yy < scaled_y_end; ++yy) { - for(int xx = scaled_x_start; xx < scaled_x_end; ++xx) { - sf::Uint32 *source_pixel = (sf::Uint32*)(source_pixels + (xx + yy * source_size.x) * 4); - sum_red += (*source_pixel >> 24); - sum_green += ((*source_pixel >> 16) & 0xFF); - sum_blue += ((*source_pixel >> 8) & 0xFF); - sum_alpha += (*source_pixel & 0xFF); - } - } - sum_red /= num_colors; - sum_green /= num_colors; - sum_blue /= num_colors; - sum_alpha /= num_colors; - *destination_pixel = (sum_red << 24) | (sum_green << 16) | (sum_blue << 8) | sum_alpha; - ++destination_pixel; - } - } - destination.create(destination_size.x, destination_size.y, (sf::Uint8*)destination_pixels); - delete []destination_pixels; - } - - static bool save_image_as_thumbnail_atomic(const sf::Image &image, const Path &thumbnail_path, const char *ext) { - Path tmp_path = thumbnail_path; - tmp_path.append(".tmp"); - const char *thumbnail_path_ext = thumbnail_path.ext(); - if(is_image_ext(ext)) - tmp_path.append(ext); - else if(is_image_ext(thumbnail_path_ext)) - tmp_path.append(thumbnail_path_ext); - else - tmp_path.append(".png"); - return image.saveToFile(tmp_path.data) && (rename(tmp_path.data.c_str(), thumbnail_path.data.c_str()) == 0); - } + // We only want the first frame if its a gif + Path thumbnail_path_first_frame = thumbnail_path; + thumbnail_path_first_frame.append("[0]"); + + Path result_path_tmp = thumbnail_path_resized; + result_path_tmp.append(".tmp"); - // Returns empty string if no extension - static const char* get_ext(const std::string &path) { - size_t slash_index = path.rfind('/'); - size_t index = path.rfind('.'); - if(index != std::string::npos && (slash_index == std::string::npos || index > slash_index)) - return path.c_str() + index; - return ""; + const char *args[] = { "convert", thumbnail_path_first_frame.data.c_str(), "-thumbnail", new_size.c_str(), result_path_tmp.data.c_str(), nullptr}; + int convert_res = exec_program(args, nullptr, nullptr); + if(convert_res == 0 && rename(result_path_tmp.data.c_str(), thumbnail_path_resized.data.c_str()) == 0) + return true; + else + return false; } - static void create_thumbnail_if_thumbnail_smaller_than_image(const std::string &original_url, const Path &thumbnail_path, ThumbnailData *thumbnail_data, sf::Vector2i resize_target_size) { - sf::Vector2u new_image_size = clamp_to_size(thumbnail_data->image->getSize(), sf::Vector2u(resize_target_size.x, resize_target_size.y)); - if(new_image_size.x < thumbnail_data->image->getSize().x || new_image_size.y < thumbnail_data->image->getSize().y) { - auto destination_image = std::make_unique(); - copy_resize(*thumbnail_data->image, *destination_image, new_image_size); - thumbnail_data->image = std::move(destination_image); - save_image_as_thumbnail_atomic(*thumbnail_data->image, thumbnail_path, get_ext(original_url)); - thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + // Create thumbnail and load it. On failure load the original image + static void create_thumbnail(const Path &thumbnail_path, const Path &thumbnail_path_resized, ThumbnailData *thumbnail_data, sf::Vector2i resize_target_size) { + if(create_thumbnail(thumbnail_path, thumbnail_path_resized, resize_target_size)) { + load_image_from_file(*thumbnail_data->image, thumbnail_path_resized.data); + } else { + load_image_from_file(*thumbnail_data->image, thumbnail_path.data); + fprintf(stderr, "Failed to convert %s to a thumbnail, using the original image\n", thumbnail_path.data.c_str()); } + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; } AsyncImageLoader& AsyncImageLoader::get_instance() { @@ -106,7 +50,7 @@ namespace QuickMedia { loading_image[i] = false; } - load_image_thread = std::thread([this]{ + load_image_thread = std::thread([this]() mutable { std::optional thumbnail_load_data_opt; while(true) { thumbnail_load_data_opt = image_load_queue.pop_wait(); @@ -114,24 +58,31 @@ namespace QuickMedia { break; ThumbnailLoadData &thumbnail_load_data = thumbnail_load_data_opt.value(); - thumbnail_load_data.thumbnail_data->image = std::make_unique(); - if(load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_load_data.thumbnail_path.data)) { - fprintf(stderr, "Loaded %s from thumbnail cache\n", thumbnail_load_data.path.data.c_str()); + + Path thumbnail_path_resized = thumbnail_load_data.thumbnail_path; + if(thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0) + thumbnail_path_resized.append("_" + std::to_string(thumbnail_load_data.resize_target_size.x) + "x" + std::to_string(thumbnail_load_data.resize_target_size.y)); + + if(get_file_type(thumbnail_path_resized) == FileType::REGULAR) { + load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_path_resized.data); thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + fprintf(stderr, "Loaded %s from thumbnail cache\n", thumbnail_path_resized.data.c_str()); continue; } - if(thumbnail_load_data.local) { - if(load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_load_data.path.data) - && thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0) - { - create_thumbnail_if_thumbnail_smaller_than_image(thumbnail_load_data.path.data, thumbnail_load_data.thumbnail_path, thumbnail_load_data.thumbnail_data.get(), thumbnail_load_data.resize_target_size); - } - thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; - } else { - thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; - } + Path thumbnail_original_path; + if(thumbnail_load_data.local) + thumbnail_original_path = thumbnail_load_data.path; + else + thumbnail_original_path = thumbnail_load_data.thumbnail_path; + + if(thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0) + create_thumbnail(thumbnail_original_path, thumbnail_path_resized, thumbnail_load_data.thumbnail_data.get(), thumbnail_load_data.resize_target_size); + else + load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_original_path.data); + + thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; } }); } @@ -164,15 +115,13 @@ namespace QuickMedia { SHA256 sha256; sha256.add(url.data(), url.size()); Path thumbnail_path = get_cache_dir().join("thumbnails").join(sha256.getHash()); - if(resize_target_size.x != 0 && resize_target_size.y != 0) - thumbnail_path.append("_" + std::to_string(resize_target_size.x) + "x" + std::to_string(resize_target_size.y)); - if(get_file_type(thumbnail_path) == FileType::REGULAR) { + if(local && get_file_type(url) == FileType::REGULAR) { thumbnail_data->loading_state = LoadingState::LOADING; - image_load_queue.push({ url, thumbnail_path, local, thumbnail_data, resize_target_size }); + image_load_queue.push({ url, thumbnail_path, true, thumbnail_data, resize_target_size }); return; - } else if(local && get_file_type(url) == FileType::REGULAR) { + } else if(get_file_type(thumbnail_path) == FileType::REGULAR) { thumbnail_data->loading_state = LoadingState::LOADING; - image_load_queue.push({ url, thumbnail_path, true, thumbnail_data, resize_target_size }); + image_load_queue.push({ url, thumbnail_path, false, thumbnail_data, resize_target_size }); return; } @@ -188,30 +137,34 @@ namespace QuickMedia { // TODO: Keep the thread running and use conditional variable instead to sleep until a new image should be loaded. Same in ImageViewer. download_image_thread[free_index] = std::thread([this, free_index, thumbnail_path, url, local, resize_target_size, thumbnail_data, use_tor]() mutable { thumbnail_data->image = std::make_unique(); - if(load_image_from_file(*thumbnail_data->image, thumbnail_path.data)) { - fprintf(stderr, "Loaded %s from thumbnail cache\n", url.c_str()); + + Path thumbnail_path_resized = thumbnail_path; + if(resize_target_size.x != 0 && resize_target_size.y != 0) + thumbnail_path_resized.append("_" + std::to_string(resize_target_size.x) + "x" + std::to_string(resize_target_size.y)); + + if(get_file_type(thumbnail_path_resized) == FileType::REGULAR) { + load_image_from_file(*thumbnail_data->image, thumbnail_path_resized.data); + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + fprintf(stderr, "Loaded %s from thumbnail cache\n", thumbnail_path_resized.data.c_str()); + return; + } + + if(!local && get_file_type(thumbnail_path.data) != FileType::REGULAR && download_to_file(url, thumbnail_path.data, {}, use_tor, true) != DownloadResult::OK) { thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; loading_image[free_index] = false; return; - } else { - if(local) { - if(!load_image_from_file(*thumbnail_data->image, url)) { - thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; - loading_image[free_index] = false; - return; - } - } else { - // Use the same path as the thumbnail path, since it wont be overwritten if the image is smaller than the thumbnail target size - if(download_to_file(url, thumbnail_path.data, {}, use_tor, true) != DownloadResult::OK || !load_image_from_file(*thumbnail_data->image, thumbnail_path.data)) { - thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; - loading_image[free_index] = false; - return; - } - } } + Path thumbnail_original_path; + if(local) + thumbnail_original_path = url; + else + thumbnail_original_path = thumbnail_path; + if(resize_target_size.x != 0 && resize_target_size.y != 0) - create_thumbnail_if_thumbnail_smaller_than_image(url, thumbnail_path, thumbnail_data.get(), resize_target_size); + create_thumbnail(thumbnail_original_path, thumbnail_path_resized, thumbnail_data.get(), resize_target_size); + else + load_image_from_file(*thumbnail_data->image, thumbnail_original_path.data); thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; loading_image[free_index] = false; diff --git a/src/ImageUtils.cpp b/src/ImageUtils.cpp index a0b2c1e..25a6680 100644 --- a/src/ImageUtils.cpp +++ b/src/ImageUtils.cpp @@ -124,6 +124,6 @@ namespace QuickMedia { } bool is_image_ext(const char *ext) { - return strcasecmp(ext, ".jpg") == 0 || strcasecmp(ext, ".jpeg") == 0 || strcasecmp(ext, ".png") == 0 || strcasecmp(ext, ".gif") == 0; + return strcasecmp(ext, ".jpg") == 0 || strcasecmp(ext, ".jpeg") == 0 || strcasecmp(ext, ".png") == 0 || strcasecmp(ext, ".gif") == 0 || strcasecmp(ext, ".webp") == 0; } } \ No newline at end of file diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp index e0a9a4b..21459d0 100644 --- a/src/plugins/Matrix.cpp +++ b/src/plugins/Matrix.cpp @@ -6,6 +6,7 @@ #include "../../include/Program.hpp" #include "../../include/base64_url.hpp" #include "../../include/Json.hpp" +#include "../../include/AsyncImageLoader.hpp" #include "../../include/Utils.hpp" #include #include @@ -3133,7 +3134,7 @@ namespace QuickMedia { return post_message(room, filename, event_id_response, file_info_opt, thumbnail_info_opt); } - PluginResult Matrix::upload_file(RoomData *room, const std::string &filepath, UploadInfo &file_info, UploadInfo &thumbnail_info, std::string &err_msg) { + PluginResult Matrix::upload_file(RoomData *room, const std::string &filepath, UploadInfo &file_info, UploadInfo &thumbnail_info, std::string &err_msg, bool upload_thumbnail) { FileAnalyzer file_analyzer; if(!file_analyzer.load_file(filepath.c_str())) { err_msg = "Failed to load " + filepath; @@ -3163,8 +3164,7 @@ namespace QuickMedia { return PluginResult::ERR; } - if(is_content_type_video(file_analyzer.get_content_type())) { - // TODO: Also upload thumbnail for images. Take into consideration below upload_file, we dont want to upload thumbnail of thumbnail + if(upload_thumbnail && is_content_type_video(file_analyzer.get_content_type())) { char tmp_filename[] = "/tmp/quickmedia_video_frame_XXXXXX"; int tmp_file = mkstemp(tmp_filename); if(tmp_file != -1) { @@ -3184,6 +3184,29 @@ namespace QuickMedia { } else { fprintf(stderr, "Failed to create temporary file for video thumbnail, ignoring thumbnail...\n"); } + } else if(upload_thumbnail && is_content_type_image(file_analyzer.get_content_type())) { + char tmp_filename[] = "/tmp/quickmedia_thumbnail_XXXXXX"; + int tmp_file = mkstemp(tmp_filename); + if(tmp_file != -1) { + std::string thumbnail_path; + if(create_thumbnail(filepath, tmp_filename, sf::Vector2i(600, 337))) + thumbnail_path = tmp_filename; + else + thumbnail_path = filepath; + + UploadInfo upload_info_ignored; // Ignore because it wont be set anyways. Thumbnails dont have thumbnails. + PluginResult upload_thumbnail_result = upload_file(room, thumbnail_path, thumbnail_info, upload_info_ignored, err_msg, false); + if(upload_thumbnail_result != PluginResult::OK) { + close(tmp_file); + remove(tmp_filename); + return upload_thumbnail_result; + } + + close(tmp_file); + remove(tmp_filename); + } else { + fprintf(stderr, "Failed to create temporary file for image thumbnail, ignoring thumbnail...\n"); + } } std::vector additional_args = { -- cgit v1.2.3