From 1569d02aa38baa53d5442b3babdbf1a3aaa3aaa0 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Mon, 19 Oct 2020 14:44:55 +0200 Subject: Load thumbnails with multiple threads, use sha256 for saving image to path instead of base64 (filename limit is 256 on linux...) --- src/AsyncImageLoader.cpp | 140 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 105 insertions(+), 35 deletions(-) (limited to 'src/AsyncImageLoader.cpp') diff --git a/src/AsyncImageLoader.cpp b/src/AsyncImageLoader.cpp index 00ca7ee..2f0f869 100644 --- a/src/AsyncImageLoader.cpp +++ b/src/AsyncImageLoader.cpp @@ -1,9 +1,8 @@ #include "../include/AsyncImageLoader.hpp" -#include "../include/base64_url.hpp" -#include "../include/Storage.hpp" #include "../include/DownloadUtils.hpp" #include "../include/ImageUtils.hpp" #include "../include/Scale.hpp" +#include "../external/hash-library/sha256.h" #include namespace QuickMedia { @@ -90,70 +89,141 @@ namespace QuickMedia { return ""; } - // TODO: Run in 5 different threads, and add download_to_file(_cache) to reduce memory usage in each (use -O curl option) - bool AsyncImageLoader::load_thumbnail(const std::string &url, bool local, sf::Vector2i resize_target_size, bool use_tor, std::shared_ptr thumbnail_data) { - if(loading_image) - return false; + static void create_thumbnail_if_thumbnail_smaller_than_image(const std::string &original_url, const Path &thumbnail_path, ThumbnailData *thumbnail_data, sf::Vector2i resize_target_size) { + sf::Vector2u new_image_size = to_vec2u( + clamp_to_size( + to_vec2f(thumbnail_data->image->getSize()), to_vec2f(resize_target_size))); + if(new_image_size.x < thumbnail_data->image->getSize().x || new_image_size.y < thumbnail_data->image->getSize().y) { + auto destination_image = std::make_unique(); + copy_resize(*thumbnail_data->image, *destination_image, new_image_size); + thumbnail_data->image = std::move(destination_image); + save_image_as_thumbnail_atomic(*thumbnail_data->image, thumbnail_path, get_ext(original_url)); + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + } + } - loading_image = true; + AsyncImageLoader::AsyncImageLoader() { + for(size_t i = 0; i < NUM_IMAGE_LOAD_THREADS; ++i) { + loading_image[i] = false; + } - assert(thumbnail_data->loading_state == LoadingState::NOT_LOADED); - thumbnail_data->loading_state = LoadingState::LOADING; + load_image_thread = std::thread([this]{ + ThumbnailLoadData thumbnail_load_data; + while(true) { + { + std::unique_lock lock(load_image_mutex); + while(images_to_load.empty() && running) load_image_cv.wait(lock); + if(!running) + break; + thumbnail_load_data = images_to_load.front(); + images_to_load.pop_front(); + } + + thumbnail_load_data.thumbnail_data->image = std::make_unique(); + if(thumbnail_load_data.thumbnail_data->image->loadFromFile(thumbnail_load_data.thumbnail_path.data)) { + fprintf(stderr, "Loaded %s from thumbnail cache\n", thumbnail_load_data.path.data.c_str()); + thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + continue; + } + + if(thumbnail_load_data.local) { + if(thumbnail_load_data.thumbnail_data->image->loadFromFile(thumbnail_load_data.path.data) + && thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0) + { + create_thumbnail_if_thumbnail_smaller_than_image(thumbnail_load_data.path.data, thumbnail_load_data.thumbnail_path, thumbnail_load_data.thumbnail_data.get(), thumbnail_load_data.resize_target_size); + } + thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + } else { + thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + } + } + }); + } + + AsyncImageLoader::~AsyncImageLoader() { + running = false; + { + std::unique_lock lock(load_image_mutex); + load_image_cv.notify_one(); + } + load_image_thread.join(); + } + + void AsyncImageLoader::load_thumbnail(const std::string &url, bool local, sf::Vector2i resize_target_size, bool use_tor, std::shared_ptr thumbnail_data) { + if(thumbnail_data->loading_state != LoadingState::NOT_LOADED) + return; if(url.empty()) { thumbnail_data->image = std::make_unique(); thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; - loading_image = false; - return true; + return; + } + + SHA256 sha256; + sha256.add(url.data(), url.size()); + Path thumbnail_path = get_cache_dir().join("thumbnails").join(sha256.getHash()); + if(get_file_type(thumbnail_path) == FileType::REGULAR) { + thumbnail_data->loading_state = LoadingState::LOADING; + std::unique_lock lock(load_image_mutex); + images_to_load.push_back({ url, thumbnail_path, local, thumbnail_data, resize_target_size }); + load_image_cv.notify_one(); + return; + } else if(local && get_file_type(url) == FileType::REGULAR) { + thumbnail_data->loading_state = LoadingState::LOADING; + std::unique_lock lock(load_image_mutex); + images_to_load.push_back({ url, thumbnail_path, true, thumbnail_data, resize_target_size }); + load_image_cv.notify_one(); + return; } - // TODO: Keep the thread running and use conditional variable instead to sleep until a new image should be loaded. Same in ImageViewer. - load_image_thread = std::thread([this, url, local, resize_target_size, thumbnail_data, use_tor]() mutable { - // TODO: Use sha256 instead of base64_url encoding - Path thumbnail_path = get_cache_dir().join("thumbnails").join(base64_url::encode(url)); + int free_index = get_free_load_index(); + if(free_index == -1) + return; + + loading_image[free_index] = true; + thumbnail_data->loading_state = LoadingState::LOADING; + // TODO: Keep the thread running and use conditional variable instead to sleep until a new image should be loaded. Same in ImageViewer. + download_image_thread[free_index] = std::thread([this, free_index, thumbnail_path, url, local, resize_target_size, thumbnail_data, use_tor]() mutable { thumbnail_data->image = std::make_unique(); if(thumbnail_data->image->loadFromFile(thumbnail_path.data)) { fprintf(stderr, "Loaded %s from thumbnail cache\n", url.c_str()); thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; - loading_image = false; + loading_image[free_index] = false; return; } else { if(local) { if(!thumbnail_data->image->loadFromFile(url)) { thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; - loading_image = false; + loading_image[free_index] = false; return; } } else { - std::string texture_data; - if(download_to_string_cache(url, texture_data, {}, use_tor, true) != DownloadResult::OK || !thumbnail_data->image->loadFromMemory(texture_data.data(), texture_data.size())) { + Path download_path = thumbnail_path; + download_path.append(".orig"); + if(download_to_file(url, download_path.data, {}, use_tor, true) != DownloadResult::OK || !thumbnail_data->image->loadFromFile(download_path.data)) { thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; - loading_image = false; + loading_image[free_index] = false; return; } } } - if(resize_target_size.x != 0 && resize_target_size.y != 0) { - sf::Vector2u new_image_size = to_vec2u(clamp_to_size(to_vec2f(thumbnail_data->image->getSize()), to_vec2f(resize_target_size))); - if(new_image_size.x < thumbnail_data->image->getSize().x || new_image_size.y < thumbnail_data->image->getSize().y) { - auto destination_image = std::make_unique(); - copy_resize(*thumbnail_data->image, *destination_image, new_image_size); - thumbnail_data->image = std::move(destination_image); - save_image_as_thumbnail_atomic(*thumbnail_data->image, thumbnail_path, get_ext(url)); - thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; - loading_image = false; - return; - } - } + if(resize_target_size.x != 0 && resize_target_size.y != 0) + create_thumbnail_if_thumbnail_smaller_than_image(url, thumbnail_path, thumbnail_data.get(), resize_target_size); thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; - loading_image = false; + loading_image[free_index] = false; return; }); - load_image_thread.detach(); + download_image_thread[free_index].detach(); + } - return true; + int AsyncImageLoader::get_free_load_index() const { + for(int i = 0; i < NUM_IMAGE_LOAD_THREADS; ++i) { + if(!loading_image[i]) + return i; + } + return -1; } } \ No newline at end of file -- cgit v1.2.3