Remove async image loader threads and instead check if the curl download process has finished

1 files changed, 102 insertions, 67 deletions

diff --git a/src/AsyncImageLoader.cpp b/src/AsyncImageLoader.cpp
index a871078..fce745a 100644
--- a/src/AsyncImageLoader.cpp
+++ b/src/AsyncImageLoader.cpp
@@ -11,6 +11,7 @@
 #include <sys/stat.h>
 #include <sys/wait.h>
 #include <sys/sendfile.h>
+#include <sys/time.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <malloc.h>
@@ -128,6 +129,7 @@ namespace QuickMedia {
 
         // parent
 
+        // TODO: Do not wait here and instead check if finished in |load_thread|
         int status = 0;
         if(waitpid(pid, &status, 0) == -1) {
             perror("waitpid failed");
@@ -176,44 +178,82 @@ namespace QuickMedia {
         return *instance;
     }
 
+    static void process_thumbnail(ThumbnailLoadData &thumbnail_load_data) {
+        thumbnail_load_data.thumbnail_data->image = std::make_unique<sf::Image>();
+
+        Path thumbnail_path_resized = thumbnail_load_data.thumbnail_path;
+        if(thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0)
+            thumbnail_path_resized.append("_" + std::to_string(thumbnail_load_data.resize_target_size.x) + "x" + std::to_string(thumbnail_load_data.resize_target_size.y));
+
+        if(get_file_type(thumbnail_path_resized) == FileType::REGULAR) {
+            load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_path_resized.data);
+            thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
+            fprintf(stderr, "Loaded %s from thumbnail cache\n", thumbnail_path_resized.data.c_str());
+            return;
+        }
+
+        Path thumbnail_original_path;
+        if(thumbnail_load_data.local)
+            thumbnail_original_path = thumbnail_load_data.path;
+        else
+            thumbnail_original_path = thumbnail_load_data.thumbnail_path;
+
+        if(thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0)
+            create_thumbnail(thumbnail_original_path, thumbnail_path_resized, thumbnail_load_data.thumbnail_data.get(), thumbnail_load_data.resize_target_size);
+        else
+            load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_original_path.data);
+
+        thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
+    }
+
+    static int64_t timeval_to_milliseconds(struct timeval &time) {
+        return (int64_t)time.tv_sec * 1000 + (int64_t)time.tv_usec / 1000;
+    }
+
     AsyncImageLoader::AsyncImageLoader() {
-        for(int i = 0; i < NUM_IMAGE_LOAD_THREADS; ++i) {
-            loading_image[i] = false;
+        for(int i = 0; i < NUM_IMAGE_LOAD_PARALLEL; ++i) {
+            downloads[i].read_program.pid = -1;
+            downloads[i].read_program.read_fd = -1;
         }
 
-        load_image_thread = AsyncTask<void>([this]() mutable {
+        load_thread = AsyncTask<void>([this]() mutable {
             std::optional<ThumbnailLoadData> thumbnail_load_data_opt;
-            while(true) {
-                thumbnail_load_data_opt = image_load_queue.pop_wait();
-                if(!thumbnail_load_data_opt)
-                    break;
-
-                ThumbnailLoadData &thumbnail_load_data = thumbnail_load_data_opt.value();
-                thumbnail_load_data.thumbnail_data->image = std::make_unique<sf::Image>();
-
-                Path thumbnail_path_resized = thumbnail_load_data.thumbnail_path;
-                if(thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0)
-                    thumbnail_path_resized.append("_" + std::to_string(thumbnail_load_data.resize_target_size.x) + "x" + std::to_string(thumbnail_load_data.resize_target_size.y));
-
-                if(get_file_type(thumbnail_path_resized) == FileType::REGULAR) {
-                    load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_path_resized.data);
-                    thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
-                    fprintf(stderr, "Loaded %s from thumbnail cache\n", thumbnail_path_resized.data.c_str());
-                    continue;
+            while(image_load_queue.is_running()) {
+                thumbnail_load_data_opt = image_load_queue.pop_if_available();
+                if(thumbnail_load_data_opt)
+                    process_thumbnail(thumbnail_load_data_opt.value());
+
+                for(int i = 0; i < NUM_IMAGE_LOAD_PARALLEL; ++i) {
+                    Download &download = downloads[i];
+                    if(download.read_program.pid == -1)
+                        continue;
+
+                    int status = 0;
+                    if(wait_program_non_blocking(download.read_program.pid, &status)) {
+                        Path tmp_thumbnail_path = download.thumbnail_path;
+                        tmp_thumbnail_path.append(".tmp");
+                        if(status == 0 && rename_atomic(tmp_thumbnail_path.data.c_str(), download.thumbnail_path.data.c_str()) == 0) {
+                            struct timeval time;
+                            gettimeofday(&time, nullptr);
+                            fprintf(stderr, "Download duration for %s: %ld ms\n", download.url.c_str(), timeval_to_milliseconds(time) - download.download_start);
+
+                            ThumbnailLoadData load_data = { std::move(download.url), std::move(download.thumbnail_path), false, download.thumbnail_data, download.resize_target_size };
+                            process_thumbnail(load_data);
+                        } else {
+                            fprintf(stderr, "Thumbnail download failed for %s\n", download.url.c_str());
+                        }
+
+                        std::lock_guard<std::mutex> lock(download_mutex);
+                        close(download.read_program.read_fd);
+                        download.read_program.pid = -1;
+                        download.read_program.read_fd = -1;
+                        download.thumbnail_path.data.clear();
+                        download.url.c_str();
+                        download.thumbnail_data = nullptr;
+                    }
                 }
 
-                Path thumbnail_original_path;
-                if(thumbnail_load_data.local)
-                    thumbnail_original_path = thumbnail_load_data.path;
-                else
-                    thumbnail_original_path = thumbnail_load_data.thumbnail_path;
-
-                if(thumbnail_load_data.resize_target_size.x != 0 && thumbnail_load_data.resize_target_size.y != 0)
-                    create_thumbnail(thumbnail_original_path, thumbnail_path_resized, thumbnail_load_data.thumbnail_data.get(), thumbnail_load_data.resize_target_size);
-                else
-                    load_image_from_file(*thumbnail_load_data.thumbnail_data->image, thumbnail_original_path.data);
-
-                thumbnail_load_data.thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
+                std::this_thread::sleep_for(std::chrono::milliseconds(2));
             }
         });
     }
@@ -222,6 +262,17 @@ namespace QuickMedia {
         image_load_queue.close();
     }
 
+    static bool download_file_async(const char *url, const char *save_filepath, ReadProgram *read_program) {
+        const char *args[] = {
+            "curl", "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", "-g", "-s", "-L", "-f",
+            "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
+            "-o", save_filepath,
+            "--", url,
+            nullptr
+        };
+        return exec_program_pipe(args, read_program) == 0;
+    }
+
     void AsyncImageLoader::load_thumbnail(const std::string &url, bool local, sf::Vector2i resize_target_size, std::shared_ptr<ThumbnailData> thumbnail_data) {
         if(thumbnail_data->loading_state != LoadingState::NOT_LOADED)
             return;
@@ -239,6 +290,7 @@ namespace QuickMedia {
             struct stat file_stat;
             memset(&file_stat, 0, sizeof(file_stat));
             if(stat(url.c_str(), &file_stat) != 0 || !S_ISREG(file_stat.st_mode)) {
+                fprintf(stderr, "Failed to load thumbnail %s: no such file\n", url.c_str());
                 thumbnail_data->image = std::make_unique<sf::Image>();
                 thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
                 return;
@@ -246,13 +298,13 @@ namespace QuickMedia {
 
             thumbnail_path.append("_" + std::to_string(file_stat.st_mtim.tv_sec));
             thumbnail_data->loading_state = LoadingState::LOADING;
-            image_load_queue.push({ url, thumbnail_path, true, thumbnail_data, resize_target_size });
+            image_load_queue.push({ url, std::move(thumbnail_path), true, thumbnail_data, resize_target_size });
             return;
         }
 
         if(get_file_type(thumbnail_path) == FileType::REGULAR) {
             thumbnail_data->loading_state = LoadingState::LOADING;
-            image_load_queue.push({ url, thumbnail_path, false, thumbnail_data, resize_target_size });
+            image_load_queue.push({ url, std::move(thumbnail_path), false, thumbnail_data, resize_target_size });
             return;
         }
 
@@ -260,39 +312,22 @@ namespace QuickMedia {
         if(free_index == -1)
             return;
 
-        loading_image[free_index] = true;
+        std::lock_guard<std::mutex> lock(download_mutex);
         thumbnail_data->loading_state = LoadingState::LOADING;
-
-        // TODO: Keep the thread running and use conditional variable instead to sleep until a new image should be loaded. Same in ImageViewer.
-        download_image_thread[free_index] = AsyncTask<void>([this, free_index, thumbnail_path, url, resize_target_size, thumbnail_data]() mutable {
-            thumbnail_data->image = std::make_unique<sf::Image>();
-
-            Path thumbnail_path_resized = thumbnail_path;
-            if(resize_target_size.x != 0 && resize_target_size.y != 0)
-                thumbnail_path_resized.append("_" + std::to_string(resize_target_size.x) + "x" + std::to_string(resize_target_size.y));
-
-            if(get_file_type(thumbnail_path_resized) == FileType::REGULAR) {
-                load_image_from_file(*thumbnail_data->image, thumbnail_path_resized.data);
-                thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
-                fprintf(stderr, "Loaded %s from thumbnail cache\n", thumbnail_path_resized.data.c_str());
-                return;
-            }
-
-            if(get_file_type(thumbnail_path.data) == FileType::FILE_NOT_FOUND && download_to_file(url, thumbnail_path.data, {}, true) != DownloadResult::OK) {
-                thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
-                loading_image[free_index] = false;
-                return;
-            }
-
-            if(resize_target_size.x != 0 && resize_target_size.y != 0)
-                create_thumbnail(thumbnail_path, thumbnail_path_resized, thumbnail_data.get(), resize_target_size);
-            else
-                load_image_from_file(*thumbnail_data->image, thumbnail_path.data);
-
-            thumbnail_data->loading_state = LoadingState::FINISHED_LOADING;
-            loading_image[free_index] = false;
+        Path tmp_thumbnail_path = thumbnail_path;
+        tmp_thumbnail_path.append(".tmp");
+        if(!download_file_async(url.c_str(), tmp_thumbnail_path.data.c_str(), &downloads[free_index].read_program)) {
+            fprintf(stderr, "Failed to start download of %s\n", url.c_str());
             return;
-        });
+        }
+
+        struct timeval time;
+        gettimeofday(&time, nullptr);
+        downloads[free_index].download_start = timeval_to_milliseconds(time);
+        downloads[free_index].thumbnail_path = std::move(thumbnail_path);
+        downloads[free_index].thumbnail_data = thumbnail_data;
+        downloads[free_index].resize_target_size = resize_target_size;
+        downloads[free_index].url = url;
     }
 
     std::shared_ptr<ThumbnailData> AsyncImageLoader::get_thumbnail(const std::string &url, bool local, sf::Vector2i resize_target_size) {
@@ -325,8 +360,8 @@ namespace QuickMedia {
     }
 
     int AsyncImageLoader::get_free_load_index() const {
-        for(int i = 0; i < NUM_IMAGE_LOAD_THREADS; ++i) {
-            if(!loading_image[i])
+        for(int i = 0; i < NUM_IMAGE_LOAD_PARALLEL; ++i) {
+            if(downloads[i].read_program.pid == -1)
                 return i;
         }
         return -1;