From 4b97ef51562df8d606cb8fe21cf9b4464767febb Mon Sep 17 00:00:00 2001 From: dec05eba Date: Mon, 1 Jun 2020 21:36:58 +0200 Subject: Improve 4chan thread list performance by asynchronous update and caching --- plugins/Fourchan.hpp | 23 ++++++++++++- src/plugins/Fourchan.cpp | 89 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/plugins/Fourchan.hpp b/plugins/Fourchan.hpp index 6534cfd..7b76f83 100644 --- a/plugins/Fourchan.hpp +++ b/plugins/Fourchan.hpp @@ -1,11 +1,15 @@ #pragma once #include "ImageBoard.hpp" +#include +#include +#include namespace QuickMedia { class Fourchan : public ImageBoard { public: - Fourchan() : ImageBoard("4chan") {} + Fourchan(); + ~Fourchan() override; PluginResult get_front_page(BodyItems &result_items) override; SearchResult search(const std::string &url, BodyItems &result_items) override; SuggestionResult update_search_suggestions(const std::string &text, BodyItems &result_items) override; @@ -17,5 +21,22 @@ namespace QuickMedia { int get_search_delay() const override { return 150; } Page get_page_after_search() const override { return Page::IMAGE_BOARD_THREAD_LIST; } bool search_is_filter() override { return true; } + private: + PluginResult get_threads_internal(const std::string &url, BodyItems &result_items); + void set_board_url(const std::string &new_url); + std::string get_board_url(); + void set_board_thread_list(BodyItems body_items); + BodyItems get_board_thread_list(); + private: + std::string current_board_url; + std::thread thread_list_update_thread; + BodyItems cached_thread_list_items; + std::mutex board_url_mutex; + std::mutex board_list_mutex; + std::condition_variable thread_list_cached_cv; + std::mutex thread_list_cache_mutex; + std::condition_variable thread_list_update_cv; + bool thread_list_cached = false; + bool running = true; }; } \ No newline at end of file diff --git a/src/plugins/Fourchan.cpp b/src/plugins/Fourchan.cpp index 212e197..d8c80b0 100644 --- a/src/plugins/Fourchan.cpp +++ b/src/plugins/Fourchan.cpp @@ -11,6 +11,37 @@ static const std::string fourchan_url = "https://a.4cdn.org/"; static const std::string fourchan_image_url = "https://i.4cdn.org/"; namespace QuickMedia { + Fourchan::Fourchan() : ImageBoard("4chan") { + thread_list_update_thread = std::thread([this]() { + BodyItems new_thread_list_items; + while(running) { + new_thread_list_items.clear(); + auto start_time = std::chrono::steady_clock::now(); + + std::string board_url = get_board_url(); + if(!board_url.empty()) { + PluginResult plugin_result = get_threads_internal(board_url, new_thread_list_items); + if(plugin_result == PluginResult::OK) + set_board_thread_list(std::move(new_thread_list_items)); + } + + auto time_passed = std::chrono::steady_clock::now() - start_time; + if(time_passed < std::chrono::seconds(15)) { + auto time_to_sleep = std::chrono::seconds(15) - time_passed; + std::unique_lock lock(thread_list_cache_mutex); + thread_list_update_cv.wait_for(lock, time_to_sleep); + } + } + }); + } + + Fourchan::~Fourchan() { + running = false; + std::unique_lock lock(thread_list_cache_mutex); + thread_list_update_cv.notify_one(); + thread_list_update_thread.join(); + } + PluginResult Fourchan::get_front_page(BodyItems &result_items) { std::string server_response; if(download_to_string(fourchan_url + "boards.json", server_response, {}, use_tor) != DownloadResult::OK) @@ -156,7 +187,7 @@ namespace QuickMedia { tidyRelease(doc); } - PluginResult Fourchan::get_threads(const std::string &url, BodyItems &result_items) { + PluginResult Fourchan::get_threads_internal(const std::string &url, BodyItems &result_items) { std::string server_response; if(download_to_string(fourchan_url + url + "/catalog.json", server_response, {}, use_tor) != DownloadResult::OK) return PluginResult::NET_ERR; @@ -284,6 +315,62 @@ namespace QuickMedia { return PluginResult::OK; } + void Fourchan::set_board_url(const std::string &new_url) { + { + std::lock_guard lock(board_url_mutex); + if(current_board_url == new_url) + return; + current_board_url = new_url; + } + + std::lock_guard thread_list_lock(thread_list_cache_mutex); + thread_list_update_cv.notify_one(); + thread_list_cached = false; + } + + std::string Fourchan::get_board_url() { + std::lock_guard lock(board_url_mutex); + return current_board_url; + } + + void Fourchan::set_board_thread_list(BodyItems body_items) { + { + std::lock_guard lock(board_list_mutex); + cached_thread_list_items.clear(); + for(auto &body_item : body_items) { + cached_thread_list_items.push_back(std::make_unique(*body_item)); + } + } + + std::unique_lock thread_list_cache_lock(thread_list_cache_mutex); + if(!thread_list_cached) { + thread_list_cached = true; + thread_list_cached_cv.notify_one(); + } + } + + BodyItems Fourchan::get_board_thread_list() { + std::lock_guard lock(board_list_mutex); + BodyItems body_items; + for(auto &cached_body_item : cached_thread_list_items) { + body_items.push_back(std::make_unique(*cached_body_item)); + } + return body_items; + } + + PluginResult Fourchan::get_threads(const std::string &url, BodyItems &result_items) { + set_board_url(url); + + std::unique_lock lock(thread_list_cache_mutex); + if(!thread_list_cached) { + if(thread_list_cached_cv.wait_for(lock, std::chrono::seconds(10)) == std::cv_status::timeout) + return PluginResult::NET_ERR; + } + + result_items = get_board_thread_list(); + return PluginResult::OK; + } + PluginResult Fourchan::get_thread_comments(const std::string &list_url, const std::string &url, BodyItems &result_items) { std::string server_response; if(download_to_string(fourchan_url + list_url + "/thread/" + url + ".json", server_response, {}, use_tor) != DownloadResult::OK) -- cgit v1.2.3