#include "../include/dchat/Cache.hpp" #include "../include/env.hpp" #include "../include/dchat/FileUtil.hpp" #include "../include/dchat/Gif.hpp" #include "../include/dchat/Storage.hpp" #include #include #include #include #include #include #include #if OS_FAMILY == OS_FAMILY_POSIX #define toNativeString(str) str #else #include #include #include #include static std::wstring toNativeString(const std::string &str) { std::wstring_convert> converter; return converter.from_bytes(str); } #endif using namespace std; using namespace TinyProcessLib; namespace dchat { const i64 CONTENT_NOT_VISIBLE_AGE_MS = 30000; // Delete content from cache after a specified amount of time if the content is not visible on the screen static bool downscaleImage(const boost::filesystem::path &filepath, void *data, const int size, const int newWidth, const int newHeight) { gdImagePtr imgPtr = gdImageCreateFromPngPtr(size, data); if(!imgPtr) return false; int width = gdImageSX(imgPtr); if(width < newWidth) { gdImageDestroy(imgPtr); return false; } int height = gdImageSX(imgPtr); if(height < newHeight) { gdImageDestroy(imgPtr); return false; } gdImageSetInterpolationMethod(imgPtr, GD_BILINEAR_FIXED); gdImagePtr newImgPtr = gdImageScale(imgPtr, newWidth, newHeight); if(!newImgPtr) { gdImageDestroy(imgPtr); return false; } bool success = (gdImageFile(newImgPtr, filepath.string().c_str()) == 0); gdImageDestroy(imgPtr); gdImageDestroy(newImgPtr); return success; } ContentByUrlResult Cache::loadImageFromFile(const boost::filesystem::path &filepath, bool loadFromCache) { StringView fileContent; try { fileContent = getFileContent(filepath); std::string webPageTitle; std::string webPageDescription; bool foundHtmlContent = false; preview_state state; preview_init(&state); size_t offset = 0; do { // TODO: Get file content before doing this, the file might be in utf-16 encoding. That can happen for example if file contains html. // Content type can be retrieved from HTTP response header when downloading content offset += preview_step(&state, fileContent.data + offset, fileContent.size - offset); if(state.step_result == PREVIEW_FOUND_IMAGE) { if(Gif::isDataGif(fileContent)) { Gif *gif = createGifFunc(fileContent); return { gif, ContentByUrlResult::Type::CACHED }; } else { /* if(!loadFromCache) { if(!downscaleImage(filepath, (void*)fileContent.data, fileContent.size, 100, 100)) { fprintf(stderr, "Failed to resize image: %s, using original file\n", filepath.c_str()); } } */ StaticImage *image = createStaticImageFunc(filepath); delete[] fileContent.data; fileContent.data = nullptr; return { image, ContentByUrlResult::Type::CACHED }; } } else if(state.step_result == PREVIEW_FOUND_TITLE) { foundHtmlContent = true; webPageTitle = std::string(state.meta_content, state.meta_content + state.meta_content_length); } else if(state.step_result == PREVIEW_FOUND_DESCRIPTION) { foundHtmlContent = true; webPageDescription = std::string(state.meta_content, state.meta_content + state.meta_content_length); } } while(offset < fileContent.size); delete[] fileContent.data; fileContent.data = nullptr; if(foundHtmlContent) { WebPagePreview *webPagePreview = createWebPagePreviewFunc(webPageTitle, webPageDescription); return { webPagePreview, ContentByUrlResult::Type::CACHED }; } } catch(std::exception &e) { fprintf(stderr, "Failed to load image %s, reason: %s\n", filepath.string().c_str(), e.what()); } return { (StaticImage*)nullptr, ContentByUrlResult::Type::FAILED_DOWNLOAD }; } Cache::Cache(CreateGifFunc _createGifFunc, CreateStaticImageFunc _createStaticImageFunc, CreateWebPagePreviewFunc _createWebPagePreviewFunc) : alive(true), createGifFunc(_createGifFunc), createStaticImageFunc(_createStaticImageFunc), createWebPagePreviewFunc(_createWebPagePreviewFunc) { assert(createGifFunc); assert(createStaticImageFunc); assert(createWebPagePreviewFunc); downloadWaitThread = thread([this] { while(alive) { for(vector::iterator it = imageDownloadProcesses.begin(); it != imageDownloadProcesses.end();) { int exitStatus; if(it->process->try_get_exit_status(exitStatus)) { boost::filesystem::path filepath = getImagesDir(); odhtdb::Hash urlHash(it->url.data(), it->url.size()); filepath /= urlHash.toString(); ContentByUrlResult contentByUrlResult; bool failed = exitStatus != 0; if(failed) { contentByUrlResult = { (StaticImage*)nullptr, ContentByUrlResult::Type::FAILED_DOWNLOAD }; } else { contentByUrlResult = loadImageFromFile(filepath, false); contentByUrlResult.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); if(contentByUrlResult.type == ContentByUrlResult::Type::CACHED) { printf("Download content from url: %s\n", it->url.c_str()); } } imageDownloadMutex.lock(); contentUrlCache[it->url] = contentByUrlResult; boost::filesystem::path downloadingFilepath = filepath; downloadingFilepath += ".downloading"; // Intentionally ignore failure, program should not crash if we fail to remove these files... boost::system::error_code err; boost::filesystem::remove(downloadingFilepath, err); imageDownloadMutex.unlock(); it = imageDownloadProcesses.erase(it); } else ++it; } while(alive && imageDownloadProcesses.empty() && imageDownloadProcessesQueue.empty()) this_thread::sleep_for(chrono::milliseconds(20)); if(!imageDownloadProcessesQueue.empty()) { imageDownloadMutex.lock(); for(auto imageDownloadInfo : imageDownloadProcessesQueue) { imageDownloadProcesses.push_back(imageDownloadInfo); } imageDownloadProcessesQueue.clear(); imageDownloadMutex.unlock(); } this_thread::sleep_for(chrono::milliseconds(20)); } }); checkContentAccessTimeThread = thread([this] { // TODO: Add this back #if 0 while(alive) { this_thread::sleep_for(chrono::milliseconds(500)); lock_guard lock(imageDownloadMutex); i64 now = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); for(unordered_map::iterator it = contentUrlCache.begin(); it != contentUrlCache.end();) { if(it->second.type == ContentByUrlResult::Type::CACHED && now - it->second.lastAccessed > CONTENT_NOT_VISIBLE_AGE_MS) { switch(it->second.cachedType) { case ContentByUrlResult::CachedType::TEXTURE_FILEPATH: { delete it->second.textureFilePath; break; } case ContentByUrlResult::CachedType::GIF: { delete it->second.gif; break; } case ContentByUrlResult::CachedType::WEB_PAGE_PREVIEW: { delete it->second.webPagePreview; break; } default: ++it; continue; } it = contentUrlCache.erase(it); } else ++it; } } #endif }); } Cache::~Cache() { alive = false; downloadWaitThread.join(); checkContentAccessTimeThread.join(); // TODO: Add this back #if 0 for(auto &it : contentUrlCache) { if(!it.second.textureFilePath) continue; switch(it.second.cachedType) { case ContentByUrlResult::CachedType::NONE: break; case ContentByUrlResult::CachedType::TEXTURE_FILEPATH: { delete it.second.textureFilePath; break; } case ContentByUrlResult::CachedType::GIF: { delete it.second.gif; break; } case ContentByUrlResult::CachedType::WEB_PAGE_PREVIEW: { delete it.second.webPagePreview; break; } default: // Did we forget to implement cleanup for a new cache content type? assert(false); break; } } #endif } static void createFileIgnoreError(const boost::filesystem::path &path) { try { fileReplace(path, StringView("", 0)); } catch(FileException &e) { fprintf(stderr, "Failed to create empty file: %s, reason: %s\n", path.string().c_str(), e.what()); } } const ContentByUrlResult Cache::getContentByUrl(const string &url, int downloadLimitBytes) { lock_guard lock(imageDownloadMutex); auto it = contentUrlCache.find(url); if(it != contentUrlCache.end()) { it->second.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); return it->second; } // TODO: Verify hashed url is not too long for filepath on windows boost::filesystem::path filepath = getImagesDir(); odhtdb::Hash urlHash(url.data(), url.size()); filepath /= urlHash.toString(); boost::filesystem::path downloadingFilepath = filepath; downloadingFilepath += ".downloading"; if(boost::filesystem::exists(downloadingFilepath)) { // Intentionally ignore failure, program should not crash if we fail to remove these files... boost::system::error_code err; boost::filesystem::remove(filepath, err); boost::filesystem::remove(downloadingFilepath, err); } // TODO: Do not load content in this thread. Return LOADING status and load it in another thread, because with a lot of images, chat can freeze ContentByUrlResult contentByUrlResult = loadImageFromFile(filepath, true); if(contentByUrlResult.type == ContentByUrlResult::Type::CACHED) { contentByUrlResult.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); contentUrlCache[url] = contentByUrlResult; printf("Loaded content from file cache: %s\n", url.c_str()); return contentByUrlResult; } else if(contentByUrlResult.type == ContentByUrlResult::Type::FAILED_DOWNLOAD && boost::filesystem::exists(filepath)) { contentUrlCache[url] = contentByUrlResult; return contentByUrlResult; } createFileIgnoreError(downloadingFilepath); ContentByUrlResult result((StaticImage*)nullptr, ContentByUrlResult::Type::DOWNLOADING); contentUrlCache[url] = result; string downloadLimitBytesStr = to_string(downloadLimitBytes); std::string cmdUtf8 = "curl -L --silent -o '"; cmdUtf8 += filepath.string(); cmdUtf8 += "' --max-filesize " + downloadLimitBytesStr + " --range 0-" + downloadLimitBytesStr + " --url '" + url + "'"; Process::string_type cmd = toNativeString(cmdUtf8); // TODO: Use this instead of curl on windows: certutil.exe -urlcache -split -f "https://url/to/file" path/and/name/to/save/as/file Process *process = new Process(cmd, toNativeString(""), nullptr, nullptr, false); ImageDownloadInfo imageDownloadInfo { process, url }; imageDownloadProcessesQueue.emplace_back(imageDownloadInfo); return result; } }