From b1296f2c97c6fdc1c6a9922dc09c951b5cafdc12 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Mon, 29 Oct 2018 21:49:54 +0100 Subject: Initial commit --- src/Cache.cpp | 434 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 src/Cache.cpp (limited to 'src/Cache.cpp') diff --git a/src/Cache.cpp b/src/Cache.cpp new file mode 100644 index 0000000..0bde89d --- /dev/null +++ b/src/Cache.cpp @@ -0,0 +1,434 @@ +#include "../include/dchat/Cache.hpp" +#include "../include/env.hpp" +#include "../include/FileUtil.hpp" +#include "../include/dchat/Gif.hpp" +#include +#include +#include +#include +#include +#include +#include + +#if OS_FAMILY == OS_FAMILY_POSIX +#include +#else +#include +#endif + +using namespace std; +using namespace TinyProcessLib; + +namespace dchat +{ + const i64 CONTENT_NOT_VISIBLE_AGE_MS = 30000; // Delete content from cache after a specified amount of time if the content is not visible on the screen + + static boost::filesystem::path getHomeDir() + { + #if OS_FAMILY == OS_FAMILY_POSIX + const char *homeDir = getenv("HOME"); + if(!homeDir) + { + passwd *pw = getpwuid(getuid()); + homeDir = pw->pw_dir; + } + return boost::filesystem::path(homeDir); + #elif OS_FAMILY == OS_FAMILY_WINDOWS + BOOL ret; + HANDLE hToken; + std::wstring homeDir; + DWORD homeDirLen = MAX_PATH; + homeDir.resize(homeDirLen); + + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_READ, &hToken)) + return Result::Err("Failed to open process token"); + + if (!GetUserProfileDirectory(hToken, &homeDir[0], &homeDirLen)) + { + CloseHandle(hToken); + return Result::Err("Failed to get home directory"); + } + + CloseHandle(hToken); + homeDir.resize(wcslen(homeDir.c_str())); + return boost::filesystem::path(homeDir); + #endif + } + + boost::filesystem::path Cache::getDchatDir() + { + boost::filesystem::path dchatHomeDir = getHomeDir() / ".local" / "share" / "dchat"; + boost::filesystem::create_directories(dchatHomeDir); + return dchatHomeDir; + } + + boost::filesystem::path Cache::getImagesDir() + { + boost::filesystem::path imagesDir = getDchatDir() / "images"; + boost::filesystem::create_directories(imagesDir); + return imagesDir; + } + + void Cache::loadBindsFromFile(LoadBindsCallbackFunc callbackFunc) + { + assert(callbackFunc); + StringView fileContent; + try + { + fileContent = getFileContent(getDchatDir() / "binds"); + sibs::SafeDeserializer deserializer((const u8*)fileContent.data, fileContent.size); + + while(!deserializer.empty()) + { + u8 keySize = deserializer.extract(); + string key; + key.resize(keySize); + deserializer.extract((u8*)&key[0], keySize); + + u8 valueSize = deserializer.extract(); + string value; + value.resize(valueSize); + deserializer.extract((u8*)&value[0], valueSize); + + callbackFunc(key, value); + } + } + catch(FileException &e) + { + fprintf(stderr, "Failed to read binds from file, reason: %s\n", e.what()); + } + + delete[] fileContent.data; + } + + void Cache::replaceBindsInFile(const unordered_map &binds) + { + sibs::SafeSerializer serializer; + for(auto &it : binds) + { + serializer.add((u8)it.first.size()); + serializer.add((const u8*)it.first.data(), it.first.size()); + + serializer.add((u8)it.second.size()); + serializer.add((const u8*)it.second.data(), it.second.size()); + } + fileReplace(getDchatDir() / "binds", StringView((const char*)serializer.getBuffer().data(), serializer.getBuffer().size())); + } + + static bool downscaleImage(const boost::filesystem::path &filepath, void *data, const int size, const int newWidth, const int newHeight) + { + gdImagePtr imgPtr = gdImageCreateFromPngPtr(size, data); + if(!imgPtr) + return false; + + int width = gdImageSX(imgPtr); + if(width < newWidth) + { + gdImageDestroy(imgPtr); + return false; + } + + int height = gdImageSX(imgPtr); + if(height < newHeight) + { + gdImageDestroy(imgPtr); + return false; + } + + gdImageSetInterpolationMethod(imgPtr, GD_BILINEAR_FIXED); + gdImagePtr newImgPtr = gdImageScale(imgPtr, newWidth, newHeight); + if(!newImgPtr) + { + gdImageDestroy(imgPtr); + return false; + } + + bool success = (gdImageFile(newImgPtr, filepath.c_str()) == 0); + gdImageDestroy(imgPtr); + gdImageDestroy(newImgPtr); + return success; + } + + static ContentByUrlResult loadImageFromFile(const boost::filesystem::path &filepath, bool loadFromCache, CreateGifFunc createGifFunc) + { + StringView fileContent; + try + { + fileContent = getFileContent(filepath); + + std::string webPageTitle; + std::string webPageDescription; + bool foundHtmlContent = false; + preview_state state; + preview_init(&state); + size_t offset = 0; + do + { + // TODO: Get file content before doing this, the file might be in utf-16 encoding. That can happen for example if file contains html. + // Content type can be retrieved from HTTP response header when downloading content + offset += preview_step(&state, fileContent.data + offset, fileContent.size - offset); + if(state.step_result == PREVIEW_FOUND_IMAGE) + { + if(Gif::isDataGif(fileContent)) + { + Gif *gif = createGifFunc(fileContent); + return { gif, ContentByUrlResult::Type::CACHED }; + } + else + { + if(!loadFromCache) + { + if(!downscaleImage(filepath, (void*)fileContent.data, fileContent.size, 100, 100)) + { + fprintf(stderr, "Failed to resize image: %s, using original file\n", filepath.c_str()); + } + } + + delete[] fileContent.data; + fileContent.data = nullptr; + return { new boost::filesystem::path(filepath), ContentByUrlResult::Type::CACHED }; + } + break; + } + else if(state.step_result == PREVIEW_FOUND_TITLE) + { + foundHtmlContent = true; + webPageTitle = std::string(state.meta_content, state.meta_content + state.meta_content_length); + } + else if(state.step_result == PREVIEW_FOUND_DESCRIPTION) + { + foundHtmlContent = true; + webPageDescription = std::string(state.meta_content, state.meta_content + state.meta_content_length); + } + } while(offset < fileContent.size); + + delete[] fileContent.data; + fileContent.data = nullptr; + + if(foundHtmlContent) + { + WebPagePreview *webPagePreview = new WebPagePreview { webPageTitle, webPageDescription }; + return { webPagePreview, ContentByUrlResult::Type::CACHED }; + } + } + catch(std::exception &e) + { + fprintf(stderr, "Failed to load image %s, reason: %s\n", filepath.string().c_str(), e.what()); + } + return { (boost::filesystem::path*)nullptr, ContentByUrlResult::Type::FAILED_DOWNLOAD }; + } + + Cache::Cache(CreateGifFunc _createGifFunc) : + alive(true), + createGifFunc(_createGifFunc) + { + assert(createGifFunc); + downloadWaitThread = thread([this] + { + while(alive) + { + for(vector::iterator it = imageDownloadProcesses.begin(); it != imageDownloadProcesses.end();) + { + int exitStatus; + if(it->process->try_get_exit_status(exitStatus)) + { + boost::filesystem::path filepath = getImagesDir(); + odhtdb::Hash urlHash(it->url.data(), it->url.size()); + filepath /= urlHash.toString(); + + ContentByUrlResult contentByUrlResult; + bool failed = exitStatus != 0; + if(failed) + { + contentByUrlResult = { (boost::filesystem::path*)nullptr, ContentByUrlResult::Type::FAILED_DOWNLOAD }; + } + else + { + contentByUrlResult = loadImageFromFile(filepath, false, createGifFunc); + contentByUrlResult.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); + if(contentByUrlResult.type == ContentByUrlResult::Type::CACHED) + { + printf("Download content from url: %s\n", it->url.c_str()); + } + } + + imageDownloadMutex.lock(); + contentUrlCache[it->url] = contentByUrlResult; + boost::filesystem::path downloadingFilepath = filepath; + downloadingFilepath += ".downloading"; + // Intentionally ignore failure, program should not crash if we fail to remove these files... + boost::system::error_code err; + boost::filesystem::remove(downloadingFilepath, err); + imageDownloadMutex.unlock(); + it = imageDownloadProcesses.erase(it); + } + else + ++it; + } + + while(alive && imageDownloadProcesses.empty() && imageDownloadProcessesQueue.empty()) + this_thread::sleep_for(chrono::milliseconds(20)); + + if(!imageDownloadProcessesQueue.empty()) + { + imageDownloadMutex.lock(); + for(auto imageDownloadInfo : imageDownloadProcessesQueue) + { + imageDownloadProcesses.push_back(imageDownloadInfo); + } + imageDownloadProcessesQueue.clear(); + imageDownloadMutex.unlock(); + } + + this_thread::sleep_for(chrono::milliseconds(20)); + } + }); + + checkContentAccessTimeThread = thread([this] + { + while(alive) + { + this_thread::sleep_for(chrono::milliseconds(500)); + lock_guard lock(imageDownloadMutex); + + i64 now = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); + for(unordered_map::iterator it = contentUrlCache.begin(); it != contentUrlCache.end();) + { + if(it->second.type == ContentByUrlResult::Type::CACHED && now - it->second.lastAccessed > CONTENT_NOT_VISIBLE_AGE_MS) + { + switch(it->second.cachedType) + { + case ContentByUrlResult::CachedType::TEXTURE_FILEPATH: + { + delete it->second.textureFilePath; + break; + } + case ContentByUrlResult::CachedType::GIF: + { + delete it->second.gif; + break; + } + case ContentByUrlResult::CachedType::WEB_PAGE_PREVIEW: + { + delete it->second.webPagePreview; + break; + } + default: + ++it; + continue; + } + it = contentUrlCache.erase(it); + } + else + ++it; + } + } + }); + } + + Cache::~Cache() + { + alive = false; + downloadWaitThread.join(); + checkContentAccessTimeThread.join(); + + for(auto &it : contentUrlCache) + { + if(!it.second.textureFilePath) + continue; + + switch(it.second.cachedType) + { + case ContentByUrlResult::CachedType::NONE: + break; + case ContentByUrlResult::CachedType::TEXTURE_FILEPATH: + { + delete it.second.textureFilePath; + break; + } + case ContentByUrlResult::CachedType::GIF: + { + delete it.second.gif; + break; + } + case ContentByUrlResult::CachedType::WEB_PAGE_PREVIEW: + { + delete it.second.webPagePreview; + break; + } + default: + // Did we forget to implement cleanup for a new cache content type? + assert(false); + break; + } + } + } + + static void createFileIgnoreError(const boost::filesystem::path &path) + { + try + { + fileReplace(path, StringView("", 0)); + } + catch(FileException &e) + { + fprintf(stderr, "Failed to create empty file: %s, reason: %s\n", path.string().c_str(), e.what()); + } + } + + const ContentByUrlResult Cache::getContentByUrl(const string &url, int downloadLimitBytes) + { + lock_guard lock(imageDownloadMutex); + auto it = contentUrlCache.find(url); + if(it != contentUrlCache.end()) + { + it->second.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); + return it->second; + } + + // TODO: Verify hashed url is not too long for filepath on windows + boost::filesystem::path filepath = getImagesDir(); + odhtdb::Hash urlHash(url.data(), url.size()); + filepath /= urlHash.toString(); + + boost::filesystem::path downloadingFilepath = filepath; + downloadingFilepath += ".downloading"; + if(boost::filesystem::exists(downloadingFilepath)) + { + // Intentionally ignore failure, program should not crash if we fail to remove these files... + boost::system::error_code err; + boost::filesystem::remove(filepath, err); + boost::filesystem::remove(downloadingFilepath, err); + } + + // TODO: Do not load content in this thread. Return LOADING status and load it in another thread, because with a lot of images, chat can freeze + ContentByUrlResult contentByUrlResult = loadImageFromFile(filepath, true, createGifFunc); + if(contentByUrlResult.type == ContentByUrlResult::Type::CACHED) + { + contentByUrlResult.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); + contentUrlCache[url] = contentByUrlResult; + printf("Loaded content from file cache: %s\n", url.c_str()); + return contentByUrlResult; + } + else if(contentByUrlResult.type == ContentByUrlResult::Type::FAILED_DOWNLOAD && boost::filesystem::exists(filepath)) + { + contentUrlCache[url] = contentByUrlResult; + return contentByUrlResult; + } + + createFileIgnoreError(downloadingFilepath); + ContentByUrlResult result((boost::filesystem::path*)nullptr, ContentByUrlResult::Type::DOWNLOADING); + contentUrlCache[url] = result; + + string downloadLimitBytesStr = to_string(downloadLimitBytes); + + Process::string_type cmd = "curl -L --silent -o '"; + cmd += filepath.native(); + cmd += "' --max-filesize " + downloadLimitBytesStr + " --range 0-" + downloadLimitBytesStr + " --url '" + url + "'"; + // TODO: Use this instead of curl on windows: certutil.exe -urlcache -split -f "https://url/to/file" path/and/name/to/save/as/file + Process *process = new Process(cmd, "", nullptr, nullptr, false); + ImageDownloadInfo imageDownloadInfo { process, url }; + imageDownloadProcessesQueue.emplace_back(imageDownloadInfo); + return result; + } +} -- cgit v1.2.3