#include "../include/dchat/Cache.hpp" #include "../include/env.hpp" #include "../include/FileUtil.hpp" #include "../include/dchat/Gif.hpp" #include #include #include #include #include #include #include #if OS_FAMILY == OS_FAMILY_POSIX #include #else #include #endif using namespace std; using namespace TinyProcessLib; namespace dchat { const i64 CONTENT_NOT_VISIBLE_AGE_MS = 30000; // Delete content from cache after a specified amount of time if the content is not visible on the screen static boost::filesystem::path getHomeDir() { #if OS_FAMILY == OS_FAMILY_POSIX const char *homeDir = getenv("HOME"); if(!homeDir) { passwd *pw = getpwuid(getuid()); homeDir = pw->pw_dir; } return boost::filesystem::path(homeDir); #elif OS_FAMILY == OS_FAMILY_WINDOWS BOOL ret; HANDLE hToken; std::wstring homeDir; DWORD homeDirLen = MAX_PATH; homeDir.resize(homeDirLen); if (!OpenProcessToken(GetCurrentProcess(), TOKEN_READ, &hToken)) return Result::Err("Failed to open process token"); if (!GetUserProfileDirectory(hToken, &homeDir[0], &homeDirLen)) { CloseHandle(hToken); return Result::Err("Failed to get home directory"); } CloseHandle(hToken); homeDir.resize(wcslen(homeDir.c_str())); return boost::filesystem::path(homeDir); #endif } boost::filesystem::path Cache::getDchatDir() { boost::filesystem::path dchatHomeDir = getHomeDir() / ".local" / "share" / "dchat"; boost::filesystem::create_directories(dchatHomeDir); return dchatHomeDir; } boost::filesystem::path Cache::getImagesDir() { boost::filesystem::path imagesDir = getDchatDir() / "images"; boost::filesystem::create_directories(imagesDir); return imagesDir; } void Cache::loadBindsFromFile(LoadBindsCallbackFunc callbackFunc) { assert(callbackFunc); StringView fileContent; try { fileContent = getFileContent(getDchatDir() / "binds"); sibs::SafeDeserializer deserializer((const u8*)fileContent.data, fileContent.size); while(!deserializer.empty()) { u8 keySize = deserializer.extract(); string key; key.resize(keySize); deserializer.extract((u8*)&key[0], keySize); u8 valueSize = deserializer.extract(); string value; value.resize(valueSize); deserializer.extract((u8*)&value[0], valueSize); callbackFunc(key, value); } } catch(FileException &e) { fprintf(stderr, "Failed to read binds from file, reason: %s\n", e.what()); } delete[] fileContent.data; } void Cache::replaceBindsInFile(const unordered_map &binds) { sibs::SafeSerializer serializer; for(auto &it : binds) { serializer.add((u8)it.first.size()); serializer.add((const u8*)it.first.data(), it.first.size()); serializer.add((u8)it.second.size()); serializer.add((const u8*)it.second.data(), it.second.size()); } fileReplace(getDchatDir() / "binds", StringView((const char*)serializer.getBuffer().data(), serializer.getBuffer().size())); } static bool downscaleImage(const boost::filesystem::path &filepath, void *data, const int size, const int newWidth, const int newHeight) { gdImagePtr imgPtr = gdImageCreateFromPngPtr(size, data); if(!imgPtr) return false; int width = gdImageSX(imgPtr); if(width < newWidth) { gdImageDestroy(imgPtr); return false; } int height = gdImageSX(imgPtr); if(height < newHeight) { gdImageDestroy(imgPtr); return false; } gdImageSetInterpolationMethod(imgPtr, GD_BILINEAR_FIXED); gdImagePtr newImgPtr = gdImageScale(imgPtr, newWidth, newHeight); if(!newImgPtr) { gdImageDestroy(imgPtr); return false; } bool success = (gdImageFile(newImgPtr, filepath.c_str()) == 0); gdImageDestroy(imgPtr); gdImageDestroy(newImgPtr); return success; } static ContentByUrlResult loadImageFromFile(const boost::filesystem::path &filepath, bool loadFromCache, CreateGifFunc createGifFunc) { StringView fileContent; try { fileContent = getFileContent(filepath); std::string webPageTitle; std::string webPageDescription; bool foundHtmlContent = false; preview_state state; preview_init(&state); size_t offset = 0; do { // TODO: Get file content before doing this, the file might be in utf-16 encoding. That can happen for example if file contains html. // Content type can be retrieved from HTTP response header when downloading content offset += preview_step(&state, fileContent.data + offset, fileContent.size - offset); if(state.step_result == PREVIEW_FOUND_IMAGE) { if(Gif::isDataGif(fileContent)) { Gif *gif = createGifFunc(fileContent); return { gif, ContentByUrlResult::Type::CACHED }; } else { if(!loadFromCache) { if(!downscaleImage(filepath, (void*)fileContent.data, fileContent.size, 100, 100)) { fprintf(stderr, "Failed to resize image: %s, using original file\n", filepath.c_str()); } } delete[] fileContent.data; fileContent.data = nullptr; return { new boost::filesystem::path(filepath), ContentByUrlResult::Type::CACHED }; } break; } else if(state.step_result == PREVIEW_FOUND_TITLE) { foundHtmlContent = true; webPageTitle = std::string(state.meta_content, state.meta_content + state.meta_content_length); } else if(state.step_result == PREVIEW_FOUND_DESCRIPTION) { foundHtmlContent = true; webPageDescription = std::string(state.meta_content, state.meta_content + state.meta_content_length); } } while(offset < fileContent.size); delete[] fileContent.data; fileContent.data = nullptr; if(foundHtmlContent) { WebPagePreview *webPagePreview = new WebPagePreview { webPageTitle, webPageDescription }; return { webPagePreview, ContentByUrlResult::Type::CACHED }; } } catch(std::exception &e) { fprintf(stderr, "Failed to load image %s, reason: %s\n", filepath.string().c_str(), e.what()); } return { (boost::filesystem::path*)nullptr, ContentByUrlResult::Type::FAILED_DOWNLOAD }; } Cache::Cache(CreateGifFunc _createGifFunc) : alive(true), createGifFunc(_createGifFunc) { assert(createGifFunc); downloadWaitThread = thread([this] { while(alive) { for(vector::iterator it = imageDownloadProcesses.begin(); it != imageDownloadProcesses.end();) { int exitStatus; if(it->process->try_get_exit_status(exitStatus)) { boost::filesystem::path filepath = getImagesDir(); odhtdb::Hash urlHash(it->url.data(), it->url.size()); filepath /= urlHash.toString(); ContentByUrlResult contentByUrlResult; bool failed = exitStatus != 0; if(failed) { contentByUrlResult = { (boost::filesystem::path*)nullptr, ContentByUrlResult::Type::FAILED_DOWNLOAD }; } else { contentByUrlResult = loadImageFromFile(filepath, false, createGifFunc); contentByUrlResult.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); if(contentByUrlResult.type == ContentByUrlResult::Type::CACHED) { printf("Download content from url: %s\n", it->url.c_str()); } } imageDownloadMutex.lock(); contentUrlCache[it->url] = contentByUrlResult; boost::filesystem::path downloadingFilepath = filepath; downloadingFilepath += ".downloading"; // Intentionally ignore failure, program should not crash if we fail to remove these files... boost::system::error_code err; boost::filesystem::remove(downloadingFilepath, err); imageDownloadMutex.unlock(); it = imageDownloadProcesses.erase(it); } else ++it; } while(alive && imageDownloadProcesses.empty() && imageDownloadProcessesQueue.empty()) this_thread::sleep_for(chrono::milliseconds(20)); if(!imageDownloadProcessesQueue.empty()) { imageDownloadMutex.lock(); for(auto imageDownloadInfo : imageDownloadProcessesQueue) { imageDownloadProcesses.push_back(imageDownloadInfo); } imageDownloadProcessesQueue.clear(); imageDownloadMutex.unlock(); } this_thread::sleep_for(chrono::milliseconds(20)); } }); checkContentAccessTimeThread = thread([this] { while(alive) { this_thread::sleep_for(chrono::milliseconds(500)); lock_guard lock(imageDownloadMutex); i64 now = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); for(unordered_map::iterator it = contentUrlCache.begin(); it != contentUrlCache.end();) { if(it->second.type == ContentByUrlResult::Type::CACHED && now - it->second.lastAccessed > CONTENT_NOT_VISIBLE_AGE_MS) { switch(it->second.cachedType) { case ContentByUrlResult::CachedType::TEXTURE_FILEPATH: { delete it->second.textureFilePath; break; } case ContentByUrlResult::CachedType::GIF: { delete it->second.gif; break; } case ContentByUrlResult::CachedType::WEB_PAGE_PREVIEW: { delete it->second.webPagePreview; break; } default: ++it; continue; } it = contentUrlCache.erase(it); } else ++it; } } }); } Cache::~Cache() { alive = false; downloadWaitThread.join(); checkContentAccessTimeThread.join(); for(auto &it : contentUrlCache) { if(!it.second.textureFilePath) continue; switch(it.second.cachedType) { case ContentByUrlResult::CachedType::NONE: break; case ContentByUrlResult::CachedType::TEXTURE_FILEPATH: { delete it.second.textureFilePath; break; } case ContentByUrlResult::CachedType::GIF: { delete it.second.gif; break; } case ContentByUrlResult::CachedType::WEB_PAGE_PREVIEW: { delete it.second.webPagePreview; break; } default: // Did we forget to implement cleanup for a new cache content type? assert(false); break; } } } static void createFileIgnoreError(const boost::filesystem::path &path) { try { fileReplace(path, StringView("", 0)); } catch(FileException &e) { fprintf(stderr, "Failed to create empty file: %s, reason: %s\n", path.string().c_str(), e.what()); } } const ContentByUrlResult Cache::getContentByUrl(const string &url, int downloadLimitBytes) { lock_guard lock(imageDownloadMutex); auto it = contentUrlCache.find(url); if(it != contentUrlCache.end()) { it->second.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); return it->second; } // TODO: Verify hashed url is not too long for filepath on windows boost::filesystem::path filepath = getImagesDir(); odhtdb::Hash urlHash(url.data(), url.size()); filepath /= urlHash.toString(); boost::filesystem::path downloadingFilepath = filepath; downloadingFilepath += ".downloading"; if(boost::filesystem::exists(downloadingFilepath)) { // Intentionally ignore failure, program should not crash if we fail to remove these files... boost::system::error_code err; boost::filesystem::remove(filepath, err); boost::filesystem::remove(downloadingFilepath, err); } // TODO: Do not load content in this thread. Return LOADING status and load it in another thread, because with a lot of images, chat can freeze ContentByUrlResult contentByUrlResult = loadImageFromFile(filepath, true, createGifFunc); if(contentByUrlResult.type == ContentByUrlResult::Type::CACHED) { contentByUrlResult.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); contentUrlCache[url] = contentByUrlResult; printf("Loaded content from file cache: %s\n", url.c_str()); return contentByUrlResult; } else if(contentByUrlResult.type == ContentByUrlResult::Type::FAILED_DOWNLOAD && boost::filesystem::exists(filepath)) { contentUrlCache[url] = contentByUrlResult; return contentByUrlResult; } createFileIgnoreError(downloadingFilepath); ContentByUrlResult result((boost::filesystem::path*)nullptr, ContentByUrlResult::Type::DOWNLOADING); contentUrlCache[url] = result; string downloadLimitBytesStr = to_string(downloadLimitBytes); Process::string_type cmd = "curl -L --silent -o '"; cmd += filepath.native(); cmd += "' --max-filesize " + downloadLimitBytesStr + " --range 0-" + downloadLimitBytesStr + " --url '" + url + "'"; // TODO: Use this instead of curl on windows: certutil.exe -urlcache -split -f "https://url/to/file" path/and/name/to/save/as/file Process *process = new Process(cmd, "", nullptr, nullptr, false); ImageDownloadInfo imageDownloadInfo { process, url }; imageDownloadProcessesQueue.emplace_back(imageDownloadInfo); return result; } }