#include "../include/Cache.hpp" #include "../include/env.hpp" #include "../include/ResourceCache.hpp" #include "../include/FileUtil.hpp" #include "../include/Gif.hpp" #include "../include/Chatbar.hpp" #include "../include/WebPagePreview.hpp" #include "../include/ImagePreview.hpp" #include "../include/StringUtils.hpp" #include #include #include #include #include #include #include #include #if OS_FAMILY == OS_FAMILY_POSIX #include #else #include #endif using namespace std; using namespace TinyProcessLib; namespace dchat { static unordered_map contentUrlCache; const i64 CONTENT_NOT_VISIBLE_AGE_MS = 30000; // Delete content from cache after a specified amount of time if the content is not visible on the screen static boost::filesystem::path getHomeDir() { #if OS_FAMILY == OS_FAMILY_POSIX const char *homeDir = getenv("HOME"); if(!homeDir) { passwd *pw = getpwuid(getuid()); homeDir = pw->pw_dir; } return boost::filesystem::path(homeDir); #elif OS_FAMILY == OS_FAMILY_WINDOWS BOOL ret; HANDLE hToken; std::wstring homeDir; DWORD homeDirLen = MAX_PATH; homeDir.resize(homeDirLen); if (!OpenProcessToken(GetCurrentProcess(), TOKEN_READ, &hToken)) return Result::Err("Failed to open process token"); if (!GetUserProfileDirectory(hToken, &homeDir[0], &homeDirLen)) { CloseHandle(hToken); return Result::Err("Failed to get home directory"); } CloseHandle(hToken); homeDir.resize(wcslen(homeDir.c_str())); return boost::filesystem::path(homeDir); #endif } boost::filesystem::path Cache::getDchatDir() { boost::filesystem::path dchatHomeDir = getHomeDir() / ".local" / "share" / "dchat"; boost::filesystem::create_directories(dchatHomeDir); return dchatHomeDir; } boost::filesystem::path Cache::getImagesDir() { boost::filesystem::path imagesDir = getDchatDir() / "images"; boost::filesystem::create_directories(imagesDir); return imagesDir; } void Cache::loadBindsFromFile() { StringView fileContent; try { fileContent = getFileContent(getDchatDir() / "binds"); sibs::SafeDeserializer deserializer((const u8*)fileContent.data, fileContent.size); while(!deserializer.empty()) { u8 keySize = deserializer.extract(); string key; key.resize(keySize); deserializer.extract((u8*)&key[0], keySize); u8 valueSize = deserializer.extract(); string value; value.resize(valueSize); deserializer.extract((u8*)&value[0], valueSize); Chatbar::addBind(key, value, false); } } catch(FileException &e) { fprintf(stderr, "Failed to read binds from file, reason: %s\n", e.what()); } delete[] fileContent.data; } void Cache::replaceBindsInFile(const unordered_map &binds) { sibs::SafeSerializer serializer; for(auto &it : binds) { serializer.add((u8)it.first.size()); serializer.add((const u8*)it.first.data(), it.first.size()); serializer.add((u8)it.second.size()); serializer.add((const u8*)it.second.data(), it.second.size()); } fileReplace(getDchatDir() / "binds", StringView((const char*)serializer.getBuffer().data(), serializer.getBuffer().size())); } static bool downscaleImage(const boost::filesystem::path &filepath, void *data, const int size, const int newWidth, const int newHeight) { gdImagePtr imgPtr = gdImageCreateFromPngPtr(size, data); if(!imgPtr) return false; int width = gdImageSX(imgPtr); if(width < newWidth) { gdImageDestroy(imgPtr); return false; } int height = gdImageSX(imgPtr); if(height < newHeight) { gdImageDestroy(imgPtr); return false; } gdImageSetInterpolationMethod(imgPtr, GD_BILINEAR_FIXED); gdImagePtr newImgPtr = gdImageScale(imgPtr, newWidth, newHeight); if(!newImgPtr) { gdImageDestroy(imgPtr); return false; } bool success = (gdImageFile(newImgPtr, filepath.c_str()) == 0); gdImageDestroy(imgPtr); gdImageDestroy(newImgPtr); return success; } static ContentByUrlResult loadImageFromFile(const boost::filesystem::path &filepath, bool loadFromCache) { StringView fileContent; try { fileContent = getFileContent(filepath); sf::String webPageTitle; sf::String webPageDescription; bool foundHtmlContent = false; preview_state state; preview_init(&state); size_t offset = 0; do { // TODO: Get file content before doing this, the file might be in utf-16 encoding. That can happen for example if file contains html. // Content type can be retrieved from HTTP response header when downloading content offset += preview_step(&state, fileContent.data + offset, fileContent.size - offset); if(state.step_result == PREVIEW_FOUND_IMAGE) { if(Gif::isDataGif(fileContent)) { Gif *gif = new Gif(move(fileContent)); return { gif, ContentByUrlResult::Type::CACHED }; } else { if(!loadFromCache) { if(!downscaleImage(filepath, (void*)fileContent.data, fileContent.size, 100, 100)) { fprintf(stderr, "Failed to resize image: %s, using original file\n", filepath.c_str()); } } sf::Texture *texture = new sf::Texture(); if(texture->loadFromFile(filepath.c_str())) { delete[] fileContent.data; fileContent.data = nullptr; texture->setSmooth(true); texture->generateMipmap(); return { texture, ContentByUrlResult::Type::CACHED }; } delete texture; } break; } else if(state.step_result == PREVIEW_FOUND_TITLE) { foundHtmlContent = true; webPageTitle = sf::String::fromUtf8(state.meta_content, state.meta_content + state.meta_content_length); } else if(state.step_result == PREVIEW_FOUND_DESCRIPTION) { foundHtmlContent = true; webPageDescription = sf::String::fromUtf8(state.meta_content, state.meta_content + state.meta_content_length); } } while(offset < fileContent.size); delete[] fileContent.data; fileContent.data = nullptr; if(foundHtmlContent) { // TODO: Use move semantics for webPageTitle and webPageDescription when SFML supports it WebPagePreview *webPagePreview = new WebPagePreview(webPageTitle, webPageDescription); return { webPagePreview, ContentByUrlResult::Type::CACHED }; } } catch(std::exception &e) { fprintf(stderr, "Failed to load image %s, reason: %s\n", filepath.string().c_str(), e.what()); } return { (sf::Texture*)nullptr, ContentByUrlResult::Type::FAILED_DOWNLOAD }; } Cache::Cache() : alive(true) { downloadWaitThread = thread([this] { while(alive) { for(vector::iterator it = imageDownloadProcesses.begin(); it != imageDownloadProcesses.end();) { int exitStatus; if(it->process->try_get_exit_status(exitStatus)) { boost::filesystem::path filepath = getImagesDir(); odhtdb::Hash urlHash(it->url.data(), it->url.size()); filepath /= urlHash.toString(); ContentByUrlResult contentByUrlResult; bool failed = exitStatus != 0; if(failed) { contentByUrlResult = { (sf::Texture*)nullptr, ContentByUrlResult::Type::FAILED_DOWNLOAD }; } else { contentByUrlResult = loadImageFromFile(filepath, false); contentByUrlResult.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); if(contentByUrlResult.type == ContentByUrlResult::Type::CACHED) { printf("Download content from url: %s\n", it->url.c_str()); } } imageDownloadMutex.lock(); contentUrlCache[it->url] = contentByUrlResult; boost::filesystem::path downloadingFilepath = filepath; downloadingFilepath += ".downloading"; // Intentionally ignore failure, program should not crash if we fail to remove these files... boost::system::error_code err; boost::filesystem::remove(downloadingFilepath, err); imageDownloadMutex.unlock(); it = imageDownloadProcesses.erase(it); } else ++it; } while(alive && imageDownloadProcesses.empty() && imageDownloadProcessesQueue.empty()) this_thread::sleep_for(chrono::milliseconds(20)); if(!imageDownloadProcessesQueue.empty()) { imageDownloadMutex.lock(); for(auto imageDownloadInfo : imageDownloadProcessesQueue) { imageDownloadProcesses.push_back(imageDownloadInfo); } imageDownloadProcessesQueue.clear(); imageDownloadMutex.unlock(); } this_thread::sleep_for(chrono::milliseconds(20)); } }); checkContentAccessTimeThread = thread([this] { while(alive) { this_thread::sleep_for(chrono::milliseconds(500)); lock_guard lock(imageDownloadMutex); i64 now = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); for(unordered_map::iterator it = contentUrlCache.begin(); it != contentUrlCache.end();) { if(it->second.type == ContentByUrlResult::Type::CACHED && now - it->second.lastAccessed > CONTENT_NOT_VISIBLE_AGE_MS) { switch(it->second.cachedType) { case ContentByUrlResult::CachedType::TEXTURE: { if(ImagePreview::getPreviewContentPtr() == it->second.texture) { ++it; continue; } delete it->second.texture; break; } case ContentByUrlResult::CachedType::GIF: { if(ImagePreview::getPreviewContentPtr() == it->second.texture) { ++it; continue; } delete it->second.gif; break; } case ContentByUrlResult::CachedType::WEB_PAGE_PREVIEW: { delete it->second.webPagePreview; break; } default: ++it; continue; } it = contentUrlCache.erase(it); } else ++it; } } }); } Cache::~Cache() { alive = false; downloadWaitThread.join(); checkContentAccessTimeThread.join(); } static void createFileIgnoreError(const boost::filesystem::path &path) { try { fileReplace(path, StringView("", 0)); } catch(FileException &e) { fprintf(stderr, "Failed to create empty file: %s, reason: %s\n", path.string().c_str(), e.what()); } } const ContentByUrlResult Cache::getContentByUrl(const string &url, int downloadLimitBytes) { lock_guard lock(imageDownloadMutex); auto it = contentUrlCache.find(url); if(it != contentUrlCache.end()) { it->second.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); return it->second; } // TODO: Verify hashed url is not too long for filepath on windows boost::filesystem::path filepath = getImagesDir(); odhtdb::Hash urlHash(url.data(), url.size()); filepath /= urlHash.toString(); boost::filesystem::path downloadingFilepath = filepath; downloadingFilepath += ".downloading"; if(boost::filesystem::exists(downloadingFilepath)) { // Intentionally ignore failure, program should not crash if we fail to remove these files... boost::system::error_code err; boost::filesystem::remove(filepath, err); boost::filesystem::remove(downloadingFilepath, err); } // TODO: Do not load content in this thread. Return LOADING status and load it in another thread, because with a lot of images, chat can freeze ContentByUrlResult contentByUrlResult = loadImageFromFile(filepath, true); if(contentByUrlResult.type == ContentByUrlResult::Type::CACHED) { contentByUrlResult.lastAccessed = chrono::duration_cast(chrono::steady_clock::now().time_since_epoch()).count(); contentUrlCache[url] = contentByUrlResult; printf("Loaded content from file cache: %s\n", url.c_str()); return contentByUrlResult; } else if(contentByUrlResult.type == ContentByUrlResult::Type::FAILED_DOWNLOAD && boost::filesystem::exists(filepath)) { contentUrlCache[url] = contentByUrlResult; return contentByUrlResult; } createFileIgnoreError(downloadingFilepath); ContentByUrlResult result((sf::Texture*)nullptr, ContentByUrlResult::Type::DOWNLOADING); contentUrlCache[url] = result; string downloadLimitBytesStr = to_string(downloadLimitBytes); string escapedUrl = stringReplaceChar(url, "'", ""); escapedUrl = stringReplaceChar(escapedUrl, "\\", ""); Process::string_type cmd = "curl -L --silent -o '"; cmd += filepath.native(); cmd += "' --max-filesize " + downloadLimitBytesStr + " --range 0-" + downloadLimitBytesStr + " --url '" + escapedUrl + "'"; // TODO: Use this instead of curl on windows: certutil.exe -urlcache -split -f "https://url/to/file" path/and/name/to/save/as/file Process *process = new Process(cmd, "", nullptr, nullptr, false); ImageDownloadInfo imageDownloadInfo { process, url }; imageDownloadProcessesQueue.emplace_back(imageDownloadInfo); return result; } }