path: root/src/Cache.cpp

#include "../include/dchat/Cache.hpp"
#include "../include/env.hpp"
#include "../include/dchat/FileUtil.hpp"
#include "../include/dchat/Gif.hpp"
#include "../include/dchat/Storage.hpp"
#include "../include/dchat/Process.hpp"
#include <boost/filesystem/convenience.hpp>
#include <process.hpp>
#include <odhtdb/Hash.hpp>
#include <sibs/SafeSerializer.hpp>
#include <sibs/SafeDeserializer.hpp>
#include <libpreview.h>
#include <gd.h>

#if OS_FAMILY == OS_FAMILY_POSIX
#define toNativeString(str) str
#else
#include <string>
#include <userenv.h>
#include <locale>
#include <codecvt>

static std::wstring toNativeString(const std::string &str)
{
    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
    return converter.from_bytes(str);
}
#endif

using namespace std;
using namespace TinyProcessLib;

namespace dchat
{
    const i64 CONTENT_NOT_VISIBLE_AGE_MS = 30000; // Delete content from cache after a specified amount of time if the content is not visible on the screen

    static bool downscaleImage(const boost::filesystem::path &filepath, void *data, const int size, const int newWidth, const int newHeight)
    {
        gdImagePtr imgPtr = gdImageCreateFromPngPtr(size, data);
        if(!imgPtr)
            return false;

        int width = gdImageSX(imgPtr);
        if(width < newWidth)
        {
            gdImageDestroy(imgPtr);
            return false;
        }

        int height = gdImageSX(imgPtr);
        if(height < newHeight)
        {
            gdImageDestroy(imgPtr);
            return false;
        }

        gdImageSetInterpolationMethod(imgPtr, GD_BILINEAR_FIXED);
        gdImagePtr newImgPtr = gdImageScale(imgPtr, newWidth, newHeight);
        if(!newImgPtr)
        {
            gdImageDestroy(imgPtr);
            return false;
        }

        bool success = (gdImageFile(newImgPtr, filepath.string().c_str()) == 0);
        gdImageDestroy(imgPtr);
        gdImageDestroy(newImgPtr);
        return success;
    }
    
    ContentByUrlResult Cache::loadImageFromFile(const boost::filesystem::path &filepath, bool loadFromCache)
    {
        StringView fileContent;
        try
        {
            fileContent = getFileContent(filepath);
            
            std::string webPageTitle;
            std::string webPageDescription;
            bool foundHtmlContent = false;
            preview_state state;
            preview_init(&state);
            size_t offset = 0;
            do
            {
                // TODO: Get file content before doing this, the file might be in utf-16 encoding. That can happen for example if file contains html.
                // Content type can be retrieved from HTTP response header when downloading content
                offset += preview_step(&state, fileContent.data + offset, fileContent.size - offset);
                if(state.step_result == PREVIEW_FOUND_IMAGE)
                {
                    if(Gif::isDataGif(fileContent))
                    {
                        Gif *gif = createGifFunc(fileContent);
                        return { gif, ContentByUrlResult::Type::CACHED };
                    }
                    else
                    {
                        /*
                        if(!loadFromCache)
                        {
                            if(!downscaleImage(filepath, (void*)fileContent.data, fileContent.size, 100, 100))
                            {
                                fprintf(stderr, "Failed to resize image: %s, using original file\n", filepath.c_str());
                            }
                        }
                        */
                        StaticImage *image = createStaticImageFunc(filepath);
                        delete[] fileContent.data;
                        fileContent.data = nullptr;
                        return { image, ContentByUrlResult::Type::CACHED };
                    }
                }
                else if(state.step_result == PREVIEW_FOUND_TITLE)
                {
                    foundHtmlContent = true;
                    webPageTitle = std::string(state.meta_content, state.meta_content + state.meta_content_length);
                }
                else if(state.step_result == PREVIEW_FOUND_DESCRIPTION)
                {
                    foundHtmlContent = true;
                    webPageDescription = std::string(state.meta_content, state.meta_content + state.meta_content_length);
                }
            } while(offset < fileContent.size);
            
            delete[] fileContent.data;
            fileContent.data = nullptr;
            
            if(foundHtmlContent)
            {
                WebPagePreview *webPagePreview = createWebPagePreviewFunc(webPageTitle, webPageDescription);
                return { webPagePreview, ContentByUrlResult::Type::CACHED };
            }
        }
        catch(std::exception &e)
        {
            fprintf(stderr, "Failed to load image %s, reason: %s\n", filepath.string().c_str(), e.what());
        }
        return { (StaticImage*)nullptr, ContentByUrlResult::Type::FAILED_DOWNLOAD };
    }
    
    Cache::Cache(CreateGifFunc _createGifFunc, CreateStaticImageFunc _createStaticImageFunc, CreateWebPagePreviewFunc _createWebPagePreviewFunc) : 
        alive(true),
        createGifFunc(_createGifFunc),
        createStaticImageFunc(_createStaticImageFunc),
        createWebPagePreviewFunc(_createWebPagePreviewFunc)
    {
        assert(createGifFunc);
        assert(createStaticImageFunc);
        assert(createWebPagePreviewFunc);
        downloadWaitThread = thread([this]
        {
            while(alive)
            {
                for(vector<ImageDownloadInfo>::iterator it = imageDownloadProcesses.begin(); it != imageDownloadProcesses.end();)
                {
                    int exitStatus;
                    if(it->process->try_get_exit_status(exitStatus))
                    {
                        boost::filesystem::path filepath = getImagesDir();
                        odhtdb::Hash urlHash(it->url.data(), it->url.size());
                        filepath /= urlHash.toString();
                            
                        ContentByUrlResult contentByUrlResult;
                        bool failed = exitStatus != 0;
                        if(failed)
                        {
                            contentByUrlResult = { (StaticImage*)nullptr, ContentByUrlResult::Type::FAILED_DOWNLOAD };
                        }
                        else
                        {
                            contentByUrlResult = loadImageFromFile(filepath, false);
                            contentByUrlResult.lastAccessed = chrono::duration_cast<chrono::milliseconds>(chrono::steady_clock::now().time_since_epoch()).count();
                            if(contentByUrlResult.type == ContentByUrlResult::Type::CACHED)
                            {
                                printf("Download content from url: %s\n", it->url.c_str());
                            }
                        }
                        
                        imageDownloadMutex.lock();
                        contentUrlCache[it->url] = contentByUrlResult;
                        boost::filesystem::path downloadingFilepath = filepath;
                        downloadingFilepath += ".downloading";
                        // Intentionally ignore failure, program should not crash if we fail to remove these files...
                        boost::system::error_code err;
                        boost::filesystem::remove(downloadingFilepath, err);
                        imageDownloadMutex.unlock();
                        it = imageDownloadProcesses.erase(it);
                    }
                    else
                        ++it;
                }
                
                while(alive && imageDownloadProcesses.empty() && imageDownloadProcessesQueue.empty())
                    this_thread::sleep_for(chrono::milliseconds(20));
                
                if(!imageDownloadProcessesQueue.empty())
                {
                    imageDownloadMutex.lock();
                    for(auto imageDownloadInfo : imageDownloadProcessesQueue)
                    {
                        imageDownloadProcesses.push_back(imageDownloadInfo);
                    }
                    imageDownloadProcessesQueue.clear();
                    imageDownloadMutex.unlock();
                }
                
                this_thread::sleep_for(chrono::milliseconds(20));
            }
        });
        
        checkContentAccessTimeThread = thread([this]
        {
            // TODO: Add this back
            #if 0
            while(alive)
            {
                this_thread::sleep_for(chrono::milliseconds(500));
                lock_guard<mutex> lock(imageDownloadMutex);
                
                i64 now = chrono::duration_cast<chrono::milliseconds>(chrono::steady_clock::now().time_since_epoch()).count();
                for(unordered_map<string, ContentByUrlResult>::iterator it = contentUrlCache.begin(); it != contentUrlCache.end();)
                {
                    if(it->second.type == ContentByUrlResult::Type::CACHED && now - it->second.lastAccessed > CONTENT_NOT_VISIBLE_AGE_MS)
                    {
                        switch(it->second.cachedType)
                        {
                            case ContentByUrlResult::CachedType::TEXTURE_FILEPATH:
                            {
                                delete it->second.textureFilePath;
                                break;
                            }
                            case ContentByUrlResult::CachedType::GIF:
                            {
                                delete it->second.gif;
                                break;
                            }
                            case ContentByUrlResult::CachedType::WEB_PAGE_PREVIEW:
                            {
                                delete it->second.webPagePreview;
                                break;
                            }
                            default:
                                ++it;
                                continue;
                        }
                        it = contentUrlCache.erase(it);
                    }
                    else
                        ++it;
                }
            }
            #endif
        });
    }
    
    Cache::~Cache()
    {
        alive = false;
        downloadWaitThread.join();
        checkContentAccessTimeThread.join();
        // TODO: Add this back
        #if 0
        for(auto &it : contentUrlCache)
        {
            if(!it.second.textureFilePath)
                continue;

            switch(it.second.cachedType)
            {
                case ContentByUrlResult::CachedType::NONE:
                    break;
                case ContentByUrlResult::CachedType::TEXTURE_FILEPATH:
                {
                    delete it.second.textureFilePath;
                    break;
                }
                case ContentByUrlResult::CachedType::GIF:
                {
                    delete it.second.gif;
                    break;
                }
                case ContentByUrlResult::CachedType::WEB_PAGE_PREVIEW:
                {
                    delete it.second.webPagePreview;
                    break;
                }
                default:
                    // Did we forget to implement cleanup for a new cache content type?
                    assert(false);
                    break;
            }
        }
        #endif
    }
    
    static void createFileIgnoreError(const boost::filesystem::path &path)
    {
        try
        {
            fileReplace(path, StringView("", 0));
        }
        catch(FileException &e)
        {
            fprintf(stderr, "Failed to create empty file: %s, reason: %s\n", path.string().c_str(), e.what());
        }
    }
    
    const ContentByUrlResult Cache::getContentByUrl(const string &url, int downloadLimitBytes)
    {
        lock_guard<mutex> lock(imageDownloadMutex);
        auto it = contentUrlCache.find(url);
        if(it != contentUrlCache.end())
        {
            it->second.lastAccessed = chrono::duration_cast<chrono::milliseconds>(chrono::steady_clock::now().time_since_epoch()).count();
            return it->second;
        }
        
        // TODO: Verify hashed url is not too long for filepath on windows
        boost::filesystem::path filepath = getImagesDir();
        odhtdb::Hash urlHash(url.data(), url.size());
        filepath /= urlHash.toString();
        
        boost::filesystem::path downloadingFilepath = filepath;
        downloadingFilepath += ".downloading";
        if(boost::filesystem::exists(downloadingFilepath))
        {
            // Intentionally ignore failure, program should not crash if we fail to remove these files...
            boost::system::error_code err;
            boost::filesystem::remove(filepath, err);
            boost::filesystem::remove(downloadingFilepath, err);
        }
        
        // TODO: Do not load content in this thread. Return LOADING status and load it in another thread, because with a lot of images, chat can freeze
        ContentByUrlResult contentByUrlResult = loadImageFromFile(filepath, true);
        if(contentByUrlResult.type == ContentByUrlResult::Type::CACHED)
        {
            contentByUrlResult.lastAccessed = chrono::duration_cast<chrono::milliseconds>(chrono::steady_clock::now().time_since_epoch()).count();
            contentUrlCache[url] = contentByUrlResult;
            printf("Loaded content from file cache: %s\n", url.c_str());
            return contentByUrlResult;
        }
        else if(contentByUrlResult.type == ContentByUrlResult::Type::FAILED_DOWNLOAD && boost::filesystem::exists(filepath))
        {
            contentUrlCache[url] = contentByUrlResult;
            return contentByUrlResult;
        }
        
        createFileIgnoreError(downloadingFilepath);
        ContentByUrlResult result((StaticImage*)nullptr, ContentByUrlResult::Type::DOWNLOADING);
        contentUrlCache[url] = result;
        
        string downloadLimitBytesStr = to_string(downloadLimitBytes);
        
        std::string cmdUtf8 = "curl -L --silent -o '";
        cmdUtf8 += escapeCommand(filepath.string());
        cmdUtf8 += "' --max-filesize " + downloadLimitBytesStr + " --range 0-" + downloadLimitBytesStr + " --url '" + escapeCommand(url) + "'";
        Process::string_type cmd = toNativeString(cmdUtf8);
        // TODO: Use this instead of curl on windows: certutil.exe -urlcache -split -f "https://url/to/file" path/and/name/to/save/as/file
        Process *process = new Process(cmd, toNativeString(""), nullptr, nullptr, false);
        ImageDownloadInfo imageDownloadInfo { process, url };
        imageDownloadProcessesQueue.emplace_back(imageDownloadInfo);
        return result;
    }
}