path: root/src/DatabaseStorage.cpp

#include "../include/odhtdb/DatabaseStorage.hpp"
#include "../include/odhtdb/RemoteUser.hpp"
#include "../include/odhtdb/LocalUser.hpp"
#include "../include/odhtdb/LocalUserEncrypted.hpp"
#include "../include/odhtdb/Group.hpp"
#include "../include/odhtdb/FileUtils.hpp"
#include "../include/odhtdb/bin2hex.hpp"
#include "../include/odhtdb/PasswordHash.hpp"
#include "../include/odhtdb/Log.hpp"
#include <cstring>
#include <chrono>
#include <boost/filesystem/convenience.hpp>
#include <sibs/SafeSerializer.hpp>
#include <sodium/randombytes.h>

using namespace std;

namespace odhtdb
{
    enum StorageType : u8
    {
        STORAGE_TYPE_CREATE,
        STORAGE_TYPE_APPEND
    };
    
    const u64 QUARANTINE_STORAGE_TIME_MICROSECONDS = 60 * 1.0e6;
    
    DatabaseStorageObject::DatabaseStorageObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : 
        data(_data), createdTimestamp(_timestamp), creatorPublicKey(_creatorPublicKey)
    {
        
    }
    
    DatabaseStorageQuarantineObject::DatabaseStorageQuarantineObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : 
        data(_data), createdTimestamp(_timestamp), creatorPublicKey(_creatorPublicKey)
    {
        auto time = chrono::high_resolution_clock::now().time_since_epoch();
        storedTimestamp = chrono::duration_cast<chrono::microseconds>(time).count();
    }
    
    DatabaseStorage::DatabaseStorage(const boost::filesystem::path &storagePath) : 
        groupsFilePath(storagePath / "groups"),
        usersFilePath(storagePath / "users"),
        dataFilePath(storagePath / "data"),
        metadataFilePath(storagePath / "metadata")
    {
        boost::filesystem::create_directories(storagePath);
        
        bool metadataLoaded = false;
        try
        {
            loadMetadataFromFile();
            metadataLoaded = true;
            loadGroupsFromFile();
            loadUsersFromFile();
            loadDataFromFile();
            //loadQuarantineFromFile();
        }
        catch(FileException &e)
        {
            Log::warn("Failed to load data from file: %s, reason: %s. Ignoring...", dataFilePath.string().c_str(), e.what());
            if(!metadataLoaded)
            {
                sibs::SafeSerializer metadataSerializer;
                metadataSerializer.add((u16)0); // Storage version
                randombytes_buf(passwordSalt, PASSWORD_SALT_LEN);
                metadataSerializer.add(passwordSalt, PASSWORD_SALT_LEN);
                fileAppend(metadataFilePath, { metadataSerializer.getBuffer().data(), metadataSerializer.getBuffer().size() });
            }
        }
    }
    
    void DatabaseStorage::loadGroupsFromFile()
    {
        if(!boost::filesystem::exists(groupsFilePath)) return;
        
        OwnedMemory groupsFileContent = fileGetContent(groupsFilePath);
        sibs::SafeDeserializer deserializer((u8*)groupsFileContent.data, groupsFileContent.size);
        
        while(!deserializer.empty())
        {
            Hash nodeHash;
            deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
            
            DataViewMap<Group*> *groupByIdMap = nullptr;
            auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash);
            if(groupByIdMapIt == nodeGroupByIdMap.end())
            {
                groupByIdMap = new DataViewMap<Group*>();
                nodeGroupByIdMap[nodeHash] = groupByIdMap;
            }
            else
                groupByIdMap = groupByIdMapIt->second;
            
            u8 groupNameSize = deserializer.extract<u8>();
            string groupName;
            groupName.resize(groupNameSize);
            deserializer.extract((u8*)&groupName[0], groupNameSize);
            
            u8 groupId[GROUP_ID_LENGTH];
            deserializer.extract(groupId, GROUP_ID_LENGTH);
            
            u8 permissionLevel = deserializer.extract<u8>();
            u32 permissionFlags = deserializer.extract<u32>();
            
            Group *group = new Group(groupName, groupId, { permissionLevel, permissionFlags });
            (*groupByIdMap)[group->getId()] = group;
            
            string groupIdStr = bin2hex((const char*)groupId, GROUP_ID_LENGTH);
            Log::debug("DatabaseStorage: Loaded group from file: %s", groupIdStr.c_str());
        }
    }
    
    void DatabaseStorage::loadUsersFromFile()
    {
        if(!boost::filesystem::exists(usersFilePath)) return;
        
        OwnedMemory usersFileContent = fileGetContent(usersFilePath);
        sibs::SafeDeserializer deserializer((u8*)usersFileContent.data, usersFileContent.size);
        
        while(!deserializer.empty())
        {
            Hash nodeHash;
            deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
            
            Signature::MapPublicKey<UserData*> *publicKeyUserDataMap = nullptr;
            auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash);
            if(publicKeyUserDataMapIt == nodePublicKeyUserDataMap.end())
            {
                publicKeyUserDataMap = new Signature::MapPublicKey<UserData*>();
                nodePublicKeyUserDataMap[nodeHash] = publicKeyUserDataMap;
            }
            else
                publicKeyUserDataMap = publicKeyUserDataMapIt->second;
            
            DataViewMap<Group*> *groupByIdMap = nullptr;
            auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash);
            if(groupByIdMapIt == nodeGroupByIdMap.end())
            {
                groupByIdMap = new DataViewMap<Group*>();
                nodeGroupByIdMap[nodeHash] = groupByIdMap;
            }
            else
                groupByIdMap = groupByIdMapIt->second;
            
            User::Type userType = deserializer.extract<User::Type>();
            
            u8 usernameSize = deserializer.extract<u8>();
            string username;
            username.resize(usernameSize);
            deserializer.extract((u8*)&username[0], usernameSize);
            
            u8 userPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
            deserializer.extract(userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
            Signature::PublicKey userPublicKey((const char*)userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
            
            UserData *userData = new UserData();
            if(userType == User::Type::LOCAL)
            {
                EncryptedPrivateKey encryptedPrivateKey;
                deserializer.extract(encryptedPrivateKey.nonce, ENCRYPTION_NONCE_BYTE_SIZE);
                deserializer.extract(encryptedPrivateKey.encryptedPrivateKey, 16 + PRIVATE_KEY_NUM_BYTES);
                userData->user = LocalUserEncrypted::create(userPublicKey, encryptedPrivateKey, username, nullptr);
            }
            else
            {
                userData->user = RemoteUser::create(userPublicKey, username, nullptr);
            }
            
            u8 numGroups = deserializer.extract<u8>();
            for(int i = 0; i < numGroups; ++i)
            {
                u8 groupId[GROUP_ID_LENGTH];
                deserializer.extract(groupId, GROUP_ID_LENGTH);
                
                auto groupIt = groupByIdMap->find(DataView(groupId, GROUP_ID_LENGTH));
                if(groupIt == groupByIdMap->end())
                {
                    string errMsg = "User group with id ";
                    errMsg += bin2hex((const char*)groupId, GROUP_ID_LENGTH);
                    errMsg += " does not exist";
                    throw DatabaseStorageCorrupt(errMsg);
                }
                userData->user->addToGroup(groupIt->second);
            }
            
            (*publicKeyUserDataMap)[userData->user->getPublicKey()] = userData;
        }
    }
    
    void DatabaseStorage::loadStorageCreate(sibs::SafeDeserializer &deserializer)
    {
        u64 timestamp = deserializer.extract<u64>();
        
        u8 groupId[GROUP_ID_LENGTH];
        deserializer.extract(groupId, GROUP_ID_LENGTH);
        
        u32 dataSize = deserializer.extract<u32>();
        u8 *data = new u8[dataSize];
        deserializer.extract(data, dataSize);
        
        Hash nodeHash;
        deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        
        auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash);
        if(groupByIdMapIt == nodeGroupByIdMap.end())
        {
            string errMsg = "No groups exists in node ";
            errMsg += nodeHash.toString();
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        auto groupIt = groupByIdMapIt->second->find(DataView(groupId, GROUP_ID_LENGTH));
        if(groupIt == groupByIdMapIt->second->end())
        {
            string errMsg = "User group with id ";
            errMsg += bin2hex((const char*)groupId, GROUP_ID_LENGTH);
            errMsg += " does not exist in node ";
            errMsg += nodeHash.toString();
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList();
        databaseStorageObjectList->createdTimestamp = timestamp;
        databaseStorageObjectList->groups.push_back(groupIt->second);
        databaseStorageObjectList->data = DataView(data, dataSize);
        storageMap[nodeHash] = databaseStorageObjectList;
    }
    
    void DatabaseStorage::loadStorageAppend(sibs::SafeDeserializer &deserializer)
    {
        u64 timestamp = deserializer.extract<u64>();
        
        u8 creatorPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
        deserializer.extract(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
        Signature::PublicKey creatorPublicKey((const char*)creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
        
        u32 dataSize = deserializer.extract<u32>();
        u8 *data = new u8[dataSize];
        deserializer.extract(data, dataSize);
        
        Hash nodeHash;
        deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        
        Hash dataHash;
        deserializer.extract((u8*)dataHash.getData(), HASH_BYTE_SIZE);
        storedDataHash.insert(dataHash);
        
        auto storageIt = storageMap.find(nodeHash);
        if(storageIt == storageMap.end())
        {
            string errMsg = "Database storage with hash ";
            errMsg += nodeHash.toString();
            errMsg += " not found";
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        DataView storageData { data, dataSize };
        DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(storageData, timestamp, creatorPublicKey);
        storageIt->second->objects.push_back(databaseStorageObject);
    }
    
    void DatabaseStorage::loadDataFromFile()
    {
        if(!boost::filesystem::exists(dataFilePath)) return;
        
        OwnedMemory dataFileContent = fileGetContent(dataFilePath);
        sibs::SafeDeserializer deserializer((u8*)dataFileContent.data, dataFileContent.size);
        
        while(!deserializer.empty())
        {
            StorageType storageType = deserializer.extract<StorageType>();
            switch(storageType)
            {
                case STORAGE_TYPE_CREATE:
                    loadStorageCreate(deserializer);
                    break;
                case STORAGE_TYPE_APPEND:
                    loadStorageAppend(deserializer);
                    break;
            }
        }
    }
    
    void DatabaseStorage::loadMetadataFromFile()
    {
        OwnedMemory metadataFileContent = fileGetContent(metadataFilePath);
        sibs::SafeDeserializer deserializer((u8*)metadataFileContent.data, metadataFileContent.size);
        
        u16 storageVersion = deserializer.extract<u16>();
        
        u8 passwordSalt[PASSWORD_SALT_LEN];
        deserializer.extract(passwordSalt, PASSWORD_SALT_LEN);
        
        assert(deserializer.empty());
    }
    
    void DatabaseStorage::createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize)
    {
        if(storageMap.find(hash) != storageMap.end())
        {
            string errMsg = "Database storage with hash ";
            errMsg += hash.toString();
            errMsg += " already exists";
            throw DatabaseStorageAlreadyExists(errMsg);
        }
        
        addGroup(hash, creatorGroup);
        for(auto user : creatorGroup->getUsers())
        {
            addUser(hash, (User*)user);
        }
        
        DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList();
        databaseStorageObjectList->createdTimestamp = timestamp;
        databaseStorageObjectList->groups.push_back(creatorGroup);
        databaseStorageObjectList->data = DataView(new u8[dataSize], dataSize);
        memcpy(databaseStorageObjectList->data.data, data, dataSize);
        storageMap[hash] = databaseStorageObjectList;
        
        sibs::SafeSerializer serializer;
        serializer.add(STORAGE_TYPE_CREATE);
        serializer.add(timestamp);
        serializer.add((u8*)creatorGroup->getId().data, GROUP_ID_LENGTH);
        
        serializer.add((u32)dataSize);
        serializer.add(data, dataSize);
        
        serializer.add((u8*)hash.getData(), HASH_BYTE_SIZE);
        
        fileAppend(dataFilePath, { serializer.getBuffer().data(), serializer.getBuffer().size() });
    }
    
    void DatabaseStorage::appendStorage(const Hash &nodeHash, const Hash &dataHash, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize)
    {
        auto it = storageMap.find(nodeHash);
        if(it == storageMap.end())
        {
            string errMsg = "Database storage with hash ";
            errMsg += nodeHash.toString();
            errMsg += " not found. Storage for a hash needs to be created before data can be appended to it";
            throw DatabaseStorageNotFound(errMsg);
        }
        
        auto storeDataHashResult = storedDataHash.insert(dataHash);
        if(!storeDataHashResult.second)
        {
            string errMsg = "Database already contains data with hash: ";
            errMsg += dataHash.toString();
            throw DatabaseStorageAlreadyExists(errMsg);
        }
        
        DataView storageData { new u8[dataSize], dataSize };
        memcpy(storageData.data, data, dataSize);
        DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(storageData, timestamp, creatorUser->getPublicKey());
        it->second->objects.push_back(databaseStorageObject);
        
        sibs::SafeSerializer serializer;
        serializer.add(STORAGE_TYPE_APPEND);
        serializer.add(timestamp);
        serializer.add((u8*)creatorUser->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES);
        
        serializer.add((u32)dataSize);
        serializer.add(data, dataSize);
        
        serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        serializer.add((u8*)dataHash.getData(), HASH_BYTE_SIZE);
        
        fileAppend(dataFilePath, { serializer.getBuffer().data(), serializer.getBuffer().size() });
    }
    
    void DatabaseStorage::addToQuarantine(const Hash &dataHash, const Signature::PublicKey &creatorPublicKey, u64 timestamp, const u8 *data, usize dataSize)
    {
        auto storeDataHashResult = storedDataHash.insert(dataHash);
        if(!storeDataHashResult.second)
        {
            string errMsg = "Database already contains data with hash: ";
            errMsg += dataHash.toString();
            throw DatabaseStorageAlreadyExists(errMsg);
        }
        
        DataView storageData { new u8[dataSize], dataSize };
        memcpy(storageData.data, data, dataSize);
        DatabaseStorageQuarantineObject *databaseQuarantineStorageObject = new DatabaseStorageQuarantineObject(storageData, timestamp, creatorPublicKey);
        quarantineStorageMap[creatorPublicKey].emplace_back(databaseQuarantineStorageObject);
    }
    
    bool DatabaseStorage::addGroup(const Hash &nodeHash, Group *group)
    {
        DataViewMap<Group*> *groupByIdMap = nullptr;
        auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash);
        if(groupByIdMapIt == nodeGroupByIdMap.end())
        {
            groupByIdMap = new DataViewMap<Group*>();
            nodeGroupByIdMap[nodeHash] = groupByIdMap;
        }
        else
            groupByIdMap = groupByIdMapIt->second;
        
        if(groupByIdMap->find(group->getId()) != groupByIdMap->end())
            return false;
        
        (*groupByIdMap)[group->getId()] = group;
        
        sibs::SafeSerializer serializer;
        
        serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        
        serializer.add((u8)group->getName().size());
        serializer.add((u8*)group->getName().data(), group->getName().size());
        
        serializer.add((u8*)group->getId().data, GROUP_ID_LENGTH);
        
        serializer.add(group->getPermission().getPermissionLevel());
        serializer.add(group->getPermission().getPermissionFlags());
        
        fileAppend(groupsFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size()));
        
        return true;
    }
    
    bool DatabaseStorage::addUser(const Hash &nodeHash, User *user)
    {
        Signature::MapPublicKey<UserData*> *publicKeyUserDataMap = nullptr;
        auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash);
        if(publicKeyUserDataMapIt == nodePublicKeyUserDataMap.end())
        {
            publicKeyUserDataMap = new Signature::MapPublicKey<UserData*>();
            nodePublicKeyUserDataMap[nodeHash] = publicKeyUserDataMap;
        }
        else
            publicKeyUserDataMap = publicKeyUserDataMapIt->second;
        
        if(publicKeyUserDataMap->find(user->getPublicKey()) != publicKeyUserDataMap->end())
            return false;
        
        UserData *userData = new UserData();
        userData->user = user;
        if(user->getType() == User::Type::LOCAL)
        {
            LocalUser *localUser = static_cast<LocalUser*>(user);
            //DataView plainPassword((void*)localUser->getPlainPassword().data(), localUser->getPlainPassword().size());
            OwnedMemory hashedPassword = hashPassword(DataView((void*)localUser->getPlainPassword().data(), localUser->getPlainPassword().size()), DataView(passwordSalt, PASSWORD_SALT_LEN));
            memcpy(userData->hashedPassword, hashedPassword.data, hashedPassword.size);
        }
        else
        {
            memset(userData->hashedPassword, 0, HASHED_PASSWORD_LEN);
        }
        (*publicKeyUserDataMap)[user->getPublicKey()] = userData;
        
        // TODO: Instead of directly saving user to file, maybe user should be added to buffer and then save to file after we have several users (performance improvement)
        
        sibs::SafeSerializer serializer;
        
        serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        
        serializer.add(user->getType());
        
        serializer.add((u8)user->getName().size());
        serializer.add((u8*)user->getName().data(), user->getName().size());
        
        serializer.add((u8*)user->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES);
        if(user->getType() == User::Type::LOCAL)
        {
            LocalUser *localUser = static_cast<LocalUser*>(user);
            static_assert(HASHED_PASSWORD_LEN == ENCRYPTION_KEY_BYTE_SIZE, "Encryption key size has changed but hashed password size hasn't");
            Encryption encryptedPrivateKey(DataView((void*)localUser->getPrivateKey().getData(), localUser->getPrivateKey().getSize()), 
                                                DataView(),
                                                DataView(userData->hashedPassword, HASHED_PASSWORD_LEN));
            serializer.add((u8*)encryptedPrivateKey.getNonce().data, ENCRYPTION_NONCE_BYTE_SIZE);
            assert(16 + PRIVATE_KEY_NUM_BYTES == encryptedPrivateKey.getCipherText().size);
            serializer.add((u8*)encryptedPrivateKey.getCipherText().data, encryptedPrivateKey.getCipherText().size);
        }
        
        serializer.add((u8)user->getGroups().size());
        for(Group *group : user->getGroups())
        {
            serializer.add((u8*)group->getId().data, GROUP_ID_LENGTH);
        }
        
        fileAppend(usersFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size()));
        
        return true;
    }
    
    const DatabaseStorageObjectList* DatabaseStorage::getStorage(const Hash &hash) const
    {
        auto it = storageMap.find(hash);
        if(it != storageMap.end())
            return it->second;
        return nullptr;
    }
    
    Group* DatabaseStorage::getGroupById(const Hash &nodeHash, uint8_t groupId[GROUP_ID_LENGTH])
    {
        auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash);
        if(groupByIdMapIt != nodeGroupByIdMap.end())
        {
            auto it = groupByIdMapIt->second->find(DataView(groupId, GROUP_ID_LENGTH));
            if(it != groupByIdMapIt->second->end())
                return it->second;
        }
        return nullptr;
    }
    
    User* DatabaseStorage::getUserByPublicKey(const Hash &nodeHash, Signature::PublicKey &userPublicKey)
    {
        auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash);
        if(publicKeyUserDataMapIt != nodePublicKeyUserDataMap.end())
        {
            auto it = publicKeyUserDataMapIt->second->find(userPublicKey);
            if(it != publicKeyUserDataMapIt->second->end())
                return it->second->user;
        }
        return nullptr;
    }
    
    const Signature::MapPublicKey<DatabaseStorage::UserData*>* DatabaseStorage::getUsersData(const Hash &nodeHash) const
    {
        auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash);
        if(publicKeyUserDataMapIt != nodePublicKeyUserDataMap.end())
            return publicKeyUserDataMapIt->second;
        return nullptr;
    }
    
    void DatabaseStorage::update()
    {
        // TODO: Modify this to iterate backwards. Because list is sorted in order of timestamp, we can remove data in range
        auto time = chrono::high_resolution_clock::now().time_since_epoch();
        auto timeMicroseconds = chrono::duration_cast<chrono::microseconds>(time).count();
        for(auto mapIt = quarantineStorageMap.begin(); mapIt != quarantineStorageMap.end(); )
        {
            for(auto vecIt = mapIt->second.begin(); vecIt != mapIt->second.end(); )
            {
                if(timeMicroseconds - (*vecIt)->storedTimestamp > QUARANTINE_STORAGE_TIME_MICROSECONDS)
                    vecIt = mapIt->second.erase(vecIt);
                else
                    ++vecIt;
            }
            
            if(mapIt->second.empty())
                mapIt = quarantineStorageMap.erase(mapIt);
            else
                ++mapIt;
        }
    }
}