#include "../include/odhtdb/DatabaseStorage.hpp"
#include "../include/odhtdb/RemoteUser.hpp"
#include "../include/odhtdb/LocalUser.hpp"
#include "../include/odhtdb/LocalUserEncrypted.hpp"
#include "../include/odhtdb/Group.hpp"
#include "../include/odhtdb/FileUtils.hpp"
#include "../include/odhtdb/bin2hex.hpp"
#include "../include/odhtdb/PasswordHash.hpp"
#include "../include/odhtdb/Log.hpp"
#include "../include/odhtdb/Database.hpp"
#include <cstring>
#include <chrono>
#include <boost/filesystem/convenience.hpp>
#include <sibs/SafeSerializer.hpp>
#include <sodium/randombytes.h>

using namespace std;

namespace odhtdb
{
    enum StorageType : u8
    {
        STORAGE_TYPE_CREATE,
        STORAGE_TYPE_APPEND
    };
    
    enum class DecryptedDataType : u8
    {
        STORAGE_CREATE,
        STORAGE_ADD_DATA,
        STORAGE_ADD_USER
    };
    
    const u64 QUARANTINE_STORAGE_TIME_MICROSECONDS = 60 * 1.0e6;
    const u16 STORAGE_VERSION = 1;

    DatabaseStorageObject::DatabaseStorageObject(const Hash &_requestHash, DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : 
        requestHash(_requestHash),
        data(_data),
        createdTimestamp(_timestamp), 
        creatorPublicKey(_creatorPublicKey)
    {
        
    }
    
    DatabaseStorageQuarantineObject::DatabaseStorageQuarantineObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : 
        data(_data), createdTimestamp(_timestamp), creatorPublicKey(_creatorPublicKey)
    {
        auto time = chrono::high_resolution_clock::now().time_since_epoch();
        storedTimestamp = chrono::duration_cast<chrono::microseconds>(time).count();
    }
    
    DatabaseStorage::DatabaseStorage(Database *_database, const boost::filesystem::path &storagePath) : 
        database(_database),
        groupsFilePath(storagePath / "groups"),
        usersFilePath(storagePath / "users"),
        dataFilePath(storagePath / "data"),
        metadataFilePath(storagePath / "metadata"),
        localUsersFilePath(storagePath / "local_users"),
        nodeDecryptionKeysFilePath(storagePath / "node_keys"),
        decryptedDataFilePath(storagePath / "decrypted_data")
    {
        boost::filesystem::create_directories(storagePath);
        
        bool metadataLoaded = false;
        try
        {
            loadMetadataFromFile();
            metadataLoaded = true;
            loadGroupsFromFile();
            loadUsersFromFile();
            loadDataFromFile();
            loadLocalUsersFromFile();
            loadNodeDecryptionKeysFromFile();
            loadDecryptedDataFromFile();
            //loadQuarantineFromFile();
        }
        catch(FileException &e)
        {
            if(metadataLoaded)
            {
                string errMsg = "Failed to load storage, reason: ";
                errMsg += e.what();
                throw DatabaseStorageCorrupt(errMsg);
            }
            else
            {
                Log::warn("Failed to load storage meta data, reason: %s. Ignoring... (new storage probably)", e.what());
                sibs::SafeSerializer metadataSerializer;
                metadataSerializer.add(STORAGE_VERSION);
                randombytes_buf(passwordSalt, PASSWORD_SALT_LEN);
                metadataSerializer.add(passwordSalt, PASSWORD_SALT_LEN);
                
                //string passwordSaltStr((const char*)passwordSalt, PASSWORD_SALT_LEN);
                identity = dht::crypto::generateIdentity();
                dht::Blob privateKeyData = identity.first->serialize();
                metadataSerializer.add((u16)privateKeyData.size());
                metadataSerializer.add(privateKeyData.data(), privateKeyData.size());

                dht::Blob certificateData;
                identity.second->pack(certificateData);
                metadataSerializer.add((u16)certificateData.size());
                metadataSerializer.add(certificateData.data(), certificateData.size());
                
                fileAppend(metadataFilePath, { metadataSerializer.getBuffer().data(), metadataSerializer.getBuffer().size() });
            }
        }
        catch(sibs::DeserializeException &e)
        {
            string errMsg = "Failed to load storage, reason: ";
            errMsg += e.what();
            throw DatabaseStorageCorrupt(errMsg);
        }
    }
    
    void DatabaseStorage::loadGroupsFromFile()
    {
        if(!boost::filesystem::exists(groupsFilePath)) return;
        
        OwnedMemory groupsFileContent = fileGetContent(groupsFilePath);
        sibs::SafeDeserializer deserializer((u8*)groupsFileContent.data, groupsFileContent.size);
        
        while(!deserializer.empty())
        {
            Hash nodeHash;
            deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
            
            DataViewMap<Group*> *groupByIdMap = nullptr;
            auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash);
            if(groupByIdMapIt == nodeGroupByIdMap.end())
            {
                groupByIdMap = new DataViewMap<Group*>();
                nodeGroupByIdMap[nodeHash] = groupByIdMap;
            }
            else
                groupByIdMap = groupByIdMapIt->second;
            
            u8 groupNameSize = deserializer.extract<u8>();
            string groupName;
            groupName.resize(groupNameSize);
            deserializer.extract((u8*)&groupName[0], groupNameSize);
            
            u8 groupId[GROUP_ID_LENGTH];
            deserializer.extract(groupId, GROUP_ID_LENGTH);
            
            u8 permissionLevel = deserializer.extract<u8>();
            u32 permissionFlags = deserializer.extract<u32>();
            
            Group *group = new Group(groupName, groupId, { permissionLevel, permissionFlags });
            (*groupByIdMap)[group->getId()] = group;
            
            string groupIdStr = bin2hex((const char*)groupId, GROUP_ID_LENGTH);
            Log::debug("DatabaseStorage: Loaded group %s from file", groupIdStr.c_str());
        }
    }
    
    void DatabaseStorage::loadUsersFromFile()
    {
        if(!boost::filesystem::exists(usersFilePath)) return;
        
        OwnedMemory usersFileContent = fileGetContent(usersFilePath);
        sibs::SafeDeserializer deserializer((u8*)usersFileContent.data, usersFileContent.size);
        
        while(!deserializer.empty())
        {
            Hash nodeHash;
            deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
            
            Signature::MapPublicKey<User*> *publicKeyUserDataMap = nullptr;
            auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash);
            if(publicKeyUserDataMapIt == nodePublicKeyUserDataMap.end())
            {
                publicKeyUserDataMap = new Signature::MapPublicKey<User*>();
                nodePublicKeyUserDataMap[nodeHash] = publicKeyUserDataMap;
            }
            else
                publicKeyUserDataMap = publicKeyUserDataMapIt->second;
            
            DataViewMap<Group*> *groupByIdMap = nullptr;
            auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash);
            if(groupByIdMapIt == nodeGroupByIdMap.end())
            {
                groupByIdMap = new DataViewMap<Group*>();
                nodeGroupByIdMap[nodeHash] = groupByIdMap;
            }
            else
                groupByIdMap = groupByIdMapIt->second;
            
            u8 usernameSize = deserializer.extract<u8>();
            string username;
            username.resize(usernameSize);
            deserializer.extract((u8*)&username[0], usernameSize);
            
            u8 userPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
            deserializer.extract(userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
            Signature::PublicKey userPublicKey((const char*)userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
            
            User *user = RemoteUser::create(userPublicKey, username, nullptr);
            
            u8 numGroups = deserializer.extract<u8>();
            for(int i = 0; i < numGroups; ++i)
            {
                u8 groupId[GROUP_ID_LENGTH];
                deserializer.extract(groupId, GROUP_ID_LENGTH);
                
                auto groupIt = groupByIdMap->find(DataView(groupId, GROUP_ID_LENGTH));
                if(groupIt == groupByIdMap->end())
                {
                    string errMsg = "User group with id ";
                    errMsg += bin2hex((const char*)groupId, GROUP_ID_LENGTH);
                    errMsg += " does not exist";
                    throw DatabaseStorageCorrupt(errMsg);
                }
                user->addToGroup(groupIt->second);
            }
            
            (*publicKeyUserDataMap)[user->getPublicKey()] = user;
        }
    }
    
    void DatabaseStorage::loadStorageCreate(sibs::SafeDeserializer &deserializer)
    {
        u64 timestamp = deserializer.extract<u64>();
        
        u8 userPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
        deserializer.extract(userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
        Signature::PublicKey userPublicKey((const char*)userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
        
        u8 groupId[GROUP_ID_LENGTH];
        deserializer.extract(groupId, GROUP_ID_LENGTH);
        
        u32 dataSize = deserializer.extract<u32>();
        u8 *data = new u8[dataSize];
        deserializer.extract(data, dataSize);
        u32 offsetToEncryptedData = deserializer.extract<u32>();
        
        Hash nodeHash;
        deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        
        auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash);
        if(groupByIdMapIt == nodeGroupByIdMap.end())
        {
            string errMsg = "No groups exists in node ";
            errMsg += nodeHash.toString();
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        auto groupIt = groupByIdMapIt->second->find(DataView(groupId, GROUP_ID_LENGTH));
        if(groupIt == groupByIdMapIt->second->end())
        {
            string errMsg = "User group with id ";
            errMsg += bin2hex((const char*)groupId, GROUP_ID_LENGTH);
            errMsg += " does not exist in node ";
            errMsg += nodeHash.toString();
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(userPublicKey);
        databaseStorageObjectList->createdTimestamp = timestamp;
        databaseStorageObjectList->groups.push_back(groupIt->second);
        databaseStorageObjectList->data = DataView(data, dataSize);
        databaseStorageObjectList->offsetToEncryptedData = offsetToEncryptedData;
        storageMap[nodeHash] = databaseStorageObjectList;
    }
    
    void DatabaseStorage::loadStorageAppend(sibs::SafeDeserializer &deserializer)
    {
        u64 timestamp = deserializer.extract<u64>();
        
        u8 creatorPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
        deserializer.extract(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
        Signature::PublicKey creatorPublicKey((const char*)creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
        
        u32 dataSize = deserializer.extract<u32>();
        u8 *data = new u8[dataSize];
        deserializer.extract(data, dataSize);
        
        Hash nodeHash;
        deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        
        Hash dataHash;
        deserializer.extract((u8*)dataHash.getData(), HASH_BYTE_SIZE);
        
        auto storageIt = storageMap.find(nodeHash);
        if(storageIt == storageMap.end())
        {
            string errMsg = "Database storage with hash ";
            errMsg += nodeHash.toString();
            errMsg += " not found";
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        DataView storageData { data, dataSize };
        DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(dataHash, storageData, timestamp, creatorPublicKey);
        storageIt->second->objects.push_back(databaseStorageObject);
        storedDataHash[dataHash] = databaseStorageObject;
    }
    
    void DatabaseStorage::loadDataFromFile()
    {
        if(!boost::filesystem::exists(dataFilePath)) return;
        
        OwnedMemory dataFileContent = fileGetContent(dataFilePath);
        sibs::SafeDeserializer deserializer((u8*)dataFileContent.data, dataFileContent.size);
        
        while(!deserializer.empty())
        {
            StorageType storageType = deserializer.extract<StorageType>();
            switch(storageType)
            {
                case STORAGE_TYPE_CREATE:
                    loadStorageCreate(deserializer);
                    break;
                case STORAGE_TYPE_APPEND:
                    loadStorageAppend(deserializer);
                    break;
            }
        }
    }
    
    void DatabaseStorage::loadLocalUsersFromFile()
    {
        if(!boost::filesystem::exists(localUsersFilePath)) return;
        
        OwnedMemory localUsersFileContent = fileGetContent(localUsersFilePath);
        sibs::SafeDeserializer deserializer((u8*)localUsersFileContent.data, localUsersFileContent.size);
        
        while(!deserializer.empty())
        {
            u8 usernameSize = deserializer.extract<u8>();
            string username;
            username.resize(usernameSize);
            deserializer.extract((u8*)&username[0], usernameSize);
            
            u8 userPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
            deserializer.extract(userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
            Signature::PublicKey userPublicKey((const char*)userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
            
            EncryptedPrivateKey encryptedPrivateKey;
            deserializer.extract(encryptedPrivateKey.nonce, ENCRYPTION_NONCE_BYTE_SIZE);
            deserializer.extract(encryptedPrivateKey.encryptedPrivateKey, ENCRYPTION_CHECKSUM_BYTE_SIZE + PRIVATE_KEY_NUM_BYTES);
            
            nameLocalUsersMap[username] = LocalUserEncrypted::create(userPublicKey, encryptedPrivateKey, username);
        }
    }
    
    void DatabaseStorage::loadNodeDecryptionKeysFromFile()
    {
        if(!boost::filesystem::exists(nodeDecryptionKeysFilePath)) return;
        
        OwnedMemory nodeKeysFileContent = fileGetContent(nodeDecryptionKeysFilePath);
        sibs::SafeDeserializer deserializer((u8*)nodeKeysFileContent.data, nodeKeysFileContent.size);
        
        while(!deserializer.empty())
        {
            Hash nodeHash;
            deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
            
            u8 *nodeKeyRaw = new u8[ENCRYPTION_KEY_BYTE_SIZE];
            deserializer.extract(nodeKeyRaw, ENCRYPTION_KEY_BYTE_SIZE);
            nodeDecryptionKeyMap[nodeHash] = make_shared<OwnedMemory>(nodeKeyRaw, ENCRYPTION_KEY_BYTE_SIZE);
        }
    }
    
    void DatabaseStorage::loadDecryptedStorageCreate(sibs::SafeDeserializer &deserializer)
    {
        Hash nodeHash;
        deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        
        auto storageIt = storageMap.find(nodeHash);
        if(storageIt == storageMap.end())
        {
            string errMsg = "Database storage with hash ";
            errMsg += nodeHash.toString();
            errMsg += " not found";
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        u8 creatorNameLength = deserializer.extract<u8>();
        auto creator = getUserByPublicKey(nodeHash, storageIt->second->creatorPublicKey);
        if(!creator)
        {
            string errMsg = "User with public key ";
            errMsg += storageIt->second->creatorPublicKey.toString();
            errMsg += " does not exist in node with hash ";
            errMsg += nodeHash.toString();
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        creator->name.resize(creatorNameLength);
        deserializer.extract((u8*)&creator->name[0], creatorNameLength);
        
        u8 nodeNameLength = deserializer.extract<u8>();
        storageIt->second->nodeName.resize(nodeNameLength);
        deserializer.extract((u8*)&storageIt->second->nodeName[0], nodeNameLength);
        storageIt->second->isDecrypted = true;
    }
    
    void DatabaseStorage::loadDecryptedStorageAddData(sibs::SafeDeserializer &deserializer)
    {
        Hash requestHash;
        deserializer.extract((u8*)requestHash.getData(), HASH_BYTE_SIZE);
        
        auto storedDataIt = storedDataHash.find(requestHash);
        if(storedDataIt == storedDataHash.end())
        {
            string errMsg = "Database doesn't contain data with hash ";
            errMsg += requestHash.toString();
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        u32 decryptedDataSize = deserializer.extract<u32>();
        u8 *decryptedDataRaw = new u8[decryptedDataSize];
        deserializer.extract(decryptedDataRaw, decryptedDataSize);
        
        storedDataIt->second->decryptedObject.data.data = decryptedDataRaw;
        storedDataIt->second->decryptedObject.data.size = decryptedDataSize;
        storedDataIt->second->decryptedObject.operation = DatabaseOperation::ADD_DATA;
    }
    
    void DatabaseStorage::loadDecryptedStorageAddUser(sibs::SafeDeserializer &deserializer)
    {
        Hash nodeHash;
        deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        
        Hash requestHash;
        deserializer.extract((u8*)requestHash.getData(), HASH_BYTE_SIZE);
        
        auto storedDataIt = storedDataHash.find(requestHash);
        if(storedDataIt == storedDataHash.end())
        {
            string errMsg = "Database doesn't contain data with hash ";
            errMsg += requestHash.toString();
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        u8 addedUserPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
        deserializer.extract(addedUserPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
        Signature::PublicKey addedUserPublicKey((const char*)addedUserPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
        auto addedUser = getUserByPublicKey(nodeHash, addedUserPublicKey);
        if(!addedUser)
        {
            string errMsg = "User with public key ";
            errMsg += bin2hex((const char*)addedUserPublicKeyRaw, PUBLIC_KEY_NUM_BYTES).c_str();
            errMsg += " does not exist in node with hash ";
            errMsg += nodeHash.toString();
            throw DatabaseStorageCorrupt(errMsg);
        }
        
        u32 decryptedUserNameSize = deserializer.extract<u32>();
        addedUser->name.resize(decryptedUserNameSize);
        deserializer.extract((u8*)&addedUser->name[0], decryptedUserNameSize);
    }
    
    void DatabaseStorage::loadDecryptedDataFromFile()
    {
        if(!boost::filesystem::exists(decryptedDataFilePath)) return;
        
        OwnedMemory decryptedDataFileContent = fileGetContent(decryptedDataFilePath);
        sibs::SafeDeserializer deserializer((u8*)decryptedDataFileContent.data, decryptedDataFileContent.size);
        
        while(!deserializer.empty())
        {
            DecryptedDataType decryptedDataType = deserializer.extract<DecryptedDataType>();
            switch(decryptedDataType)
            {
                case DecryptedDataType::STORAGE_CREATE:
                    loadDecryptedStorageCreate(deserializer);
                    break;
                case DecryptedDataType::STORAGE_ADD_DATA:
                    loadDecryptedStorageAddData(deserializer);
                    break;
                case DecryptedDataType::STORAGE_ADD_USER:
                    loadDecryptedStorageAddUser(deserializer);
                    break;
            }
        }
    }
    
    void DatabaseStorage::loadMetadataFromFile()
    {
        OwnedMemory metadataFileContent = fileGetContent(metadataFilePath);
        sibs::SafeDeserializer deserializer((u8*)metadataFileContent.data, metadataFileContent.size);
        
        u16 storageVersion = deserializer.extract<u16>();
        if(storageVersion != STORAGE_VERSION)
            throw std::runtime_error("Wrong storage version!");
        
        deserializer.extract(passwordSalt, PASSWORD_SALT_LEN);
        //string passwordSaltStr((const char*)passwordSalt, PASSWORD_SALT_LEN);
        
        u16 privateKeySize = deserializer.extract<u16>();
        dht::Blob privateKeyRaw;
        privateKeyRaw.resize(privateKeySize);
        deserializer.extract(&privateKeyRaw[0], privateKeySize);
        identity.first = make_shared<dht::crypto::PrivateKey>(privateKeyRaw);
        
        u16 certificateSize = deserializer.extract<u16>();
        dht::Blob certificateRaw;
        certificateRaw.resize(certificateSize);
        deserializer.extract(&certificateRaw[0], certificateSize);
        identity.second = make_shared<dht::crypto::Certificate>(certificateRaw);
        
        assert(deserializer.empty());
    }
    
    void DatabaseStorage::createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize, u32 offsetToEncryptedData)
    {
        if(storageMap.find(hash) != storageMap.end())
        {
            string errMsg = "Database storage with hash ";
            errMsg += hash.toString();
            errMsg += " already exists";
            throw DatabaseStorageAlreadyExists(errMsg);
        }
        
        addGroup(hash, creatorGroup);
        assert(creatorGroup->getUsers().size() == 1);
        User *creator = (User*)creatorGroup->getUsers()[0];
        addUser(hash, creator);
        
        DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(creator->getPublicKey());
        databaseStorageObjectList->createdTimestamp = timestamp;
        databaseStorageObjectList->groups.push_back(creatorGroup);
        databaseStorageObjectList->data = DataView(new u8[dataSize], dataSize);
        memcpy(databaseStorageObjectList->data.data, data, dataSize);
        databaseStorageObjectList->offsetToEncryptedData = offsetToEncryptedData;
        storageMap[hash] = databaseStorageObjectList;
        
        sibs::SafeSerializer serializer;
        serializer.add(STORAGE_TYPE_CREATE);
        serializer.add(timestamp);
        serializer.add((u8*)creator->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES);
        serializer.add((u8*)creatorGroup->getId().data, GROUP_ID_LENGTH);
        
        serializer.add((u32)dataSize);
        serializer.add(data, dataSize);
        serializer.add(offsetToEncryptedData);
        
        serializer.add((u8*)hash.getData(), HASH_BYTE_SIZE);
        
        fileAppend(dataFilePath, { serializer.getBuffer().data(), serializer.getBuffer().size() });
        
        auto nodeDecryptionKeyIt = nodeDecryptionKeyMap.find(hash);
        if(nodeDecryptionKeyIt != nodeDecryptionKeyMap.end())
            decryptNodeData(hash, databaseStorageObjectList, nodeDecryptionKeyIt->second);
    }
    
    // TODO: Use encryptedDataView to remove duplicate of unsigning request, jumping straight to decrypting encrypted data and calling callback func
    void DatabaseStorage::appendStorage(const Hash &nodeHash, const Hash &dataHash, DatabaseOperation operation, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize, const DataView &encryptedDataView)
    {
        auto it = storageMap.find(nodeHash);
        if(it == storageMap.end())
        {
            string errMsg = "Database storage with hash ";
            errMsg += nodeHash.toString();
            errMsg += " not found. Storage for a hash needs to be created before data can be appended to it";
            throw DatabaseStorageNotFound(errMsg);
        }
        
        auto storeDataHashResult = storedDataHash.find(dataHash);
        if(storeDataHashResult != storedDataHash.end())
        {
            string errMsg = "Database already contains data with hash: ";
            errMsg += dataHash.toString();
            throw DatabaseStorageAlreadyExists(errMsg);
        }
        
        DataView storageData { new u8[dataSize], dataSize };
        memcpy(storageData.data, data, dataSize);
        DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(dataHash, storageData, timestamp, creatorUser->getPublicKey());
        it->second->objects.push_back(databaseStorageObject);
        storedDataHash[dataHash] = databaseStorageObject;
        
        sibs::SafeSerializer serializer;
        serializer.add(STORAGE_TYPE_APPEND);
        serializer.add(timestamp);
        serializer.add((u8*)creatorUser->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES);
        
        serializer.add((u32)dataSize);
        serializer.add(data, dataSize);
        
        serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        serializer.add((u8*)dataHash.getData(), HASH_BYTE_SIZE);
        
        fileAppend(dataFilePath, { serializer.getBuffer().data(), serializer.getBuffer().size() });
        
        auto nodeDecryptionKeyIt = nodeDecryptionKeyMap.find(nodeHash);
        if(nodeDecryptionKeyIt != nodeDecryptionKeyMap.end())
            decryptNodeAppendedData(nodeHash, databaseStorageObject, nodeDecryptionKeyIt->second);
    }
    
    void DatabaseStorage::addToQuarantine(const Hash &dataHash, const Signature::PublicKey &creatorPublicKey, u64 timestamp, const u8 *data, usize dataSize)
    {
        auto storeDataHashResult = storedDataHash.find(dataHash);
        if(storeDataHashResult != storedDataHash.end())
        {
            string errMsg = "Database already contains data with hash: ";
            errMsg += dataHash.toString();
            throw DatabaseStorageAlreadyExists(errMsg);
        }
        
        DataView storageData { new u8[dataSize], dataSize };
        memcpy(storageData.data, data, dataSize);
        DatabaseStorageQuarantineObject *databaseQuarantineStorageObject = new DatabaseStorageQuarantineObject(storageData, timestamp, creatorPublicKey);
        quarantineStorageMap[creatorPublicKey].emplace_back(databaseQuarantineStorageObject);
        // TODO: Add quarantine object to storedDataHash
    }
    
    bool DatabaseStorage::addGroup(const Hash &nodeHash, Group *group)
    {
        DataViewMap<Group*> *groupByIdMap = nullptr;
        auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash);
        if(groupByIdMapIt == nodeGroupByIdMap.end())
        {
            groupByIdMap = new DataViewMap<Group*>();
            nodeGroupByIdMap[nodeHash] = groupByIdMap;
        }
        else
            groupByIdMap = groupByIdMapIt->second;
        
        if(groupByIdMap->find(group->getId()) != groupByIdMap->end())
            return false;
        
        (*groupByIdMap)[group->getId()] = group;
        
        sibs::SafeSerializer serializer;
        
        serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        
        serializer.add((u8)group->getName().size());
        serializer.add((u8*)group->getName().data(), group->getName().size());
        
        serializer.add((u8*)group->getId().data, GROUP_ID_LENGTH);
        
        serializer.add(group->getPermission().getPermissionLevel());
        serializer.add(group->getPermission().getPermissionFlags());
        
        fileAppend(groupsFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size()));

        Log::debug("Created group %s in node %s", bin2hex((const char*)group->getId().data, GROUP_ID_LENGTH).c_str(), nodeHash.toString().c_str());
        
        return true;
    }
    
    bool DatabaseStorage::addUser(const Hash &nodeHash, User *user)
    {
        Signature::MapPublicKey<User*> *publicKeyUserDataMap = nullptr;
        auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash);
        if(publicKeyUserDataMapIt == nodePublicKeyUserDataMap.end())
        {
            publicKeyUserDataMap = new Signature::MapPublicKey<User*>();
            nodePublicKeyUserDataMap[nodeHash] = publicKeyUserDataMap;
        }
        else
            publicKeyUserDataMap = publicKeyUserDataMapIt->second;
        
        if(publicKeyUserDataMap->find(user->getPublicKey()) != publicKeyUserDataMap->end())
            return false;
        
        (*publicKeyUserDataMap)[user->getPublicKey()] = user;
        
        // TODO: Instead of directly saving user to file, maybe user should be added to buffer and then save to file after we have several users (performance improvement)
        
        sibs::SafeSerializer serializer;
        
        serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        
        serializer.add((u8)user->getName().size());
        serializer.add((u8*)user->getName().data(), user->getName().size());
        
        serializer.add((u8*)user->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES);
        
        serializer.add((u8)user->getGroups().size());
        for(Group *group : user->getGroups())
        {
            serializer.add((u8*)group->getId().data, GROUP_ID_LENGTH);
        }
        
        fileAppend(usersFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size()));

        Log::debug("Created user %s in node %s", user->getPublicKey().toString().c_str(), nodeHash.toString().c_str());
        
        return true;
    }
    
    const DatabaseStorageObjectList* DatabaseStorage::getStorage(const Hash &hash) const
    {
        auto it = storageMap.find(hash);
        if(it != storageMap.end())
            return it->second;
        return nullptr;
    }
    
    Group* DatabaseStorage::getGroupById(const Hash &nodeHash, uint8_t groupId[GROUP_ID_LENGTH]) const
    {
        auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash);
        if(groupByIdMapIt != nodeGroupByIdMap.end())
        {
            auto it = groupByIdMapIt->second->find(DataView(groupId, GROUP_ID_LENGTH));
            if(it != groupByIdMapIt->second->end())
                return it->second;
        }
        return nullptr;
    }
    
    User* DatabaseStorage::getUserByPublicKey(const Hash &nodeHash, const Signature::PublicKey &userPublicKey) const
    {
        auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash);
        if(publicKeyUserDataMapIt != nodePublicKeyUserDataMap.end())
        {
            auto it = publicKeyUserDataMapIt->second->find(userPublicKey);
            if(it != publicKeyUserDataMapIt->second->end())
                return it->second;
        }
        return nullptr;
    }
    
    bool DatabaseStorage::storeLocalUser(const string &username, const Signature::KeyPair &keyPair, const string &password)
    {
        auto it = nameLocalUsersMap.find(username);
        if(it != nameLocalUsersMap.end())
            return false;
        
        OwnedMemory hashedPassword = hashPassword(DataView((void*)password.data(), password.size()), DataView((void*)passwordSalt, PASSWORD_SALT_LEN));
        DataView privateKeyView((void*)keyPair.getPrivateKey().getData(), PRIVATE_KEY_NUM_BYTES);
        DataView hashedPasswordView(hashedPassword.data, hashedPassword.size);
        Encryption encryptedPrivateKey(privateKeyView, {}, hashedPasswordView);
        
        EncryptedPrivateKey userEncryptedPrivateKey;
        memcpy(userEncryptedPrivateKey.nonce, encryptedPrivateKey.getNonce().data, ENCRYPTION_NONCE_BYTE_SIZE);
        assert(sizeof(userEncryptedPrivateKey.encryptedPrivateKey) == encryptedPrivateKey.getCipherText().size);
        memcpy(userEncryptedPrivateKey.encryptedPrivateKey, encryptedPrivateKey.getCipherText().data, encryptedPrivateKey.getCipherText().size);
        
        LocalUserEncrypted *localUserEncrypted = LocalUserEncrypted::create(keyPair.getPublicKey(), userEncryptedPrivateKey, username);
        nameLocalUsersMap[username] = localUserEncrypted;
        
        sibs::SafeSerializer serializer;
        serializer.add((u8)username.size());
        serializer.add((const u8*)username.data(), username.size());
        serializer.add((const u8*)keyPair.getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES);
        serializer.add((const u8*)encryptedPrivateKey.getNonce().data, ENCRYPTION_NONCE_BYTE_SIZE);
        serializer.add((const u8*)encryptedPrivateKey.getCipherText().data, ENCRYPTION_CHECKSUM_BYTE_SIZE + PRIVATE_KEY_NUM_BYTES);
        
        fileAppend(localUsersFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size()));
        return true;
    }
    
    Signature::KeyPair DatabaseStorage::decryptLocalEncryptedUser(const string &username, const string &password)
    {
        auto localUserIt = nameLocalUsersMap.find(username);
        if(localUserIt == nameLocalUsersMap.end())
        {
            string errMsg = "User ";
            errMsg += username;
            errMsg += " does not exist in local storage";
            throw DatabaseStorageNoSuchLocalStorageUser(errMsg);
        }
        
        DataView passwordView((void*)password.data(), password.size());
        DataView saltView((void*)passwordSalt, PASSWORD_SALT_LEN);
        try
        {
            auto privateKey = localUserIt->second->getPrivateKey().decrypt(passwordView, saltView);
            Signature::KeyPair keyPair(localUserIt->second->getPublicKey(), privateKey);
            return keyPair;
        }
        catch(DecryptionException &e)
        {
            string errMsg = "Wrong password provided for user ";
            errMsg += username;
            errMsg += " in local storage (";
            errMsg += e.what();
            errMsg += ")";
            throw DatabaseStorageWrongPassword(errMsg);
        }
    }
    
    vector<NodeLocalUser> DatabaseStorage::getLocalNodeUsers(const Signature::KeyPair &keyPair)
    {
        vector<NodeLocalUser> localUsers;
        
        for(auto nodeIt : nodePublicKeyUserDataMap)
        {
            auto userIt = nodeIt.second->find(keyPair.getPublicKey());
            if(userIt != nodeIt.second->end())
            {
                User *user = userIt->second;
                if(user->getType() != User::Type::LOCAL)
                {
                    LocalUser *localUser = LocalUser::create(keyPair, user->getName(), nullptr);
                    for(Group *group : user->getGroups())
                    {
                        localUser->addToGroup(group);
                    }
                    
                    (*nodeIt.second)[keyPair.getPublicKey()] = localUser;
                    localUsers.push_back({ nodeIt.first, localUser });
                    delete user;
                }
                else
                    localUsers.push_back({ nodeIt.first, static_cast<LocalUser*>(user) });
            }
        }
        
        return localUsers;
    }
    
    void DatabaseStorage::setNodeDecryptionKey(const Hash &nodeHash, const DataView &decryptionKeyView)
    {  
        bool nodeHasExistingEncryptionKey = nodeDecryptionKeyMap.find(nodeHash) != nodeDecryptionKeyMap.end();
        
        char *decryptionKeyRaw = new char[decryptionKeyView.size];
        memcpy(decryptionKeyRaw, decryptionKeyView.data, decryptionKeyView.size);
        shared_ptr<OwnedMemory> decryptionKey = make_shared<OwnedMemory>(decryptionKeyRaw, decryptionKeyView.size);
        nodeDecryptionKeyMap[nodeHash] = decryptionKey;
        
        sibs::SafeSerializer serializer;
        serializer.add((const u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        serializer.add((const u8*)decryptionKeyView.data, ENCRYPTION_KEY_BYTE_SIZE);
        fileAppend(nodeDecryptionKeysFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size()));
        
        auto storageIt = storageMap.find(nodeHash);
        if(storageIt == storageMap.end()) return;
        
        // When changing existing encryption key, do not decrypt the existing data as it has already been decrypted,
        // the new key should only be used for new data
        if(!nodeHasExistingEncryptionKey)
            decryptNodeData(nodeHash, storageIt->second, decryptionKey);
    }
    
    bool DatabaseStorage::decryptNodeData(const Hash &nodeHash, DatabaseStorageObjectList *databaseCreateObject, const shared_ptr<OwnedMemory> decryptionKey)
    {
        sibs::SafeDeserializer deserializer((u8*)databaseCreateObject->data.data + databaseCreateObject->offsetToEncryptedData, databaseCreateObject->data.size - databaseCreateObject->offsetToEncryptedData);
        
        u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE];
        deserializer.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE);
        
        DataView dataToDecrypt((void*)deserializer.getBuffer(), deserializer.getSize());
        Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(decryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE));
        sibs::SafeDeserializer bodyDeserializer((const u8*)decryptedBody.getDecryptedText().data, decryptedBody.getDecryptedText().size);
        
        u8 creatorNameLength = bodyDeserializer.extract<u8>();
        string creatorName; // TODO: Add this user name to storage creator name
        creatorName.resize(creatorNameLength);
        bodyDeserializer.extract((u8*)&creatorName[0], creatorNameLength);
        
        u8 nameLength = bodyDeserializer.extract<u8>();
        string name;
        name.resize(nameLength);
        bodyDeserializer.extract((u8*)&name[0], nameLength);
        
        sibs::SafeSerializer serializer;
        serializer.add(DecryptedDataType::STORAGE_CREATE);
        serializer.add((const u8*)nodeHash.getData(), HASH_BYTE_SIZE);
        serializer.add(creatorNameLength);
        serializer.add((const u8*)creatorName.data(), creatorNameLength);
        serializer.add(nameLength);
        serializer.add((const u8*)name.data(), nameLength);
        fileAppend(decryptedDataFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size()));
        
        User *creator = getUserByPublicKey(nodeHash, databaseCreateObject->creatorPublicKey);
        if(!creator)
        {
            Log::error("Creator with public key %s does not exist in node %s", databaseCreateObject->creatorPublicKey.toString().c_str(), nodeHash.toString().c_str());
            return false;
        }
        creator->name = move(creatorName);
        databaseCreateObject->nodeName = name;
        databaseCreateObject->isDecrypted = true;
        
        Log::debug("Deserialized node create data, name: %s", name.c_str());
        const DatabaseCreateNodeRequest createNodeRequest(&nodeHash, databaseCreateObject->createdTimestamp, creator, move(name));
        if(database->onCreateNodeCallbackFunc)
            database->onCreateNodeCallbackFunc(createNodeRequest);
        
        bool success = true;
        for(auto appendObject : databaseCreateObject->objects)
        {
            bool appendObjectResult = decryptNodeAppendedData(nodeHash, appendObject, decryptionKey);
            if(!appendObjectResult)
                success = false;
        }
        return success;
    }
    
    bool DatabaseStorage::decryptNodeAppendedData(const Hash &nodeHash, DatabaseStorageObject *databaseAppendObject, const shared_ptr<OwnedMemory> decryptionKey)
    {
        sibs::SafeDeserializer deserializer((u8*)databaseAppendObject->data.data, databaseAppendObject->data.size);
        char creatorPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
        deserializer.extract((u8*)creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
        Signature::PublicKey creatorPublicKey(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
        
        DataView signedData((void*)deserializer.getBuffer(), deserializer.getSize());
        string unsignedData = creatorPublicKey.unsign(signedData);
        sibs::SafeDeserializer deserializerUnsigned((u8*)unsignedData.data(), unsignedData.size());
        
        u16 packetStructureVersion = deserializerUnsigned.extract<u16>();
        u64 creationDate = deserializerUnsigned.extract<u64>();
        
        DatabaseOperation operation = deserializerUnsigned.extract<DatabaseOperation>();
        
        auto creatorUser = getUserByPublicKey(nodeHash, creatorPublicKey);
        if(!creatorUser)
        {
            Log::error("User with public key %s does not exist in node %s", creatorPublicKey.toString().c_str(), nodeHash.toString().c_str());
            return false;
        }

        if(operation == DatabaseOperation::ADD_DATA)
        {
            if(deserializerUnsigned.getSize() < ENCRYPTION_NONCE_BYTE_SIZE)
            {
                Log::error("Unsigned encrypted body is too small (unable to extract nonce)");
                return false;
            }
#if 0
            if(!creatorUser->isAllowedToPerformAction(PermissionType::ADD_DATA))
            {
                // TODO: User might have permission to perform operation, but we haven't got the packet that adds user to the group with the permission,
                // or we haven't received the packet that modifies group with the permission to perform the operation.
                // This also means that an user can be in a group that has permission to perform the operation and then later be removed from it,
                // and remote peers would accept our request to perform operation if they haven't received the operation that removes the user from the group.
                // How to handle this?
                Log::error("User %s is not allowed to add data to node %s", creatorUser->getName().c_str(), nodeHash.toString().c_str());
                return false;
            }
#endif
            
            u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE];
            deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE);
            DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize());
            Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(decryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE));
            
            sibs::SafeSerializer serializer;
            serializer.add(DecryptedDataType::STORAGE_ADD_DATA);
            serializer.add((const u8*)databaseAppendObject->requestHash.getData(), HASH_BYTE_SIZE);
            serializer.add((u32)decryptedBody.getDecryptedText().size);
            serializer.add((const u8*)decryptedBody.getDecryptedText().data, decryptedBody.getDecryptedText().size);
            fileAppend(decryptedDataFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size()));
            databaseAppendObject->decryptedObject.data.data = new char[decryptedBody.getDecryptedText().size];
            memcpy(databaseAppendObject->decryptedObject.data.data, decryptedBody.getDecryptedText().data, decryptedBody.getDecryptedText().size);
            databaseAppendObject->decryptedObject.data.size = decryptedBody.getDecryptedText().size;
            databaseAppendObject->decryptedObject.operation = operation;
            
            Log::debug("Got add object, timestamp: %zu, data: %.*s", creationDate, decryptedBody.getDecryptedText().size, decryptedBody.getDecryptedText().data);
            const DatabaseAddNodeRequest addNodeRequest(&nodeHash, &databaseAppendObject->requestHash, creationDate, creatorUser, decryptedBody.getDecryptedText());
            if(database->onAddNodeCallbackFunc)
                database->onAddNodeCallbackFunc(addNodeRequest);
        }
        else if(operation == DatabaseOperation::ADD_USER)
        { 
            u8 nameLength = deserializerUnsigned.extract<u8>();
            
            u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE];
            deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE);
            DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), ENCRYPTION_CHECKSUM_BYTE_SIZE + nameLength);
            Decryption decryptedUserName(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(decryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE));

            string username;
            username.resize(nameLength);
            assert(decryptedUserName.getDecryptedText().size == nameLength);
            memcpy(&username[0], decryptedUserName.getDecryptedText().data, nameLength);

            sibs::SafeDeserializer deserializerSkippedEncryptedData(deserializerUnsigned.getBuffer() + ENCRYPTION_CHECKSUM_BYTE_SIZE + nameLength, PUBLIC_KEY_NUM_BYTES + GROUP_ID_LENGTH);
            
            char userToAddPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
            deserializerSkippedEncryptedData.extract((u8*)userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
            Signature::PublicKey userToAddPublicKey(userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
            
            uint8_t groupId[GROUP_ID_LENGTH];
            deserializerSkippedEncryptedData.extract(groupId, GROUP_ID_LENGTH);
            
            auto group = getGroupById(nodeHash, groupId);
            if(!group)
            {
                // TODO: Add to quarantine?
                Log::error("There is no group with id %s in node %s", bin2hex((const char*)groupId, GROUP_ID_LENGTH).c_str(), nodeHash.toString().c_str());
                return false;
            }
            
            auto user = getUserByPublicKey(nodeHash, userToAddPublicKey);
            if(!user)
            {
                Log::error("User to add with public key %s doesn't exist in node %s", userToAddPublicKey.toString().c_str(), nodeHash.toString().c_str());
                return false;
            }
#if 0
            auto creatorUserGroupWithRights = getGroupWithRightsToAddUserToGroup(creatorUser->getGroups(), group);
            if(!creatorUserGroupWithRights)
            {
                // TODO: User might have permission to perform operation, but we haven't got the packet that adds user to the group with the permission,
                // or we haven't received the packet that modifies group with the permission to perform the operation.
                // This also means that an user can be in a group that has permission to perform the operation and then later be removed from it,
                // and remote peers would accept our request to perform operation if they haven't received the operation that removes the user from the group.
                // How to handle this?
                string errMsg = "User ";
                errMsg += creatorUser->getName();
                errMsg += " is not allowed to perform the operation: ";
                errMsg += to_string((u8)operation);
                throw PermissionDeniedException(errMsg);
            }
#endif
            sibs::SafeSerializer serializer;
            serializer.add(DecryptedDataType::STORAGE_ADD_USER);
            serializer.add((const u8*)nodeHash.getData(), HASH_BYTE_SIZE);
            serializer.add((const u8*)databaseAppendObject->requestHash.getData(), HASH_BYTE_SIZE);
            serializer.add((const u8*)userToAddPublicKey.getData(), PUBLIC_KEY_NUM_BYTES);
            serializer.add((u32)decryptedUserName.getDecryptedText().size);
            serializer.add((const u8*)decryptedUserName.getDecryptedText().data, decryptedUserName.getDecryptedText().size);
            fileAppend(decryptedDataFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size()));
            databaseAppendObject->decryptedObject.data.data = new char[decryptedUserName.getDecryptedText().size];
            memcpy(databaseAppendObject->decryptedObject.data.data, decryptedUserName.getDecryptedText().data, decryptedUserName.getDecryptedText().size);
            databaseAppendObject->decryptedObject.data.size = decryptedUserName.getDecryptedText().size;
            databaseAppendObject->decryptedObject.operation = operation;
            
            Log::debug("Got add user object, timestamp: %zu, user added: %.*s", creationDate, nameLength, username.c_str());
            DatabaseAddUserRequest addUserRequest(&nodeHash, &databaseAppendObject->requestHash, creationDate, creatorUser, user, group);
            if(database->onAddUserCallbackFunc)
                database->onAddUserCallbackFunc(addUserRequest);
        }
        else
        {
            Log::error("Got unexpected operation %d", operation);
            return false;
        }
        return true;
    }
    
    const dht::crypto::Identity& DatabaseStorage::getIdentity() const
    {
        return identity;
    }
    
    void DatabaseStorage::update()
    {
        // TODO: Modify this to iterate backwards. Because list is sorted in order of timestamp, we can remove data in range
        auto time = chrono::high_resolution_clock::now().time_since_epoch();
        auto timeMicroseconds = chrono::duration_cast<chrono::microseconds>(time).count();
        for(auto mapIt = quarantineStorageMap.begin(); mapIt != quarantineStorageMap.end(); )
        {
            for(auto vecIt = mapIt->second.begin(); vecIt != mapIt->second.end(); )
            {
                if(timeMicroseconds - (*vecIt)->storedTimestamp > QUARANTINE_STORAGE_TIME_MICROSECONDS)
                    vecIt = mapIt->second.erase(vecIt);
                else
                    ++vecIt;
            }
            
            if(mapIt->second.empty())
                mapIt = quarantineStorageMap.erase(mapIt);
            else
                ++mapIt;
        }
    }
}