diff options
Diffstat (limited to 'src/DatabaseStorage.cpp')
| -rw-r--r-- | src/DatabaseStorage.cpp | 442 |
1 files changed, 420 insertions, 22 deletions
diff --git a/src/DatabaseStorage.cpp b/src/DatabaseStorage.cpp index 212ca0f..bd98b8b 100644 --- a/src/DatabaseStorage.cpp +++ b/src/DatabaseStorage.cpp @@ -1,13 +1,28 @@ #include "../include/odhtdb/DatabaseStorage.hpp" -#include "../include/odhtdb/User.hpp" +#include "../include/odhtdb/RemoteUser.hpp" +#include "../include/odhtdb/LocalUser.hpp" +#include "../include/odhtdb/LocalUserEncrypted.hpp" #include "../include/odhtdb/Group.hpp" +#include "../include/odhtdb/FileUtils.hpp" +#include "../include/odhtdb/bin2hex.hpp" +#include "../include/odhtdb/PasswordHash.hpp" +#include "../include/odhtdb/Log.hpp" #include <cstring> #include <chrono> +#include <boost/filesystem/convenience.hpp> +#include <sibs/SafeSerializer.hpp> +#include <sodium/randombytes.h> using namespace std; namespace odhtdb { + enum StorageType : u8 + { + STORAGE_TYPE_CREATE, + STORAGE_TYPE_APPEND + }; + const u64 QUARANTINE_STORAGE_TIME_MICROSECONDS = 60 * 1.0e6; DatabaseStorageObject::DatabaseStorageObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : @@ -23,6 +38,263 @@ namespace odhtdb storedTimestamp = chrono::duration_cast<chrono::microseconds>(time).count(); } + DatabaseStorage::DatabaseStorage(const boost::filesystem::path &storagePath) : + groupsFilePath(storagePath / "groups"), + usersFilePath(storagePath / "users"), + dataFilePath(storagePath / "data"), + metadataFilePath(storagePath / "metadata") + { + boost::filesystem::create_directories(storagePath); + + bool metadataLoaded = false; + try + { + loadMetadataFromFile(); + metadataLoaded = true; + loadGroupsFromFile(); + loadUsersFromFile(); + loadDataFromFile(); + //loadQuarantineFromFile(); + } + catch(FileException &e) + { + Log::warn("Failed to load data from file: %s, reason: %s. Ignoring...", dataFilePath.string().c_str(), e.what()); + if(!metadataLoaded) + { + sibs::SafeSerializer metadataSerializer; + metadataSerializer.add((u16)0); // Storage version + randombytes_buf(passwordSalt, PASSWORD_SALT_LEN); + metadataSerializer.add(passwordSalt, PASSWORD_SALT_LEN); + fileAppend(metadataFilePath, { metadataSerializer.getBuffer().data(), metadataSerializer.getBuffer().size() }); + } + } + } + + void DatabaseStorage::loadGroupsFromFile() + { + if(!boost::filesystem::exists(groupsFilePath)) return; + + OwnedMemory groupsFileContent = fileGetContent(groupsFilePath); + sibs::SafeDeserializer deserializer((u8*)groupsFileContent.data, groupsFileContent.size); + + while(!deserializer.empty()) + { + Hash nodeHash; + deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); + + DataViewMap<Group*> *groupByIdMap = nullptr; + auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash); + if(groupByIdMapIt == nodeGroupByIdMap.end()) + { + groupByIdMap = new DataViewMap<Group*>(); + nodeGroupByIdMap[nodeHash] = groupByIdMap; + } + else + groupByIdMap = groupByIdMapIt->second; + + u8 groupNameSize = deserializer.extract<u8>(); + string groupName; + groupName.resize(groupNameSize); + deserializer.extract((u8*)&groupName[0], groupNameSize); + + u8 groupId[GROUP_ID_LENGTH]; + deserializer.extract(groupId, GROUP_ID_LENGTH); + + u8 permissionLevel = deserializer.extract<u8>(); + u32 permissionFlags = deserializer.extract<u32>(); + + Group *group = new Group(groupName, groupId, { permissionLevel, permissionFlags }); + (*groupByIdMap)[group->getId()] = group; + + string groupIdStr = bin2hex((const char*)groupId, GROUP_ID_LENGTH); + Log::debug("DatabaseStorage: Loaded group from file: %s", groupIdStr.c_str()); + } + } + + void DatabaseStorage::loadUsersFromFile() + { + if(!boost::filesystem::exists(usersFilePath)) return; + + OwnedMemory usersFileContent = fileGetContent(usersFilePath); + sibs::SafeDeserializer deserializer((u8*)usersFileContent.data, usersFileContent.size); + + while(!deserializer.empty()) + { + Hash nodeHash; + deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); + + Signature::MapPublicKey<UserData*> *publicKeyUserDataMap = nullptr; + auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash); + if(publicKeyUserDataMapIt == nodePublicKeyUserDataMap.end()) + { + publicKeyUserDataMap = new Signature::MapPublicKey<UserData*>(); + nodePublicKeyUserDataMap[nodeHash] = publicKeyUserDataMap; + } + else + publicKeyUserDataMap = publicKeyUserDataMapIt->second; + + DataViewMap<Group*> *groupByIdMap = nullptr; + auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash); + if(groupByIdMapIt == nodeGroupByIdMap.end()) + { + groupByIdMap = new DataViewMap<Group*>(); + nodeGroupByIdMap[nodeHash] = groupByIdMap; + } + else + groupByIdMap = groupByIdMapIt->second; + + User::Type userType = deserializer.extract<User::Type>(); + + u8 usernameSize = deserializer.extract<u8>(); + string username; + username.resize(usernameSize); + deserializer.extract((u8*)&username[0], usernameSize); + + u8 userPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; + deserializer.extract(userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + Signature::PublicKey userPublicKey((const char*)userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + + UserData *userData = new UserData(); + if(userType == User::Type::LOCAL) + { + EncryptedPrivateKey encryptedPrivateKey; + deserializer.extract(encryptedPrivateKey.nonce, ENCRYPTION_NONCE_BYTE_SIZE); + deserializer.extract(encryptedPrivateKey.encryptedPrivateKey, 16 + PRIVATE_KEY_NUM_BYTES); + userData->user = LocalUserEncrypted::create(userPublicKey, encryptedPrivateKey, username, nullptr); + } + else + { + userData->user = RemoteUser::create(userPublicKey, username, nullptr); + } + + u8 numGroups = deserializer.extract<u8>(); + for(int i = 0; i < numGroups; ++i) + { + u8 groupId[GROUP_ID_LENGTH]; + deserializer.extract(groupId, GROUP_ID_LENGTH); + + auto groupIt = groupByIdMap->find(DataView(groupId, GROUP_ID_LENGTH)); + if(groupIt == groupByIdMap->end()) + { + string errMsg = "User group with id "; + errMsg += bin2hex((const char*)groupId, GROUP_ID_LENGTH); + errMsg += " does not exist"; + throw DatabaseStorageCorrupt(errMsg); + } + userData->user->addToGroup(groupIt->second); + } + + (*publicKeyUserDataMap)[userData->user->getPublicKey()] = userData; + } + } + + void DatabaseStorage::loadStorageCreate(sibs::SafeDeserializer &deserializer) + { + u64 timestamp = deserializer.extract<u64>(); + + u8 groupId[GROUP_ID_LENGTH]; + deserializer.extract(groupId, GROUP_ID_LENGTH); + + u32 dataSize = deserializer.extract<u32>(); + u8 *data = new u8[dataSize]; + deserializer.extract(data, dataSize); + + Hash nodeHash; + deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); + + auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash); + if(groupByIdMapIt == nodeGroupByIdMap.end()) + { + string errMsg = "No groups exists in node "; + errMsg += nodeHash.toString(); + throw DatabaseStorageCorrupt(errMsg); + } + + auto groupIt = groupByIdMapIt->second->find(DataView(groupId, GROUP_ID_LENGTH)); + if(groupIt == groupByIdMapIt->second->end()) + { + string errMsg = "User group with id "; + errMsg += bin2hex((const char*)groupId, GROUP_ID_LENGTH); + errMsg += " does not exist in node "; + errMsg += nodeHash.toString(); + throw DatabaseStorageCorrupt(errMsg); + } + + DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(); + databaseStorageObjectList->createdTimestamp = timestamp; + databaseStorageObjectList->groups.push_back(groupIt->second); + databaseStorageObjectList->data = DataView(data, dataSize); + storageMap[nodeHash] = databaseStorageObjectList; + } + + void DatabaseStorage::loadStorageAppend(sibs::SafeDeserializer &deserializer) + { + u64 timestamp = deserializer.extract<u64>(); + + u8 creatorPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; + deserializer.extract(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + Signature::PublicKey creatorPublicKey((const char*)creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + + u32 dataSize = deserializer.extract<u32>(); + u8 *data = new u8[dataSize]; + deserializer.extract(data, dataSize); + + Hash nodeHash; + deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); + + Hash dataHash; + deserializer.extract((u8*)dataHash.getData(), HASH_BYTE_SIZE); + storedDataHash.insert(dataHash); + + auto storageIt = storageMap.find(nodeHash); + if(storageIt == storageMap.end()) + { + string errMsg = "Database storage with hash "; + errMsg += nodeHash.toString(); + errMsg += " not found"; + throw DatabaseStorageCorrupt(errMsg); + } + + DataView storageData { data, dataSize }; + DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(storageData, timestamp, creatorPublicKey); + storageIt->second->objects.push_back(databaseStorageObject); + } + + void DatabaseStorage::loadDataFromFile() + { + if(!boost::filesystem::exists(dataFilePath)) return; + + OwnedMemory dataFileContent = fileGetContent(dataFilePath); + sibs::SafeDeserializer deserializer((u8*)dataFileContent.data, dataFileContent.size); + + while(!deserializer.empty()) + { + StorageType storageType = deserializer.extract<StorageType>(); + switch(storageType) + { + case STORAGE_TYPE_CREATE: + loadStorageCreate(deserializer); + break; + case STORAGE_TYPE_APPEND: + loadStorageAppend(deserializer); + break; + } + } + } + + void DatabaseStorage::loadMetadataFromFile() + { + OwnedMemory metadataFileContent = fileGetContent(metadataFilePath); + sibs::SafeDeserializer deserializer((u8*)metadataFileContent.data, metadataFileContent.size); + + u16 storageVersion = deserializer.extract<u16>(); + + u8 passwordSalt[PASSWORD_SALT_LEN]; + deserializer.extract(passwordSalt, PASSWORD_SALT_LEN); + + assert(deserializer.empty()); + } + void DatabaseStorage::createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize) { if(storageMap.find(hash) != storageMap.end()) @@ -33,18 +305,30 @@ namespace odhtdb throw DatabaseStorageAlreadyExists(errMsg); } + addGroup(hash, creatorGroup); + for(auto user : creatorGroup->getUsers()) + { + addUser(hash, (User*)user); + } + DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(); databaseStorageObjectList->createdTimestamp = timestamp; databaseStorageObjectList->groups.push_back(creatorGroup); databaseStorageObjectList->data = DataView(new u8[dataSize], dataSize); memcpy(databaseStorageObjectList->data.data, data, dataSize); storageMap[hash] = databaseStorageObjectList; - groupByIdMap[creatorGroup->getId()] = creatorGroup; - for(auto user : creatorGroup->getUsers()) - { - addUser((User*)user, hash); - } + sibs::SafeSerializer serializer; + serializer.add(STORAGE_TYPE_CREATE); + serializer.add(timestamp); + serializer.add((u8*)creatorGroup->getId().data, GROUP_ID_LENGTH); + + serializer.add((u32)dataSize); + serializer.add(data, dataSize); + + serializer.add((u8*)hash.getData(), HASH_BYTE_SIZE); + + fileAppend(dataFilePath, { serializer.getBuffer().data(), serializer.getBuffer().size() }); } void DatabaseStorage::appendStorage(const Hash &nodeHash, const Hash &dataHash, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize) @@ -70,6 +354,19 @@ namespace odhtdb memcpy(storageData.data, data, dataSize); DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(storageData, timestamp, creatorUser->getPublicKey()); it->second->objects.push_back(databaseStorageObject); + + sibs::SafeSerializer serializer; + serializer.add(STORAGE_TYPE_APPEND); + serializer.add(timestamp); + serializer.add((u8*)creatorUser->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES); + + serializer.add((u32)dataSize); + serializer.add(data, dataSize); + + serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE); + serializer.add((u8*)dataHash.getData(), HASH_BYTE_SIZE); + + fileAppend(dataFilePath, { serializer.getBuffer().data(), serializer.getBuffer().size() }); } void DatabaseStorage::addToQuarantine(const Hash &dataHash, const Signature::PublicKey &creatorPublicKey, u64 timestamp, const u8 *data, usize dataSize) @@ -88,10 +385,103 @@ namespace odhtdb quarantineStorageMap[creatorPublicKey].emplace_back(databaseQuarantineStorageObject); } - void DatabaseStorage::addUser(User *user, const Hash &hash) + bool DatabaseStorage::addGroup(const Hash &nodeHash, Group *group) + { + DataViewMap<Group*> *groupByIdMap = nullptr; + auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash); + if(groupByIdMapIt == nodeGroupByIdMap.end()) + { + groupByIdMap = new DataViewMap<Group*>(); + nodeGroupByIdMap[nodeHash] = groupByIdMap; + } + else + groupByIdMap = groupByIdMapIt->second; + + if(groupByIdMap->find(group->getId()) != groupByIdMap->end()) + return false; + + (*groupByIdMap)[group->getId()] = group; + + sibs::SafeSerializer serializer; + + serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE); + + serializer.add((u8)group->getName().size()); + serializer.add((u8*)group->getName().data(), group->getName().size()); + + serializer.add((u8*)group->getId().data, GROUP_ID_LENGTH); + + serializer.add(group->getPermission().getPermissionLevel()); + serializer.add(group->getPermission().getPermissionFlags()); + + fileAppend(groupsFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); + + return true; + } + + bool DatabaseStorage::addUser(const Hash &nodeHash, User *user) { - userPublicKeyNodeMap[user->getPublicKey()] = new Hash(hash); - publicKeyUserMap[user->getPublicKey()] = user; + Signature::MapPublicKey<UserData*> *publicKeyUserDataMap = nullptr; + auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash); + if(publicKeyUserDataMapIt == nodePublicKeyUserDataMap.end()) + { + publicKeyUserDataMap = new Signature::MapPublicKey<UserData*>(); + nodePublicKeyUserDataMap[nodeHash] = publicKeyUserDataMap; + } + else + publicKeyUserDataMap = publicKeyUserDataMapIt->second; + + if(publicKeyUserDataMap->find(user->getPublicKey()) != publicKeyUserDataMap->end()) + return false; + + UserData *userData = new UserData(); + userData->user = user; + if(user->getType() == User::Type::LOCAL) + { + LocalUser *localUser = static_cast<LocalUser*>(user); + //DataView plainPassword((void*)localUser->getPlainPassword().data(), localUser->getPlainPassword().size()); + OwnedMemory hashedPassword = hashPassword(DataView((void*)localUser->getPlainPassword().data(), localUser->getPlainPassword().size()), DataView(passwordSalt, PASSWORD_SALT_LEN)); + memcpy(userData->hashedPassword, hashedPassword.data, hashedPassword.size); + } + else + { + memset(userData->hashedPassword, 0, HASHED_PASSWORD_LEN); + } + (*publicKeyUserDataMap)[user->getPublicKey()] = userData; + + // TODO: Instead of directly saving user to file, maybe user should be added to buffer and then save to file after we have several users (performance improvement) + + sibs::SafeSerializer serializer; + + serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE); + + serializer.add(user->getType()); + + serializer.add((u8)user->getName().size()); + serializer.add((u8*)user->getName().data(), user->getName().size()); + + serializer.add((u8*)user->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES); + if(user->getType() == User::Type::LOCAL) + { + LocalUser *localUser = static_cast<LocalUser*>(user); + static_assert(HASHED_PASSWORD_LEN == ENCRYPTION_KEY_BYTE_SIZE, "Encryption key size has changed but hashed password size hasn't"); + Encryption encryptedPrivateKey(DataView((void*)localUser->getPrivateKey().getData(), localUser->getPrivateKey().getSize()), + DataView(), + DataView(userData->hashedPassword, HASHED_PASSWORD_LEN)); + serializer.add((u8*)encryptedPrivateKey.getNonce().data, ENCRYPTION_NONCE_BYTE_SIZE); + assert(16 + PRIVATE_KEY_NUM_BYTES == encryptedPrivateKey.getCipherText().size); + serializer.add((u8*)encryptedPrivateKey.getCipherText().data, encryptedPrivateKey.getCipherText().size); + } + + serializer.add((u8)user->getGroups().size()); + for(Group *group : user->getGroups()) + { + serializer.add((u8*)group->getId().data, GROUP_ID_LENGTH); + } + + fileAppend(usersFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); + + return true; } const DatabaseStorageObjectList* DatabaseStorage::getStorage(const Hash &hash) const @@ -102,33 +492,41 @@ namespace odhtdb return nullptr; } - const Hash* DatabaseStorage::getNodeByUserPublicKey(const Signature::PublicKey &userPublicKey) const + Group* DatabaseStorage::getGroupById(const Hash &nodeHash, uint8_t groupId[GROUP_ID_LENGTH]) { - auto it = userPublicKeyNodeMap.find(userPublicKey); - if(it != userPublicKeyNodeMap.end()) - return it->second; + auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash); + if(groupByIdMapIt != nodeGroupByIdMap.end()) + { + auto it = groupByIdMapIt->second->find(DataView(groupId, GROUP_ID_LENGTH)); + if(it != groupByIdMapIt->second->end()) + return it->second; + } return nullptr; } - // Returns nullptr if no user with public key exists - const User* DatabaseStorage::getUserByPublicKey(const Signature::PublicKey &userPublicKey) const + User* DatabaseStorage::getUserByPublicKey(const Hash &nodeHash, Signature::PublicKey &userPublicKey) { - auto it = publicKeyUserMap.find(userPublicKey); - if(it != publicKeyUserMap.end()) - return it->second; + auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash); + if(publicKeyUserDataMapIt != nodePublicKeyUserDataMap.end()) + { + auto it = publicKeyUserDataMapIt->second->find(userPublicKey); + if(it != publicKeyUserDataMapIt->second->end()) + return it->second->user; + } return nullptr; } - Group* DatabaseStorage::getGroupById(uint8_t groupId[16]) + const Signature::MapPublicKey<DatabaseStorage::UserData*>* DatabaseStorage::getUsersData(const Hash &nodeHash) const { - auto it = groupByIdMap.find(DataView(groupId, 16)); - if(it != groupByIdMap.end()) - return it->second; + auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash); + if(publicKeyUserDataMapIt != nodePublicKeyUserDataMap.end()) + return publicKeyUserDataMapIt->second; return nullptr; } void DatabaseStorage::update() { + // TODO: Modify this to iterate backwards. Because list is sorted in order of timestamp, we can remove data in range auto time = chrono::high_resolution_clock::now().time_since_epoch(); auto timeMicroseconds = chrono::duration_cast<chrono::microseconds>(time).count(); for(auto mapIt = quarantineStorageMap.begin(); mapIt != quarantineStorageMap.end(); ) |