#include "../include/odhtdb/DatabaseStorage.hpp" #include "../include/odhtdb/RemoteUser.hpp" #include "../include/odhtdb/LocalUser.hpp" #include "../include/odhtdb/LocalUserEncrypted.hpp" #include "../include/odhtdb/Group.hpp" #include "../include/odhtdb/FileUtils.hpp" #include "../include/odhtdb/bin2hex.hpp" #include "../include/odhtdb/PasswordHash.hpp" #include "../include/odhtdb/Log.hpp" #include #include #include #include #include using namespace std; namespace odhtdb { enum StorageType : u8 { STORAGE_TYPE_CREATE, STORAGE_TYPE_APPEND }; const u64 QUARANTINE_STORAGE_TIME_MICROSECONDS = 60 * 1.0e6; const u16 STORAGE_VERSION = 1; DatabaseStorageObject::DatabaseStorageObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : data(_data), createdTimestamp(_timestamp), creatorPublicKey(_creatorPublicKey) { } DatabaseStorageQuarantineObject::DatabaseStorageQuarantineObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : data(_data), createdTimestamp(_timestamp), creatorPublicKey(_creatorPublicKey) { auto time = chrono::high_resolution_clock::now().time_since_epoch(); storedTimestamp = chrono::duration_cast(time).count(); } DatabaseStorage::DatabaseStorage(const boost::filesystem::path &storagePath) : groupsFilePath(storagePath / "groups"), usersFilePath(storagePath / "users"), dataFilePath(storagePath / "data"), metadataFilePath(storagePath / "metadata"), localUsersFilePath(storagePath / "local_users") { boost::filesystem::create_directories(storagePath); bool metadataLoaded = false; try { loadMetadataFromFile(); metadataLoaded = true; loadGroupsFromFile(); loadUsersFromFile(); loadDataFromFile(); loadLocalUsersFromFile(); //loadQuarantineFromFile(); } catch(FileException &e) { Log::warn("Failed to load storage data, reason: %s. Ignoring...", e.what()); if(!metadataLoaded) { sibs::SafeSerializer metadataSerializer; metadataSerializer.add(STORAGE_VERSION); randombytes_buf(passwordSalt, PASSWORD_SALT_LEN); metadataSerializer.add(passwordSalt, PASSWORD_SALT_LEN); //string passwordSaltStr((const char*)passwordSalt, PASSWORD_SALT_LEN); identity = dht::crypto::generateIdentity(); dht::Blob privateKeyData = identity.first->serialize(); metadataSerializer.add((u16)privateKeyData.size()); metadataSerializer.add(privateKeyData.data(), privateKeyData.size()); dht::Blob certificateData; identity.second->pack(certificateData); metadataSerializer.add((u16)certificateData.size()); metadataSerializer.add(certificateData.data(), certificateData.size()); fileAppend(metadataFilePath, { metadataSerializer.getBuffer().data(), metadataSerializer.getBuffer().size() }); } } } void DatabaseStorage::loadGroupsFromFile() { if(!boost::filesystem::exists(groupsFilePath)) return; OwnedMemory groupsFileContent = fileGetContent(groupsFilePath); sibs::SafeDeserializer deserializer((u8*)groupsFileContent.data, groupsFileContent.size); while(!deserializer.empty()) { Hash nodeHash; deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); DataViewMap *groupByIdMap = nullptr; auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash); if(groupByIdMapIt == nodeGroupByIdMap.end()) { groupByIdMap = new DataViewMap(); nodeGroupByIdMap[nodeHash] = groupByIdMap; } else groupByIdMap = groupByIdMapIt->second; u8 groupNameSize = deserializer.extract(); string groupName; groupName.resize(groupNameSize); deserializer.extract((u8*)&groupName[0], groupNameSize); u8 groupId[GROUP_ID_LENGTH]; deserializer.extract(groupId, GROUP_ID_LENGTH); u8 permissionLevel = deserializer.extract(); u32 permissionFlags = deserializer.extract(); Group *group = new Group(groupName, groupId, { permissionLevel, permissionFlags }); (*groupByIdMap)[group->getId()] = group; string groupIdStr = bin2hex((const char*)groupId, GROUP_ID_LENGTH); Log::debug("DatabaseStorage: Loaded group from file: %s", groupIdStr.c_str()); } } void DatabaseStorage::loadUsersFromFile() { if(!boost::filesystem::exists(usersFilePath)) return; OwnedMemory usersFileContent = fileGetContent(usersFilePath); sibs::SafeDeserializer deserializer((u8*)usersFileContent.data, usersFileContent.size); while(!deserializer.empty()) { Hash nodeHash; deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); Signature::MapPublicKey *publicKeyUserDataMap = nullptr; auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash); if(publicKeyUserDataMapIt == nodePublicKeyUserDataMap.end()) { publicKeyUserDataMap = new Signature::MapPublicKey(); nodePublicKeyUserDataMap[nodeHash] = publicKeyUserDataMap; } else publicKeyUserDataMap = publicKeyUserDataMapIt->second; DataViewMap *groupByIdMap = nullptr; auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash); if(groupByIdMapIt == nodeGroupByIdMap.end()) { groupByIdMap = new DataViewMap(); nodeGroupByIdMap[nodeHash] = groupByIdMap; } else groupByIdMap = groupByIdMapIt->second; u8 usernameSize = deserializer.extract(); string username; username.resize(usernameSize); deserializer.extract((u8*)&username[0], usernameSize); u8 userPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; deserializer.extract(userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); Signature::PublicKey userPublicKey((const char*)userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); User *user = RemoteUser::create(userPublicKey, username, nullptr); u8 numGroups = deserializer.extract(); for(int i = 0; i < numGroups; ++i) { u8 groupId[GROUP_ID_LENGTH]; deserializer.extract(groupId, GROUP_ID_LENGTH); auto groupIt = groupByIdMap->find(DataView(groupId, GROUP_ID_LENGTH)); if(groupIt == groupByIdMap->end()) { string errMsg = "User group with id "; errMsg += bin2hex((const char*)groupId, GROUP_ID_LENGTH); errMsg += " does not exist"; throw DatabaseStorageCorrupt(errMsg); } user->addToGroup(groupIt->second); } (*publicKeyUserDataMap)[user->getPublicKey()] = user; } } void DatabaseStorage::loadStorageCreate(sibs::SafeDeserializer &deserializer) { u64 timestamp = deserializer.extract(); u8 groupId[GROUP_ID_LENGTH]; deserializer.extract(groupId, GROUP_ID_LENGTH); u32 dataSize = deserializer.extract(); u8 *data = new u8[dataSize]; deserializer.extract(data, dataSize); Hash nodeHash; deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash); if(groupByIdMapIt == nodeGroupByIdMap.end()) { string errMsg = "No groups exists in node "; errMsg += nodeHash.toString(); throw DatabaseStorageCorrupt(errMsg); } auto groupIt = groupByIdMapIt->second->find(DataView(groupId, GROUP_ID_LENGTH)); if(groupIt == groupByIdMapIt->second->end()) { string errMsg = "User group with id "; errMsg += bin2hex((const char*)groupId, GROUP_ID_LENGTH); errMsg += " does not exist in node "; errMsg += nodeHash.toString(); throw DatabaseStorageCorrupt(errMsg); } DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(); databaseStorageObjectList->createdTimestamp = timestamp; databaseStorageObjectList->groups.push_back(groupIt->second); databaseStorageObjectList->data = DataView(data, dataSize); storageMap[nodeHash] = databaseStorageObjectList; } void DatabaseStorage::loadStorageAppend(sibs::SafeDeserializer &deserializer) { u64 timestamp = deserializer.extract(); u8 creatorPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; deserializer.extract(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); Signature::PublicKey creatorPublicKey((const char*)creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); u32 dataSize = deserializer.extract(); u8 *data = new u8[dataSize]; deserializer.extract(data, dataSize); Hash nodeHash; deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); Hash dataHash; deserializer.extract((u8*)dataHash.getData(), HASH_BYTE_SIZE); storedDataHash.insert(dataHash); auto storageIt = storageMap.find(nodeHash); if(storageIt == storageMap.end()) { string errMsg = "Database storage with hash "; errMsg += nodeHash.toString(); errMsg += " not found"; throw DatabaseStorageCorrupt(errMsg); } DataView storageData { data, dataSize }; DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(storageData, timestamp, creatorPublicKey); storageIt->second->objects.push_back(databaseStorageObject); } void DatabaseStorage::loadDataFromFile() { if(!boost::filesystem::exists(dataFilePath)) return; OwnedMemory dataFileContent = fileGetContent(dataFilePath); sibs::SafeDeserializer deserializer((u8*)dataFileContent.data, dataFileContent.size); while(!deserializer.empty()) { StorageType storageType = deserializer.extract(); switch(storageType) { case STORAGE_TYPE_CREATE: loadStorageCreate(deserializer); break; case STORAGE_TYPE_APPEND: loadStorageAppend(deserializer); break; } } } void DatabaseStorage::loadLocalUsersFromFile() { if(!boost::filesystem::exists(localUsersFilePath)) return; OwnedMemory localUsersFileContent = fileGetContent(localUsersFilePath); sibs::SafeDeserializer deserializer((u8*)localUsersFileContent.data, localUsersFileContent.size); while(!deserializer.empty()) { u8 usernameSize = deserializer.extract(); string username; username.resize(usernameSize); deserializer.extract((u8*)&username[0], usernameSize); u8 userPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; deserializer.extract(userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); Signature::PublicKey userPublicKey((const char*)userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); EncryptedPrivateKey encryptedPrivateKey; deserializer.extract(encryptedPrivateKey.nonce, ENCRYPTION_NONCE_BYTE_SIZE); deserializer.extract(encryptedPrivateKey.encryptedPrivateKey, ENCRYPTION_CHECKSUM_BYTE_SIZE + PRIVATE_KEY_NUM_BYTES); nameLocalUsersMap[username] = LocalUserEncrypted::create(userPublicKey, encryptedPrivateKey, username); } } void DatabaseStorage::loadMetadataFromFile() { OwnedMemory metadataFileContent = fileGetContent(metadataFilePath); sibs::SafeDeserializer deserializer((u8*)metadataFileContent.data, metadataFileContent.size); u16 storageVersion = deserializer.extract(); if(storageVersion != STORAGE_VERSION) throw std::runtime_error("Wrong storage version!"); u8 passwordSalt[PASSWORD_SALT_LEN]; deserializer.extract(passwordSalt, PASSWORD_SALT_LEN); //string passwordSaltStr((const char*)passwordSalt, PASSWORD_SALT_LEN); u16 privateKeySize = deserializer.extract(); dht::Blob privateKeyRaw; privateKeyRaw.resize(privateKeySize); deserializer.extract(&privateKeyRaw[0], privateKeySize); identity.first = make_shared(privateKeyRaw); u16 certificateSize = deserializer.extract(); dht::Blob certificateRaw; certificateRaw.resize(certificateSize); deserializer.extract(&certificateRaw[0], certificateSize); identity.second = make_shared(certificateRaw); assert(deserializer.empty()); } void DatabaseStorage::createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize) { if(storageMap.find(hash) != storageMap.end()) { string errMsg = "Database storage with hash "; errMsg += hash.toString(); errMsg += " already exists"; throw DatabaseStorageAlreadyExists(errMsg); } addGroup(hash, creatorGroup); for(auto user : creatorGroup->getUsers()) { addUser(hash, (User*)user); } DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(); databaseStorageObjectList->createdTimestamp = timestamp; databaseStorageObjectList->groups.push_back(creatorGroup); databaseStorageObjectList->data = DataView(new u8[dataSize], dataSize); memcpy(databaseStorageObjectList->data.data, data, dataSize); storageMap[hash] = databaseStorageObjectList; sibs::SafeSerializer serializer; serializer.add(STORAGE_TYPE_CREATE); serializer.add(timestamp); serializer.add((u8*)creatorGroup->getId().data, GROUP_ID_LENGTH); serializer.add((u32)dataSize); serializer.add(data, dataSize); serializer.add((u8*)hash.getData(), HASH_BYTE_SIZE); fileAppend(dataFilePath, { serializer.getBuffer().data(), serializer.getBuffer().size() }); } void DatabaseStorage::appendStorage(const Hash &nodeHash, const Hash &dataHash, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize) { auto it = storageMap.find(nodeHash); if(it == storageMap.end()) { string errMsg = "Database storage with hash "; errMsg += nodeHash.toString(); errMsg += " not found. Storage for a hash needs to be created before data can be appended to it"; throw DatabaseStorageNotFound(errMsg); } auto storeDataHashResult = storedDataHash.insert(dataHash); if(!storeDataHashResult.second) { string errMsg = "Database already contains data with hash: "; errMsg += dataHash.toString(); throw DatabaseStorageAlreadyExists(errMsg); } DataView storageData { new u8[dataSize], dataSize }; memcpy(storageData.data, data, dataSize); DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(storageData, timestamp, creatorUser->getPublicKey()); it->second->objects.push_back(databaseStorageObject); sibs::SafeSerializer serializer; serializer.add(STORAGE_TYPE_APPEND); serializer.add(timestamp); serializer.add((u8*)creatorUser->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES); serializer.add((u32)dataSize); serializer.add(data, dataSize); serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE); serializer.add((u8*)dataHash.getData(), HASH_BYTE_SIZE); fileAppend(dataFilePath, { serializer.getBuffer().data(), serializer.getBuffer().size() }); } void DatabaseStorage::addToQuarantine(const Hash &dataHash, const Signature::PublicKey &creatorPublicKey, u64 timestamp, const u8 *data, usize dataSize) { auto storeDataHashResult = storedDataHash.insert(dataHash); if(!storeDataHashResult.second) { string errMsg = "Database already contains data with hash: "; errMsg += dataHash.toString(); throw DatabaseStorageAlreadyExists(errMsg); } DataView storageData { new u8[dataSize], dataSize }; memcpy(storageData.data, data, dataSize); DatabaseStorageQuarantineObject *databaseQuarantineStorageObject = new DatabaseStorageQuarantineObject(storageData, timestamp, creatorPublicKey); quarantineStorageMap[creatorPublicKey].emplace_back(databaseQuarantineStorageObject); } bool DatabaseStorage::addGroup(const Hash &nodeHash, Group *group) { DataViewMap *groupByIdMap = nullptr; auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash); if(groupByIdMapIt == nodeGroupByIdMap.end()) { groupByIdMap = new DataViewMap(); nodeGroupByIdMap[nodeHash] = groupByIdMap; } else groupByIdMap = groupByIdMapIt->second; if(groupByIdMap->find(group->getId()) != groupByIdMap->end()) return false; (*groupByIdMap)[group->getId()] = group; sibs::SafeSerializer serializer; serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE); serializer.add((u8)group->getName().size()); serializer.add((u8*)group->getName().data(), group->getName().size()); serializer.add((u8*)group->getId().data, GROUP_ID_LENGTH); serializer.add(group->getPermission().getPermissionLevel()); serializer.add(group->getPermission().getPermissionFlags()); fileAppend(groupsFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); return true; } bool DatabaseStorage::addUser(const Hash &nodeHash, User *user) { Signature::MapPublicKey *publicKeyUserDataMap = nullptr; auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash); if(publicKeyUserDataMapIt == nodePublicKeyUserDataMap.end()) { publicKeyUserDataMap = new Signature::MapPublicKey(); nodePublicKeyUserDataMap[nodeHash] = publicKeyUserDataMap; } else publicKeyUserDataMap = publicKeyUserDataMapIt->second; if(publicKeyUserDataMap->find(user->getPublicKey()) != publicKeyUserDataMap->end()) return false; (*publicKeyUserDataMap)[user->getPublicKey()] = user; // TODO: Instead of directly saving user to file, maybe user should be added to buffer and then save to file after we have several users (performance improvement) sibs::SafeSerializer serializer; serializer.add((u8*)nodeHash.getData(), HASH_BYTE_SIZE); serializer.add((u8)user->getName().size()); serializer.add((u8*)user->getName().data(), user->getName().size()); serializer.add((u8*)user->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES); serializer.add((u8)user->getGroups().size()); for(Group *group : user->getGroups()) { serializer.add((u8*)group->getId().data, GROUP_ID_LENGTH); } fileAppend(usersFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); return true; } const DatabaseStorageObjectList* DatabaseStorage::getStorage(const Hash &hash) const { auto it = storageMap.find(hash); if(it != storageMap.end()) return it->second; return nullptr; } Group* DatabaseStorage::getGroupById(const Hash &nodeHash, uint8_t groupId[GROUP_ID_LENGTH]) const { auto groupByIdMapIt = nodeGroupByIdMap.find(nodeHash); if(groupByIdMapIt != nodeGroupByIdMap.end()) { auto it = groupByIdMapIt->second->find(DataView(groupId, GROUP_ID_LENGTH)); if(it != groupByIdMapIt->second->end()) return it->second; } return nullptr; } User* DatabaseStorage::getUserByPublicKey(const Hash &nodeHash, const Signature::PublicKey &userPublicKey) const { auto publicKeyUserDataMapIt = nodePublicKeyUserDataMap.find(nodeHash); if(publicKeyUserDataMapIt != nodePublicKeyUserDataMap.end()) { auto it = publicKeyUserDataMapIt->second->find(userPublicKey); if(it != publicKeyUserDataMapIt->second->end()) return it->second; } return nullptr; } bool DatabaseStorage::storeLocalUser(const string &username, const Signature::PublicKey &publicKey, const Signature::PrivateKey &privateKey, const string &password) { auto it = nameLocalUsersMap.find(username); if(it != nameLocalUsersMap.end()) return false; OwnedMemory hashedPassword = hashPassword(DataView((void*)password.data(), password.size()), DataView((void*)passwordSalt, PASSWORD_SALT_LEN)); DataView privateKeyView((void*)privateKey.getData(), PRIVATE_KEY_NUM_BYTES); DataView hashedPasswordView(hashedPassword.data, hashedPassword.size); Encryption encryptedPrivateKey(privateKeyView, {}, hashedPasswordView); EncryptedPrivateKey userEncryptedPrivateKey; memcpy(userEncryptedPrivateKey.nonce, encryptedPrivateKey.getNonce().data, ENCRYPTION_NONCE_BYTE_SIZE); assert(sizeof(userEncryptedPrivateKey.encryptedPrivateKey) == encryptedPrivateKey.getCipherText().size); memcpy(userEncryptedPrivateKey.encryptedPrivateKey, encryptedPrivateKey.getCipherText().data, encryptedPrivateKey.getCipherText().size); LocalUserEncrypted *localUserEncrypted = LocalUserEncrypted::create(publicKey, userEncryptedPrivateKey, username); nameLocalUsersMap[username] = localUserEncrypted; sibs::SafeSerializer serializer; serializer.add((u8)username.size()); serializer.add((const u8*)username.data(), username.size()); serializer.add((const u8*)publicKey.getData(), PUBLIC_KEY_NUM_BYTES); serializer.add((const u8*)encryptedPrivateKey.getNonce().data, ENCRYPTION_NONCE_BYTE_SIZE); serializer.add((const u8*)encryptedPrivateKey.getCipherText().data, ENCRYPTION_CHECKSUM_BYTE_SIZE + PRIVATE_KEY_NUM_BYTES); fileAppend(localUsersFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); return true; } const dht::crypto::Identity& DatabaseStorage::getIdentity() const { return identity; } void DatabaseStorage::update() { // TODO: Modify this to iterate backwards. Because list is sorted in order of timestamp, we can remove data in range auto time = chrono::high_resolution_clock::now().time_since_epoch(); auto timeMicroseconds = chrono::duration_cast(time).count(); for(auto mapIt = quarantineStorageMap.begin(); mapIt != quarantineStorageMap.end(); ) { for(auto vecIt = mapIt->second.begin(); vecIt != mapIt->second.end(); ) { if(timeMicroseconds - (*vecIt)->storedTimestamp > QUARANTINE_STORAGE_TIME_MICROSECONDS) vecIt = mapIt->second.erase(vecIt); else ++vecIt; } if(mapIt->second.empty()) mapIt = quarantineStorageMap.erase(mapIt); else ++mapIt; } } }