From fb447b94e369114df0bc96b5c4c20b2cd102bff0 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 28 Apr 2018 10:44:11 +0200 Subject: Add decryption (and caching) of data, even when adding encryption key after data has been added --- .vscode/c_cpp_properties.json | 4 +- Scheme.md | 2 +- include/odhtdb/Database.hpp | 10 +- include/odhtdb/DatabaseOperation.hpp | 12 + include/odhtdb/DatabaseStorage.hpp | 53 +++- include/odhtdb/Signature.hpp | 2 + include/odhtdb/User.hpp | 5 + src/Database.cpp | 149 +++++------- src/DatabaseStorage.cpp | 456 +++++++++++++++++++++++++++++++++-- src/Encryption.cpp | 2 + src/FileUtils.cpp | 1 + src/Hash.cpp | 2 +- src/Signature.cpp | 6 + src/User.cpp | 10 + tests/main.cpp | 17 +- 15 files changed, 603 insertions(+), 128 deletions(-) create mode 100644 include/odhtdb/DatabaseOperation.hpp diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 9c1a147..e294ff4 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -57,7 +57,9 @@ ], "limitSymbolsToIncludedHeaders": true, "databaseFilename": "" - } + }, + "cStandard": "c11", + "cppStandard": "c++17" }, { "name": "Win32", diff --git a/Scheme.md b/Scheme.md index f573f73..c62c6f4 100644 --- a/Scheme.md +++ b/Scheme.md @@ -27,6 +27,6 @@ Packet timestamp operation type Body - name (of the user to add to group) + name (of the user to add to group, encrypted) public key (of user to add to group) group id (the group which the user should be added to) diff --git a/include/odhtdb/Database.hpp b/include/odhtdb/Database.hpp index 9aff90e..e78bc6e 100644 --- a/include/odhtdb/Database.hpp +++ b/include/odhtdb/Database.hpp @@ -12,6 +12,7 @@ #include "DatabaseNode.hpp" #include "Encryption.hpp" #include "OwnedMemory.hpp" +#include "DatabaseOperation.hpp" #include #include #include @@ -49,12 +50,6 @@ namespace odhtdb DatabaseAddException(const std::string &errMsg) : std::runtime_error(errMsg) {} }; - enum class DatabaseOperation : u8 - { - ADD_DATA, - ADD_USER - }; - struct DatabaseCreateNodeRequest { DISABLE_COPY(DatabaseCreateNodeRequest) @@ -133,6 +128,7 @@ namespace odhtdb class Database { + friend class DatabaseStorage; public: Database(const char *bootstrapNodeAddr, u16 port, const boost::filesystem::path &storageDir); ~Database(); @@ -142,7 +138,7 @@ namespace odhtdb std::unique_ptr create(const std::string &ownerName, const std::string &nodeName); // Throws DatabaseCreateException on failure. std::unique_ptr create(const std::string &ownerName, const Signature::KeyPair &keyPair, const std::string &nodeName); - // Throws DatabaseAddException on failure + // Throws PermissionDeniedException if user @userToPerformActionWith is not allowed to add data to node void addData(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, DataView dataToAdd); // Throws PermissionDeniedException if user @userToPerformActionWith is not allowed to add user @userToAdd to group @groupToAddUserTo void addUser(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, const std::string &userToAddName, const Signature::PublicKey &userToAddPublicKey, Group *groupToAddUserTo); diff --git a/include/odhtdb/DatabaseOperation.hpp b/include/odhtdb/DatabaseOperation.hpp new file mode 100644 index 0000000..369fd09 --- /dev/null +++ b/include/odhtdb/DatabaseOperation.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "types.hpp" + +namespace odhtdb +{ + enum class DatabaseOperation : u8 + { + ADD_DATA, + ADD_USER + }; +} diff --git a/include/odhtdb/DatabaseStorage.hpp b/include/odhtdb/DatabaseStorage.hpp index 34e523e..85a61eb 100644 --- a/include/odhtdb/DatabaseStorage.hpp +++ b/include/odhtdb/DatabaseStorage.hpp @@ -8,6 +8,8 @@ #include "Group.hpp" #include "LocalUser.hpp" #include "LocalUserEncrypted.hpp" +#include "OwnedMemory.hpp" +#include "DatabaseOperation.hpp" #include #include #include @@ -16,21 +18,42 @@ namespace odhtdb { + class Database; + + struct DatabaseStorageObjectDecrypted + { + DatabaseOperation operation; + OwnedMemory data; + }; + struct DatabaseStorageObject { + Hash requestHash; DataView data; u64 createdTimestamp; // In microseconds Signature::PublicKey creatorPublicKey; + DatabaseStorageObjectDecrypted decryptedObject; - DatabaseStorageObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey); + DatabaseStorageObject(const Hash &_requestHash, DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey); }; struct DatabaseStorageObjectList { + Signature::PublicKey creatorPublicKey; DataView data; + u32 offsetToEncryptedData; u64 createdTimestamp; // In microseconds + std::string nodeName; + bool isDecrypted; std::vector groups; std::vector objects; + + DatabaseStorageObjectList(const Signature::PublicKey &_creatorPublicKey) : + creatorPublicKey(_creatorPublicKey), + isDecrypted(false) + { + + } }; struct DatabaseStorageQuarantineObject @@ -86,20 +109,24 @@ namespace odhtdb const int PASSWORD_SALT_LEN = 16; const int HASHED_PASSWORD_LEN = 32; - using NodeLocalUser = std::pair; + struct NodeLocalUser + { + Hash nodeHash; + LocalUser *localUser; + }; class DatabaseStorage { public: // Throws DatabaseStorageCorrupt if storage is corrupted - DatabaseStorage(const boost::filesystem::path &storagePath); + DatabaseStorage(Database *database, const boost::filesystem::path &storagePath); // Throws DatabaseStorageAlreadyExists if data with hash already exists - void createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize); + void createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize, u32 offsetToEncryptedData); // Throws DatabaseStorageNotFound if data with @nodeHash hash has not been created yet. // Throws DatabaseStorageAlreadyExists if same data has been added before (hash of @data, in @dataHash) - void appendStorage(const Hash &nodeHash, const Hash &dataHash, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize); + void appendStorage(const Hash &nodeHash, const Hash &dataHash, DatabaseOperation operation, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize, const DataView &encryptedDataView); // Throws DatabaseStorageAlreadyExists if same data has been added before (hash of @data, in @dataHash) void addToQuarantine(const Hash &dataHash, const Signature::PublicKey &creatorPublicKey, u64 timestamp, const u8 *data, usize dataSize); @@ -132,6 +159,8 @@ namespace odhtdb // Safe to call multiple times. std::vector getLocalNodeUsers(const Signature::KeyPair &keyPair); + void setNodeDecryptionKey(const Hash &nodeHash, const DataView &decryptionKey); + const dht::crypto::Identity& getIdentity() const; // Update storage state (remove quarantine objects if they are too old, etc) @@ -141,21 +170,33 @@ namespace odhtdb void loadUsersFromFile(); void loadDataFromFile(); void loadLocalUsersFromFile(); + void loadNodeDecryptionKeysFromFile(); + void loadDecryptedDataFromFile(); void loadMetadataFromFile(); void loadStorageCreate(sibs::SafeDeserializer &deserializer); void loadStorageAppend(sibs::SafeDeserializer &deserializer); + void loadDecryptedStorageCreate(sibs::SafeDeserializer &deserializer); + void loadDecryptedStorageAddData(sibs::SafeDeserializer &deserializer); + void loadDecryptedStorageAddUser(sibs::SafeDeserializer &deserializer); + + bool decryptNodeData(const Hash &nodeHash, DatabaseStorageObjectList *databaseCreateObject, const std::shared_ptr decryptionKey); + bool decryptNodeAppendedData(const Hash &nodeHash, DatabaseStorageObject *databaseAppendObject, const std::shared_ptr decryptionKey); private: + Database *database; DatabaseStorageMap storageMap; DatabaseStorageQuarantineMap quarantineStorageMap; - SetHash storedDataHash; // Prevent duplicate data from being added + MapHash storedDataHash; // Prevent duplicate data from being added MapHash*> nodePublicKeyUserDataMap; MapHash*> nodeGroupByIdMap; + MapHash> nodeDecryptionKeyMap; std::unordered_map nameLocalUsersMap; boost::filesystem::path groupsFilePath; boost::filesystem::path usersFilePath; boost::filesystem::path dataFilePath; boost::filesystem::path metadataFilePath; boost::filesystem::path localUsersFilePath; + boost::filesystem::path nodeDecryptionKeysFilePath; + boost::filesystem::path decryptedDataFilePath; u8 passwordSalt[PASSWORD_SALT_LEN]; std::pair, std::shared_ptr> identity; }; diff --git a/include/odhtdb/Signature.hpp b/include/odhtdb/Signature.hpp index db434ac..270b099 100644 --- a/include/odhtdb/Signature.hpp +++ b/include/odhtdb/Signature.hpp @@ -121,6 +121,8 @@ namespace odhtdb // Create a key pair from existing public and private key KeyPair(const PublicKey &publicKey, const PrivateKey &privateKey); + KeyPair(const KeyPair &other); + const PublicKey& getPublicKey() const { return publicKey; } const PrivateKey& getPrivateKey() const { return privateKey; } private: diff --git a/include/odhtdb/User.hpp b/include/odhtdb/User.hpp index beb8974..4d6d9b3 100644 --- a/include/odhtdb/User.hpp +++ b/include/odhtdb/User.hpp @@ -2,6 +2,7 @@ #include "Signature.hpp" #include "types.hpp" +#include "Permission.hpp" #include #include #include @@ -9,6 +10,7 @@ namespace odhtdb { class Group; + class DatabaseStorage; class UserNameTooLongException : public std::runtime_error { @@ -22,6 +24,7 @@ namespace odhtdb class User { + friend class DatabaseStorage; public: enum class Type : u8 { @@ -37,6 +40,8 @@ namespace odhtdb const std::string& getName() const { return name; } virtual const std::vector& getGroups() const { return groups; } virtual const Signature::PublicKey& getPublicKey() const = 0; + + virtual bool isAllowedToPerformAction(PermissionType action) const; protected: User(Type type, const std::string &name, Group *group); protected: diff --git a/src/Database.cpp b/src/Database.cpp index bdf9104..6f864b4 100644 --- a/src/Database.cpp +++ b/src/Database.cpp @@ -87,7 +87,7 @@ namespace odhtdb onCreateNodeCallbackFunc(nullptr), onAddNodeCallbackFunc(nullptr), onAddUserCallbackFunc(nullptr), - databaseStorage(storageDir) + databaseStorage(this, storageDir) { node.run(port , { /*.dht_config = */{ @@ -143,7 +143,9 @@ namespace odhtdb // TODO: Use cached files and seed those. If none exists, request new files to seed. // If nobody requests my cached files in a long time, request new files to seed and remove cached files // (only if there are plenty of other seeders for the cached files. This could also cause race issue - // where all nodes with a cached file delete it at same time) + // where all nodes with a cached file delete it at same time). + + databaseStorage.setNodeDecryptionKey(*nodeToSeed.getRequestHash(), DataView(nodeToSeed.getNodeEncryptionKey()->data, nodeToSeed.getNodeEncryptionKey()->size)); Log::debug("Seeding key: %s", nodeToSeed.getRequestHash()->toString().c_str()); DhtKey dhtKey(*nodeToSeed.getRequestHash()); @@ -268,12 +270,8 @@ namespace odhtdb Encryption encryptedBody(DataView(encryptedSerializer.getBuffer().data(), encryptedSerializer.getBuffer().size())); DataView requestData = combine(serializer, encryptedBody); shared_ptr hashRequestKey = make_shared(requestData.data, requestData.size); - databaseStorage.createStorage(*hashRequestKey, adminGroup, timestampMicroseconds, (const u8*)requestData.data, requestData.size); - - string nodeNameCopy(nodeName); - DatabaseCreateNodeRequest createNodeRequest(hashRequestKey.get(), timestampMicroseconds, nodeAdminUser, move(nodeNameCopy)); - if(onCreateNodeCallbackFunc) - onCreateNodeCallbackFunc(createNodeRequest); + databaseStorage.setNodeDecryptionKey(*hashRequestKey, DataView(encryptedBody.getKey().data, encryptedBody.getKey().size)); + databaseStorage.createStorage(*hashRequestKey, adminGroup, timestampMicroseconds, (const u8*)requestData.data, requestData.size, serializer.getBuffer().size()); stagedCreateObjects.emplace_back(make_unique(requestData, hashRequestKey)); @@ -290,6 +288,19 @@ namespace odhtdb void Database::addData(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, DataView dataToAdd) { + if(!userToPerformActionWith->isAllowedToPerformAction(PermissionType::ADD_DATA)) + { + // TODO: User might have permission to perform operation, but we haven't got the packet that adds user to the group with the permission, + // or we haven't received the packet that modifies group with the permission to perform the operation. + // This also means that an user can be in a group that has permission to perform the operation and then later be removed from it, + // and remote peers would accept our request to perform operation if they haven't received the operation that removes the user from the group. + // How to handle this? + string errMsg = "User "; + errMsg += userToPerformActionWith->getName(); + errMsg += " is not allowed to perform the operation: ADD_USER"; + throw PermissionDeniedException(errMsg); + } + sibs::SafeSerializer serializer; serializer.add(DATABASE_ADD_PACKET_STRUCTURE_VERSION); // TODO: Append fractions to get real microseconds time @@ -301,14 +312,11 @@ namespace odhtdb Encryption encryptedBody(dataToAdd, DataView(), encryptionKey); DataView requestData = combine(serializer, encryptedBody); string signedRequestData = userToPerformActionWith->getPrivateKey().sign(requestData); - free(requestData.data); DataView stagedAddObject = combine(userToPerformActionWith->getPublicKey(), signedRequestData); Hash requestDataHash(stagedAddObject.data, stagedAddObject.size); - databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, userToPerformActionWith, timestampMicroseconds, (u8*)stagedAddObject.data, stagedAddObject.size); - - DatabaseAddNodeRequest addNodeRequest(&*nodeInfo.getRequestHash(), &requestDataHash, timestampMicroseconds, userToPerformActionWith, dataToAdd); - if(onAddNodeCallbackFunc) - onAddNodeCallbackFunc(addNodeRequest); + DataView encryptedDataView((char*)requestData.data + serializer.getBuffer().size(), requestData.size - serializer.getBuffer().size()); + databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, DatabaseOperation::ADD_DATA, userToPerformActionWith, timestampMicroseconds, (u8*)stagedAddObject.data, stagedAddObject.size, encryptedDataView); + delete (char*)requestData.data; stagedAddObjects.emplace_back(make_unique(stagedAddObject, nodeInfo.getRequestHash())); } @@ -350,8 +358,14 @@ namespace odhtdb serializer.add(DatabaseOperation::ADD_USER); assert(userToAddName.size() <= 255); + usize serializedEncryptedDataOffset = serializer.getBuffer().size(); serializer.add((u8)userToAddName.size()); - serializer.add((u8*)userToAddName.data(), userToAddName.size()); + DataView encryptionKey(nodeInfo.getNodeEncryptionKey()->data, ENCRYPTION_KEY_BYTE_SIZE); + Encryption encryptedUserName(DataView((void*)userToAddName.data(), userToAddName.size()), DataView(), encryptionKey); + serializer.add((u8*)encryptedUserName.getNonce().data, ENCRYPTION_NONCE_BYTE_SIZE); + assert(encryptedUserName.getCipherText().size == ENCRYPTION_CHECKSUM_BYTE_SIZE + userToAddName.size()); + serializer.add((u8*)encryptedUserName.getCipherText().data, ENCRYPTION_CHECKSUM_BYTE_SIZE + userToAddName.size()); + usize serializedEncryptedDataSize = serializer.getBuffer().size() - serializedEncryptedDataOffset; serializer.add((u8*)userToAddPublicKey.getData(), PUBLIC_KEY_NUM_BYTES); serializer.add((uint8_t*)groupToAddUserTo->getId().data, groupToAddUserTo->getId().size); @@ -359,13 +373,10 @@ namespace odhtdb string signedRequestData = userToPerformActionWith->getPrivateKey().sign(requestData); DataView stagedAddObject = combine(userToPerformActionWith->getPublicKey(), signedRequestData); Hash requestDataHash(stagedAddObject.data, stagedAddObject.size); - databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, userToPerformActionWith, timestampMicroseconds, (u8*)stagedAddObject.data, stagedAddObject.size); + DataView encryptedDataView(nullptr, 0); auto userToAdd = RemoteUser::create(userToAddPublicKey, userToAddName, groupToAddUserTo); databaseStorage.addUser(*nodeInfo.getRequestHash(), userToAdd); - - DatabaseAddUserRequest addUserRequest(&*nodeInfo.getRequestHash(), &requestDataHash, timestampMicroseconds, userToPerformActionWith, userToAdd, groupToAddUserTo); - if(onAddUserCallbackFunc) - onAddUserCallbackFunc(addUserRequest); + databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, DatabaseOperation::ADD_USER, userToPerformActionWith, timestampMicroseconds, (u8*)stagedAddObject.data, stagedAddObject.size, encryptedDataView); stagedAddObjects.emplace_back(make_unique(stagedAddObject, nodeInfo.getRequestHash())); } @@ -468,8 +479,8 @@ namespace odhtdb deserializer.extract((u8*)creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); Signature::PublicKey userPublicKey(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); - uint8_t adminGroupId[16]; - deserializer.extract(adminGroupId, 16); + uint8_t adminGroupId[GROUP_ID_LENGTH]; + deserializer.extract(adminGroupId, GROUP_ID_LENGTH); if(deserializer.getSize() < ENCRYPTION_NONCE_BYTE_SIZE) throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)"); @@ -477,39 +488,7 @@ namespace odhtdb auto adminGroup = new Group("administrator", adminGroupId, ADMIN_PERMISSION); // TODO: Username is encrypted, we dont know it... unless we have encryption key, in which case we should modify the user name and set it auto creatorUser = RemoteUser::create(userPublicKey, "ENCRYPTED USER NAME", adminGroup); - databaseStorage.createStorage(hash, adminGroup, creationDate, value->data.data(), value->data.size()); - - u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE]; - deserializer.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE); - - DataView dataToDecrypt((void*)deserializer.getBuffer(), deserializer.getSize()); - Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(encryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE)); - sibs::SafeDeserializer bodyDeserializer((const u8*)decryptedBody.getDecryptedText().data, decryptedBody.getDecryptedText().size); - - u8 creatorNameLength = bodyDeserializer.extract(); - string creatorName; // TODO: Add this user name to storage added above - creatorName.resize(creatorNameLength); - bodyDeserializer.extract((u8*)&creatorName[0], creatorNameLength); - - u8 nameLength = bodyDeserializer.extract(); - string name; - name.resize(nameLength); - bodyDeserializer.extract((u8*)&name[0], nameLength); - - Log::debug("Got create object, name: %s", name.c_str()); - DatabaseCreateNodeRequest createNodeRequest(&hash, creationDate, creatorUser, move(name)); - if(onCreateNodeCallbackFunc) - onCreateNodeCallbackFunc(createNodeRequest); - } - - bool isUserAllowedToAddData(const User *user) - { - for(Group *group : user->getGroups()) - { - if(group->getPermission().getFlag(PermissionType::ADD_DATA)) - return true; - } - return false; + databaseStorage.createStorage(hash, adminGroup, creationDate, value->data.data(), value->data.size(), value->data.size() - deserializer.getSize()); } void Database::deserializeAddRequest(const shared_ptr &value, const Hash &requestDataHash, const std::shared_ptr &nodeHash, const shared_ptr encryptionKey) @@ -552,21 +531,24 @@ namespace odhtdb } #endif auto creatorUser = databaseStorage.getUserByPublicKey(*nodeHash, creatorPublicKey); - // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine. - // TODO: We might receive 'add' data packet before 'create'. If that happens, we should put it in quarantine and process it later. - databaseStorage.appendStorage(*nodeHash, requestDataHash, creatorUser, creationDate, value->data.data(), value->data.size()); + if(!creatorUser) + { + // TODO: Add to quarantine + string errMsg = "User with public key "; + errMsg += creatorPublicKey.toString(); + errMsg += " does not exist in code "; + errMsg += nodeHash->toString(); + throw sibs::DeserializeException(errMsg); + } + + DataView encryptedDataView((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize()); if(operation == DatabaseOperation::ADD_DATA) { if(deserializerUnsigned.getSize() < ENCRYPTION_NONCE_BYTE_SIZE) throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)"); - u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE]; - deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE); - DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize()); - Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(encryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE)); - - if(!isUserAllowedToAddData(creatorUser)) + if(!creatorUser->isAllowedToPerformAction(PermissionType::ADD_DATA)) { // TODO: User might have permission to perform operation, but we haven't got the packet that adds user to the group with the permission, // or we haven't received the packet that modifies group with the permission to perform the operation. @@ -575,34 +557,36 @@ namespace odhtdb // How to handle this? string errMsg = "User "; errMsg += creatorUser->getName(); - errMsg += " is not allowed to perform the operation: "; - errMsg += to_string((u8)operation); + errMsg += " is not allowed to add data to node "; + errMsg += nodeHash->toString(); throw PermissionDeniedException(errMsg); } - Log::debug("Got add object, timestamp: %zu, data: %.*s", creationDate, decryptedBody.getDecryptedText().size, decryptedBody.getDecryptedText().data); - const DatabaseAddNodeRequest addNodeRequest(&*nodeHash, &requestDataHash, creationDate, creatorUser, decryptedBody.getDecryptedText()); - if(onAddNodeCallbackFunc) - onAddNodeCallbackFunc(addNodeRequest); + // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine. + // TODO: We might receive 'add' data packet before 'create'. If that happens, we should put it in quarantine and process it later. + databaseStorage.appendStorage(*nodeHash, requestDataHash, operation, creatorUser, creationDate, value->data.data(), value->data.size(), encryptedDataView); } else if(operation == DatabaseOperation::ADD_USER) - { + { u8 nameLength = deserializerUnsigned.extract(); - string name; - name.resize(nameLength); - deserializerUnsigned.extract((u8*)&name[0], nameLength); + + u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE]; + deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE); + DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), ENCRYPTION_CHECKSUM_BYTE_SIZE + nameLength); + + sibs::SafeDeserializer deserializerSkippedEncryptedData(deserializerUnsigned.getBuffer() + ENCRYPTION_CHECKSUM_BYTE_SIZE + nameLength, PUBLIC_KEY_NUM_BYTES + GROUP_ID_LENGTH); char userToAddPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; - deserializerUnsigned.extract((u8*)userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + deserializerSkippedEncryptedData.extract((u8*)userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); Signature::PublicKey userToAddPublicKey(userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); - uint8_t groupId[16]; - deserializerUnsigned.extract(groupId, 16); + uint8_t groupId[GROUP_ID_LENGTH]; + deserializerSkippedEncryptedData.extract(groupId, GROUP_ID_LENGTH); auto group = databaseStorage.getGroupById(*nodeHash, groupId); if(group) { - auto user = RemoteUser::create(userToAddPublicKey, name, group); + auto user = RemoteUser::create(userToAddPublicKey, "ENCRYPTED USER NAME", group); // TODO: What if we receive packets in wrong order? (maliciously or non-maliciously). You would be able to register a user to a group with given name // and further registration would be dropped (even if that is the correct one) if(!databaseStorage.addUser(*nodeHash, user)) return; @@ -622,10 +606,9 @@ namespace odhtdb throw PermissionDeniedException(errMsg); } - Log::debug("Got add user object, timestamp: %zu, user added: %.*s", creationDate, nameLength, name.c_str()); - DatabaseAddUserRequest addUserRequest(&*nodeHash, &requestDataHash, creationDate, creatorUser, user, group); - if(onAddUserCallbackFunc) - onAddUserCallbackFunc(addUserRequest); + // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine. + // TODO: We might receive 'add' data packet before 'create'. If that happens, we should put it in quarantine and process it later. + databaseStorage.appendStorage(*nodeHash, requestDataHash, operation, creatorUser, creationDate, value->data.data(), value->data.size(), encryptedDataView); } else { @@ -637,7 +620,7 @@ namespace odhtdb string errMsg = "Got unexpected operation: "; errMsg += to_string((u8)operation); throw sibs::DeserializeException(errMsg); - } + } } bool Database::listenCreateData(shared_ptr value, const Hash &hash, const shared_ptr encryptionKey) diff --git a/src/DatabaseStorage.cpp b/src/DatabaseStorage.cpp index 34e6da4..0e9fa32 100644 --- a/src/DatabaseStorage.cpp +++ b/src/DatabaseStorage.cpp @@ -7,6 +7,7 @@ #include "../include/odhtdb/bin2hex.hpp" #include "../include/odhtdb/PasswordHash.hpp" #include "../include/odhtdb/Log.hpp" +#include "../include/odhtdb/Database.hpp" #include #include #include @@ -23,11 +24,21 @@ namespace odhtdb STORAGE_TYPE_APPEND }; + enum class DecryptedDataType : u8 + { + STORAGE_CREATE, + STORAGE_ADD_DATA, + STORAGE_ADD_USER + }; + const u64 QUARANTINE_STORAGE_TIME_MICROSECONDS = 60 * 1.0e6; const u16 STORAGE_VERSION = 1; - - DatabaseStorageObject::DatabaseStorageObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : - data(_data), createdTimestamp(_timestamp), creatorPublicKey(_creatorPublicKey) + + DatabaseStorageObject::DatabaseStorageObject(const Hash &_requestHash, DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : + requestHash(_requestHash), + data(_data), + createdTimestamp(_timestamp), + creatorPublicKey(_creatorPublicKey) { } @@ -39,12 +50,15 @@ namespace odhtdb storedTimestamp = chrono::duration_cast(time).count(); } - DatabaseStorage::DatabaseStorage(const boost::filesystem::path &storagePath) : + DatabaseStorage::DatabaseStorage(Database *_database, const boost::filesystem::path &storagePath) : + database(_database), groupsFilePath(storagePath / "groups"), usersFilePath(storagePath / "users"), dataFilePath(storagePath / "data"), metadataFilePath(storagePath / "metadata"), - localUsersFilePath(storagePath / "local_users") + localUsersFilePath(storagePath / "local_users"), + nodeDecryptionKeysFilePath(storagePath / "node_keys"), + decryptedDataFilePath(storagePath / "decrypted_data") { boost::filesystem::create_directories(storagePath); @@ -57,13 +71,21 @@ namespace odhtdb loadUsersFromFile(); loadDataFromFile(); loadLocalUsersFromFile(); + loadNodeDecryptionKeysFromFile(); + loadDecryptedDataFromFile(); //loadQuarantineFromFile(); } catch(FileException &e) { - Log::warn("Failed to load storage data, reason: %s. Ignoring...", e.what()); - if(!metadataLoaded) + if(metadataLoaded) { + string errMsg = "Failed to load storage, reason: "; + errMsg += e.what(); + throw DatabaseStorageCorrupt(errMsg); + } + else + { + Log::warn("Failed to load storage meta data, reason: %s. Ignoring... (new storage probably)", e.what()); sibs::SafeSerializer metadataSerializer; metadataSerializer.add(STORAGE_VERSION); randombytes_buf(passwordSalt, PASSWORD_SALT_LEN); @@ -83,6 +105,12 @@ namespace odhtdb fileAppend(metadataFilePath, { metadataSerializer.getBuffer().data(), metadataSerializer.getBuffer().size() }); } } + catch(sibs::DeserializeException &e) + { + string errMsg = "Failed to load storage, reason: "; + errMsg += e.what(); + throw DatabaseStorageCorrupt(errMsg); + } } void DatabaseStorage::loadGroupsFromFile() @@ -122,7 +150,7 @@ namespace odhtdb (*groupByIdMap)[group->getId()] = group; string groupIdStr = bin2hex((const char*)groupId, GROUP_ID_LENGTH); - Log::debug("DatabaseStorage: Loaded group from file: %s", groupIdStr.c_str()); + Log::debug("DatabaseStorage: Loaded group %s from file", groupIdStr.c_str()); } } @@ -194,12 +222,17 @@ namespace odhtdb { u64 timestamp = deserializer.extract(); + u8 userPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; + deserializer.extract(userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + Signature::PublicKey userPublicKey((const char*)userPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + u8 groupId[GROUP_ID_LENGTH]; deserializer.extract(groupId, GROUP_ID_LENGTH); u32 dataSize = deserializer.extract(); u8 *data = new u8[dataSize]; deserializer.extract(data, dataSize); + u32 offsetToEncryptedData = deserializer.extract(); Hash nodeHash; deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); @@ -222,10 +255,11 @@ namespace odhtdb throw DatabaseStorageCorrupt(errMsg); } - DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(); + DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(userPublicKey); databaseStorageObjectList->createdTimestamp = timestamp; databaseStorageObjectList->groups.push_back(groupIt->second); databaseStorageObjectList->data = DataView(data, dataSize); + databaseStorageObjectList->offsetToEncryptedData = offsetToEncryptedData; storageMap[nodeHash] = databaseStorageObjectList; } @@ -246,7 +280,6 @@ namespace odhtdb Hash dataHash; deserializer.extract((u8*)dataHash.getData(), HASH_BYTE_SIZE); - storedDataHash.insert(dataHash); auto storageIt = storageMap.find(nodeHash); if(storageIt == storageMap.end()) @@ -258,8 +291,9 @@ namespace odhtdb } DataView storageData { data, dataSize }; - DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(storageData, timestamp, creatorPublicKey); + DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(dataHash, storageData, timestamp, creatorPublicKey); storageIt->second->objects.push_back(databaseStorageObject); + storedDataHash[dataHash] = databaseStorageObject; } void DatabaseStorage::loadDataFromFile() @@ -310,6 +344,139 @@ namespace odhtdb } } + void DatabaseStorage::loadNodeDecryptionKeysFromFile() + { + if(!boost::filesystem::exists(nodeDecryptionKeysFilePath)) return; + + OwnedMemory nodeKeysFileContent = fileGetContent(nodeDecryptionKeysFilePath); + sibs::SafeDeserializer deserializer((u8*)nodeKeysFileContent.data, nodeKeysFileContent.size); + + while(!deserializer.empty()) + { + Hash nodeHash; + deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); + + u8 *nodeKeyRaw = new u8[ENCRYPTION_KEY_BYTE_SIZE]; + deserializer.extract(nodeKeyRaw, ENCRYPTION_KEY_BYTE_SIZE); + nodeDecryptionKeyMap[nodeHash] = make_shared(nodeKeyRaw, ENCRYPTION_KEY_BYTE_SIZE); + } + } + + void DatabaseStorage::loadDecryptedStorageCreate(sibs::SafeDeserializer &deserializer) + { + Hash nodeHash; + deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); + + auto storageIt = storageMap.find(nodeHash); + if(storageIt == storageMap.end()) + { + string errMsg = "Database storage with hash "; + errMsg += nodeHash.toString(); + errMsg += " not found"; + throw DatabaseStorageCorrupt(errMsg); + } + + u8 creatorNameLength = deserializer.extract(); + auto creator = getUserByPublicKey(nodeHash, storageIt->second->creatorPublicKey); + if(!creator) + { + string errMsg = "User with public key "; + errMsg += storageIt->second->creatorPublicKey.toString(); + errMsg += " does not exist in node with hash "; + errMsg += nodeHash.toString(); + throw DatabaseStorageCorrupt(errMsg); + } + + creator->name.resize(creatorNameLength); + deserializer.extract((u8*)&creator->name[0], creatorNameLength); + + u8 nodeNameLength = deserializer.extract(); + storageIt->second->nodeName.resize(nodeNameLength); + deserializer.extract((u8*)&storageIt->second->nodeName[0], nodeNameLength); + storageIt->second->isDecrypted = true; + } + + void DatabaseStorage::loadDecryptedStorageAddData(sibs::SafeDeserializer &deserializer) + { + Hash requestHash; + deserializer.extract((u8*)requestHash.getData(), HASH_BYTE_SIZE); + + auto storedDataIt = storedDataHash.find(requestHash); + if(storedDataIt == storedDataHash.end()) + { + string errMsg = "Database doesn't contain data with hash "; + errMsg += requestHash.toString(); + throw DatabaseStorageCorrupt(errMsg); + } + + u32 decryptedDataSize = deserializer.extract(); + u8 *decryptedDataRaw = new u8[decryptedDataSize]; + deserializer.extract(decryptedDataRaw, decryptedDataSize); + + storedDataIt->second->decryptedObject.data.data = decryptedDataRaw; + storedDataIt->second->decryptedObject.data.size = decryptedDataSize; + storedDataIt->second->decryptedObject.operation = DatabaseOperation::ADD_DATA; + } + + void DatabaseStorage::loadDecryptedStorageAddUser(sibs::SafeDeserializer &deserializer) + { + Hash nodeHash; + deserializer.extract((u8*)nodeHash.getData(), HASH_BYTE_SIZE); + + Hash requestHash; + deserializer.extract((u8*)requestHash.getData(), HASH_BYTE_SIZE); + + auto storedDataIt = storedDataHash.find(requestHash); + if(storedDataIt == storedDataHash.end()) + { + string errMsg = "Database doesn't contain data with hash "; + errMsg += requestHash.toString(); + throw DatabaseStorageCorrupt(errMsg); + } + + u8 addedUserPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; + deserializer.extract(addedUserPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + Signature::PublicKey addedUserPublicKey((const char*)addedUserPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + auto addedUser = getUserByPublicKey(nodeHash, addedUserPublicKey); + if(!addedUser) + { + string errMsg = "User with public key "; + errMsg += bin2hex((const char*)addedUserPublicKeyRaw, PUBLIC_KEY_NUM_BYTES).c_str(); + errMsg += " does not exist in node with hash "; + errMsg += nodeHash.toString(); + throw DatabaseStorageCorrupt(errMsg); + } + + u32 decryptedUserNameSize = deserializer.extract(); + addedUser->name.resize(decryptedUserNameSize); + deserializer.extract((u8*)&addedUser->name[0], decryptedUserNameSize); + } + + void DatabaseStorage::loadDecryptedDataFromFile() + { + if(!boost::filesystem::exists(decryptedDataFilePath)) return; + + OwnedMemory decryptedDataFileContent = fileGetContent(decryptedDataFilePath); + sibs::SafeDeserializer deserializer((u8*)decryptedDataFileContent.data, decryptedDataFileContent.size); + + while(!deserializer.empty()) + { + DecryptedDataType decryptedDataType = deserializer.extract(); + switch(decryptedDataType) + { + case DecryptedDataType::STORAGE_CREATE: + loadDecryptedStorageCreate(deserializer); + break; + case DecryptedDataType::STORAGE_ADD_DATA: + loadDecryptedStorageAddData(deserializer); + break; + case DecryptedDataType::STORAGE_ADD_USER: + loadDecryptedStorageAddUser(deserializer); + break; + } + } + } + void DatabaseStorage::loadMetadataFromFile() { OwnedMemory metadataFileContent = fileGetContent(metadataFilePath); @@ -319,7 +486,6 @@ namespace odhtdb if(storageVersion != STORAGE_VERSION) throw std::runtime_error("Wrong storage version!"); - u8 passwordSalt[PASSWORD_SALT_LEN]; deserializer.extract(passwordSalt, PASSWORD_SALT_LEN); //string passwordSaltStr((const char*)passwordSalt, PASSWORD_SALT_LEN); @@ -338,7 +504,7 @@ namespace odhtdb assert(deserializer.empty()); } - void DatabaseStorage::createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize) + void DatabaseStorage::createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize, u32 offsetToEncryptedData) { if(storageMap.find(hash) != storageMap.end()) { @@ -349,32 +515,39 @@ namespace odhtdb } addGroup(hash, creatorGroup); - for(auto user : creatorGroup->getUsers()) - { - addUser(hash, (User*)user); - } + assert(creatorGroup->getUsers().size() == 1); + User *creator = (User*)creatorGroup->getUsers()[0]; + addUser(hash, creator); - DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(); + DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(creator->getPublicKey()); databaseStorageObjectList->createdTimestamp = timestamp; databaseStorageObjectList->groups.push_back(creatorGroup); databaseStorageObjectList->data = DataView(new u8[dataSize], dataSize); memcpy(databaseStorageObjectList->data.data, data, dataSize); + databaseStorageObjectList->offsetToEncryptedData = offsetToEncryptedData; storageMap[hash] = databaseStorageObjectList; sibs::SafeSerializer serializer; serializer.add(STORAGE_TYPE_CREATE); serializer.add(timestamp); + serializer.add((u8*)creator->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES); serializer.add((u8*)creatorGroup->getId().data, GROUP_ID_LENGTH); serializer.add((u32)dataSize); serializer.add(data, dataSize); + serializer.add(offsetToEncryptedData); serializer.add((u8*)hash.getData(), HASH_BYTE_SIZE); fileAppend(dataFilePath, { serializer.getBuffer().data(), serializer.getBuffer().size() }); + + auto nodeDecryptionKeyIt = nodeDecryptionKeyMap.find(hash); + if(nodeDecryptionKeyIt != nodeDecryptionKeyMap.end()) + decryptNodeData(hash, databaseStorageObjectList, nodeDecryptionKeyIt->second); } - void DatabaseStorage::appendStorage(const Hash &nodeHash, const Hash &dataHash, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize) + // TODO: Use encryptedDataView to remove duplicate of unsigning request, jumping straight to decrypting encrypted data and calling callback func + void DatabaseStorage::appendStorage(const Hash &nodeHash, const Hash &dataHash, DatabaseOperation operation, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize, const DataView &encryptedDataView) { auto it = storageMap.find(nodeHash); if(it == storageMap.end()) @@ -385,8 +558,8 @@ namespace odhtdb throw DatabaseStorageNotFound(errMsg); } - auto storeDataHashResult = storedDataHash.insert(dataHash); - if(!storeDataHashResult.second) + auto storeDataHashResult = storedDataHash.find(dataHash); + if(storeDataHashResult != storedDataHash.end()) { string errMsg = "Database already contains data with hash: "; errMsg += dataHash.toString(); @@ -395,8 +568,9 @@ namespace odhtdb DataView storageData { new u8[dataSize], dataSize }; memcpy(storageData.data, data, dataSize); - DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(storageData, timestamp, creatorUser->getPublicKey()); + DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(dataHash, storageData, timestamp, creatorUser->getPublicKey()); it->second->objects.push_back(databaseStorageObject); + storedDataHash[dataHash] = databaseStorageObject; sibs::SafeSerializer serializer; serializer.add(STORAGE_TYPE_APPEND); @@ -410,12 +584,16 @@ namespace odhtdb serializer.add((u8*)dataHash.getData(), HASH_BYTE_SIZE); fileAppend(dataFilePath, { serializer.getBuffer().data(), serializer.getBuffer().size() }); + + auto nodeDecryptionKeyIt = nodeDecryptionKeyMap.find(nodeHash); + if(nodeDecryptionKeyIt != nodeDecryptionKeyMap.end()) + decryptNodeAppendedData(nodeHash, databaseStorageObject, nodeDecryptionKeyIt->second); } void DatabaseStorage::addToQuarantine(const Hash &dataHash, const Signature::PublicKey &creatorPublicKey, u64 timestamp, const u8 *data, usize dataSize) { - auto storeDataHashResult = storedDataHash.insert(dataHash); - if(!storeDataHashResult.second) + auto storeDataHashResult = storedDataHash.find(dataHash); + if(storeDataHashResult != storedDataHash.end()) { string errMsg = "Database already contains data with hash: "; errMsg += dataHash.toString(); @@ -426,6 +604,7 @@ namespace odhtdb memcpy(storageData.data, data, dataSize); DatabaseStorageQuarantineObject *databaseQuarantineStorageObject = new DatabaseStorageQuarantineObject(storageData, timestamp, creatorPublicKey); quarantineStorageMap[creatorPublicKey].emplace_back(databaseQuarantineStorageObject); + // TODO: Add quarantine object to storedDataHash } bool DatabaseStorage::addGroup(const Hash &nodeHash, Group *group) @@ -458,6 +637,8 @@ namespace odhtdb serializer.add(group->getPermission().getPermissionFlags()); fileAppend(groupsFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); + + Log::debug("Created group %s in node %s", bin2hex((const char*)group->getId().data, GROUP_ID_LENGTH).c_str(), nodeHash.toString().c_str()); return true; } @@ -497,6 +678,8 @@ namespace odhtdb } fileAppend(usersFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); + + Log::debug("Created user %s in node %s", user->getPublicKey().toString().c_str(), nodeHash.toString().c_str()); return true; } @@ -612,17 +795,238 @@ namespace odhtdb } (*nodeIt.second)[keyPair.getPublicKey()] = localUser; - localUsers.push_back(make_pair(nodeIt.first, localUser)); + localUsers.push_back({ nodeIt.first, localUser }); delete user; } else - localUsers.push_back(make_pair(nodeIt.first, static_cast(user))); + localUsers.push_back({ nodeIt.first, static_cast(user) }); } } return localUsers; } + void DatabaseStorage::setNodeDecryptionKey(const Hash &nodeHash, const DataView &decryptionKeyView) + { + bool nodeHasExistingEncryptionKey = nodeDecryptionKeyMap.find(nodeHash) != nodeDecryptionKeyMap.end(); + + char *decryptionKeyRaw = new char[decryptionKeyView.size]; + memcpy(decryptionKeyRaw, decryptionKeyView.data, decryptionKeyView.size); + shared_ptr decryptionKey = make_shared(decryptionKeyRaw, decryptionKeyView.size); + nodeDecryptionKeyMap[nodeHash] = decryptionKey; + + sibs::SafeSerializer serializer; + serializer.add((const u8*)nodeHash.getData(), HASH_BYTE_SIZE); + serializer.add((const u8*)decryptionKeyView.data, ENCRYPTION_KEY_BYTE_SIZE); + fileAppend(nodeDecryptionKeysFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); + + auto storageIt = storageMap.find(nodeHash); + if(storageIt == storageMap.end()) return; + + // When changing existing encryption key, do not decrypt the existing data as it has already been decrypted, + // the new key should only be used for new data + if(!nodeHasExistingEncryptionKey) + decryptNodeData(nodeHash, storageIt->second, decryptionKey); + } + + bool DatabaseStorage::decryptNodeData(const Hash &nodeHash, DatabaseStorageObjectList *databaseCreateObject, const shared_ptr decryptionKey) + { + sibs::SafeDeserializer deserializer((u8*)databaseCreateObject->data.data + databaseCreateObject->offsetToEncryptedData, databaseCreateObject->data.size - databaseCreateObject->offsetToEncryptedData); + + u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE]; + deserializer.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE); + + DataView dataToDecrypt((void*)deserializer.getBuffer(), deserializer.getSize()); + Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(decryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE)); + sibs::SafeDeserializer bodyDeserializer((const u8*)decryptedBody.getDecryptedText().data, decryptedBody.getDecryptedText().size); + + u8 creatorNameLength = bodyDeserializer.extract(); + string creatorName; // TODO: Add this user name to storage creator name + creatorName.resize(creatorNameLength); + bodyDeserializer.extract((u8*)&creatorName[0], creatorNameLength); + + u8 nameLength = bodyDeserializer.extract(); + string name; + name.resize(nameLength); + bodyDeserializer.extract((u8*)&name[0], nameLength); + + sibs::SafeSerializer serializer; + serializer.add(DecryptedDataType::STORAGE_CREATE); + serializer.add((const u8*)nodeHash.getData(), HASH_BYTE_SIZE); + serializer.add(creatorNameLength); + serializer.add((const u8*)creatorName.data(), creatorNameLength); + serializer.add(nameLength); + serializer.add((const u8*)name.data(), nameLength); + fileAppend(decryptedDataFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); + + User *creator = getUserByPublicKey(nodeHash, databaseCreateObject->creatorPublicKey); + if(!creator) + { + Log::error("Creator with public key %s does not exist in node %s", databaseCreateObject->creatorPublicKey.toString().c_str(), nodeHash.toString().c_str()); + return false; + } + creator->name = move(creatorName); + databaseCreateObject->nodeName = name; + databaseCreateObject->isDecrypted = true; + + Log::debug("Deserialized node create data, name: %s", name.c_str()); + const DatabaseCreateNodeRequest createNodeRequest(&nodeHash, databaseCreateObject->createdTimestamp, creator, move(name)); + if(database->onCreateNodeCallbackFunc) + database->onCreateNodeCallbackFunc(createNodeRequest); + + bool success = true; + for(auto appendObject : databaseCreateObject->objects) + { + bool appendObjectResult = decryptNodeAppendedData(nodeHash, appendObject, decryptionKey); + if(!appendObjectResult) + success = false; + } + return success; + } + + bool DatabaseStorage::decryptNodeAppendedData(const Hash &nodeHash, DatabaseStorageObject *databaseAppendObject, const shared_ptr decryptionKey) + { + sibs::SafeDeserializer deserializer((u8*)databaseAppendObject->data.data, databaseAppendObject->data.size); + char creatorPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; + deserializer.extract((u8*)creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + Signature::PublicKey creatorPublicKey(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + + DataView signedData((void*)deserializer.getBuffer(), deserializer.getSize()); + string unsignedData = creatorPublicKey.unsign(signedData); + sibs::SafeDeserializer deserializerUnsigned((u8*)unsignedData.data(), unsignedData.size()); + + u16 packetStructureVersion = deserializerUnsigned.extract(); + u64 creationDate = deserializerUnsigned.extract(); + + DatabaseOperation operation = deserializerUnsigned.extract(); + + auto creatorUser = getUserByPublicKey(nodeHash, creatorPublicKey); + if(!creatorUser) + { + Log::error("User with public key %s does not exist in node %s", creatorPublicKey.toString().c_str(), nodeHash.toString().c_str()); + return false; + } + + if(operation == DatabaseOperation::ADD_DATA) + { + if(deserializerUnsigned.getSize() < ENCRYPTION_NONCE_BYTE_SIZE) + { + Log::error("Unsigned encrypted body is too small (unable to extract nonce)"); + return false; + } +#if 0 + if(!creatorUser->isAllowedToPerformAction(PermissionType::ADD_DATA)) + { + // TODO: User might have permission to perform operation, but we haven't got the packet that adds user to the group with the permission, + // or we haven't received the packet that modifies group with the permission to perform the operation. + // This also means that an user can be in a group that has permission to perform the operation and then later be removed from it, + // and remote peers would accept our request to perform operation if they haven't received the operation that removes the user from the group. + // How to handle this? + Log::error("User %s is not allowed to add data to node %s", creatorUser->getName().c_str(), nodeHash.toString().c_str()); + return false; + } +#endif + + u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE]; + deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE); + DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize()); + Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(decryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE)); + + sibs::SafeSerializer serializer; + serializer.add(DecryptedDataType::STORAGE_ADD_DATA); + serializer.add((const u8*)databaseAppendObject->requestHash.getData(), HASH_BYTE_SIZE); + serializer.add((u32)decryptedBody.getDecryptedText().size); + serializer.add((const u8*)decryptedBody.getDecryptedText().data, decryptedBody.getDecryptedText().size); + fileAppend(decryptedDataFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); + databaseAppendObject->decryptedObject.data.data = new char[decryptedBody.getDecryptedText().size]; + memcpy(databaseAppendObject->decryptedObject.data.data, decryptedBody.getDecryptedText().data, decryptedBody.getDecryptedText().size); + databaseAppendObject->decryptedObject.data.size = decryptedBody.getDecryptedText().size; + databaseAppendObject->decryptedObject.operation = operation; + + Log::debug("Got add object, timestamp: %zu, data: %.*s", creationDate, decryptedBody.getDecryptedText().size, decryptedBody.getDecryptedText().data); + const DatabaseAddNodeRequest addNodeRequest(&nodeHash, &databaseAppendObject->requestHash, creationDate, creatorUser, decryptedBody.getDecryptedText()); + if(database->onAddNodeCallbackFunc) + database->onAddNodeCallbackFunc(addNodeRequest); + } + else if(operation == DatabaseOperation::ADD_USER) + { + u8 nameLength = deserializerUnsigned.extract(); + + u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE]; + deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE); + DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), ENCRYPTION_CHECKSUM_BYTE_SIZE + nameLength); + Decryption decryptedUserName(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(decryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE)); + + string username; + username.resize(nameLength); + assert(decryptedUserName.getDecryptedText().size == nameLength); + memcpy(&username[0], decryptedUserName.getDecryptedText().data, nameLength); + + sibs::SafeDeserializer deserializerSkippedEncryptedData(deserializerUnsigned.getBuffer() + ENCRYPTION_CHECKSUM_BYTE_SIZE + nameLength, PUBLIC_KEY_NUM_BYTES + GROUP_ID_LENGTH); + + char userToAddPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; + deserializerSkippedEncryptedData.extract((u8*)userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + Signature::PublicKey userToAddPublicKey(userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + + uint8_t groupId[GROUP_ID_LENGTH]; + deserializerSkippedEncryptedData.extract(groupId, GROUP_ID_LENGTH); + + auto group = getGroupById(nodeHash, groupId); + if(!group) + { + // TODO: Add to quarantine? + Log::error("There is no group with id %s in node %s", bin2hex((const char*)groupId, GROUP_ID_LENGTH).c_str(), nodeHash.toString().c_str()); + return false; + } + + auto user = getUserByPublicKey(nodeHash, userToAddPublicKey); + if(!user) + { + Log::error("User to add with public key %s doesn't exist in node %s", userToAddPublicKey.toString().c_str(), nodeHash.toString().c_str()); + return false; + } +#if 0 + auto creatorUserGroupWithRights = getGroupWithRightsToAddUserToGroup(creatorUser->getGroups(), group); + if(!creatorUserGroupWithRights) + { + // TODO: User might have permission to perform operation, but we haven't got the packet that adds user to the group with the permission, + // or we haven't received the packet that modifies group with the permission to perform the operation. + // This also means that an user can be in a group that has permission to perform the operation and then later be removed from it, + // and remote peers would accept our request to perform operation if they haven't received the operation that removes the user from the group. + // How to handle this? + string errMsg = "User "; + errMsg += creatorUser->getName(); + errMsg += " is not allowed to perform the operation: "; + errMsg += to_string((u8)operation); + throw PermissionDeniedException(errMsg); + } +#endif + sibs::SafeSerializer serializer; + serializer.add(DecryptedDataType::STORAGE_ADD_USER); + serializer.add((const u8*)nodeHash.getData(), HASH_BYTE_SIZE); + serializer.add((const u8*)databaseAppendObject->requestHash.getData(), HASH_BYTE_SIZE); + serializer.add((const u8*)userToAddPublicKey.getData(), PUBLIC_KEY_NUM_BYTES); + serializer.add((u32)decryptedUserName.getDecryptedText().size); + serializer.add((const u8*)decryptedUserName.getDecryptedText().data, decryptedUserName.getDecryptedText().size); + fileAppend(decryptedDataFilePath, DataView(serializer.getBuffer().data(), serializer.getBuffer().size())); + databaseAppendObject->decryptedObject.data.data = new char[decryptedUserName.getDecryptedText().size]; + memcpy(databaseAppendObject->decryptedObject.data.data, decryptedUserName.getDecryptedText().data, decryptedUserName.getDecryptedText().size); + databaseAppendObject->decryptedObject.data.size = decryptedUserName.getDecryptedText().size; + databaseAppendObject->decryptedObject.operation = operation; + + Log::debug("Got add user object, timestamp: %zu, user added: %.*s", creationDate, nameLength, username.c_str()); + DatabaseAddUserRequest addUserRequest(&nodeHash, &databaseAppendObject->requestHash, creationDate, creatorUser, user, group); + if(database->onAddUserCallbackFunc) + database->onAddUserCallbackFunc(addUserRequest); + } + else + { + Log::error("Got unexpected operation %d", operation); + return false; + } + return true; + } + const dht::crypto::Identity& DatabaseStorage::getIdentity() const { return identity; diff --git a/src/Encryption.cpp b/src/Encryption.cpp index 1d6bfc0..9000519 100644 --- a/src/Encryption.cpp +++ b/src/Encryption.cpp @@ -5,6 +5,8 @@ namespace odhtdb { + static_assert(ENCRYPTION_CHECKSUM_BYTE_SIZE == crypto_aead_xchacha20poly1305_ietf_ABYTES, "Encryption checksum key size has changed for some reason, oops..."); + Encryption::Encryption(const DataView &data, const DataView &additionalData, const DataView &_key) { cipherTextLength = crypto_aead_xchacha20poly1305_ietf_ABYTES + data.size; diff --git a/src/FileUtils.cpp b/src/FileUtils.cpp index 3a78a63..6dc10b8 100644 --- a/src/FileUtils.cpp +++ b/src/FileUtils.cpp @@ -22,6 +22,7 @@ namespace odhtdb throw FileException(errMsg); } + flockfile(file); fseek(file, 0, SEEK_END); size_t fileSize = ftell(file); fseek(file, 0, SEEK_SET); diff --git a/src/Hash.cpp b/src/Hash.cpp index e46e5ba..364cddd 100644 --- a/src/Hash.cpp +++ b/src/Hash.cpp @@ -31,7 +31,7 @@ namespace odhtdb Hash::Hash(const Hash &other) { - memcpy(data, other.data, HASH_BYTE_SIZE); + memmove(data, other.data, HASH_BYTE_SIZE); } size_t Hash::operator()() const diff --git a/src/Signature.cpp b/src/Signature.cpp index 3fd52ee..9f44ad2 100644 --- a/src/Signature.cpp +++ b/src/Signature.cpp @@ -122,5 +122,11 @@ namespace odhtdb { } + + KeyPair::KeyPair(const KeyPair &other) + { + memmove(publicKey.data, other.publicKey.data, PUBLIC_KEY_NUM_BYTES); + memmove(privateKey.data, other.privateKey.data, PRIVATE_KEY_NUM_BYTES); + } } } diff --git a/src/User.cpp b/src/User.cpp index d157c74..58e350a 100644 --- a/src/User.cpp +++ b/src/User.cpp @@ -26,4 +26,14 @@ namespace odhtdb group->addUser(this); } } + + bool User::isAllowedToPerformAction(PermissionType action) const + { + for(Group *group : getGroups()) + { + if(group->getPermission().getFlag(action)) + return true; + } + return false; + } } diff --git a/tests/main.cpp b/tests/main.cpp index af3cf8d..7e8f489 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -143,19 +143,26 @@ int main() // TODO: Setup local bootstrap node for tests Database database("bootstrap.ring.cx", 4222, "storage"); - database.setOnCreateNodeCallback([](const DatabaseCreateNodeRequest &request) + int createNodeCounter = 0; + int addDataCounter = 0; + int addUserCounter = 0; + + database.setOnCreateNodeCallback([&createNodeCounter](const DatabaseCreateNodeRequest &request) { Log::debug("Create node callback"); + ++createNodeCounter; }); - database.setOnAddNodeCallback([](const DatabaseAddNodeRequest &request) + database.setOnAddNodeCallback([&addDataCounter](const DatabaseAddNodeRequest &request) { Log::debug("Add node callback"); + ++addDataCounter; }); - database.setOnAddUserCallback([](const DatabaseAddUserRequest &request) + database.setOnAddUserCallback([&addUserCounter](const DatabaseAddUserRequest &request) { Log::debug("Add user callback"); + ++addUserCounter; }); auto databaseCreateResponse = database.create("adminUserName", "latenight"); @@ -174,5 +181,9 @@ int main() this_thread::sleep_for(10ms); } + assertEquals(1, createNodeCounter); + assertEquals(2, addDataCounter); + assertEquals(1, addUserCounter); + return 0; } -- cgit v1.2.3