Add decryption (and caching) of data, even when adding encryption key after data has been added

1 files changed, 66 insertions, 83 deletions

diff --git a/src/Database.cpp b/src/Database.cpp
index bdf9104..6f864b4 100644
--- a/src/Database.cpp
+++ b/src/Database.cpp
@@ -87,7 +87,7 @@ namespace odhtdb
         onCreateNodeCallbackFunc(nullptr),
         onAddNodeCallbackFunc(nullptr),
         onAddUserCallbackFunc(nullptr),
-        databaseStorage(storageDir)
+        databaseStorage(this, storageDir)
     {
         node.run(port , {
             /*.dht_config = */{
@@ -143,7 +143,9 @@ namespace odhtdb
         // TODO: Use cached files and seed those. If none exists, request new files to seed.
         // If nobody requests my cached files in a long time, request new files to seed and remove cached files
         // (only if there are plenty of other seeders for the cached files. This could also cause race issue 
-        // where all nodes with a cached file delete it at same time)
+        // where all nodes with a cached file delete it at same time).
+        
+        databaseStorage.setNodeDecryptionKey(*nodeToSeed.getRequestHash(), DataView(nodeToSeed.getNodeEncryptionKey()->data, nodeToSeed.getNodeEncryptionKey()->size));
         
         Log::debug("Seeding key: %s", nodeToSeed.getRequestHash()->toString().c_str());
         DhtKey dhtKey(*nodeToSeed.getRequestHash());
@@ -268,12 +270,8 @@ namespace odhtdb
             Encryption encryptedBody(DataView(encryptedSerializer.getBuffer().data(), encryptedSerializer.getBuffer().size()));
             DataView requestData = combine(serializer, encryptedBody);
             shared_ptr<Hash> hashRequestKey = make_shared<Hash>(requestData.data, requestData.size);
-            databaseStorage.createStorage(*hashRequestKey, adminGroup, timestampMicroseconds, (const u8*)requestData.data, requestData.size);
-
-            string nodeNameCopy(nodeName);
-            DatabaseCreateNodeRequest createNodeRequest(hashRequestKey.get(), timestampMicroseconds, nodeAdminUser, move(nodeNameCopy));
-            if(onCreateNodeCallbackFunc)
-                onCreateNodeCallbackFunc(createNodeRequest);
+            databaseStorage.setNodeDecryptionKey(*hashRequestKey, DataView(encryptedBody.getKey().data, encryptedBody.getKey().size));
+            databaseStorage.createStorage(*hashRequestKey, adminGroup, timestampMicroseconds, (const u8*)requestData.data, requestData.size, serializer.getBuffer().size());
 
             stagedCreateObjects.emplace_back(make_unique<StagedObject>(requestData, hashRequestKey));
             
@@ -290,6 +288,19 @@ namespace odhtdb
 
     void Database::addData(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, DataView dataToAdd)
     {
+        if(!userToPerformActionWith->isAllowedToPerformAction(PermissionType::ADD_DATA))
+        {
+            // TODO: User might have permission to perform operation, but we haven't got the packet that adds user to the group with the permission,
+            // or we haven't received the packet that modifies group with the permission to perform the operation.
+            // This also means that an user can be in a group that has permission to perform the operation and then later be removed from it,
+            // and remote peers would accept our request to perform operation if they haven't received the operation that removes the user from the group.
+            // How to handle this?
+            string errMsg = "User ";
+            errMsg += userToPerformActionWith->getName();
+            errMsg += " is not allowed to perform the operation: ADD_USER";
+            throw PermissionDeniedException(errMsg);
+        }
+        
         sibs::SafeSerializer serializer;
         serializer.add(DATABASE_ADD_PACKET_STRUCTURE_VERSION);
         // TODO: Append fractions to get real microseconds time
@@ -301,14 +312,11 @@ namespace odhtdb
         Encryption encryptedBody(dataToAdd, DataView(), encryptionKey);
         DataView requestData = combine(serializer, encryptedBody);
         string signedRequestData = userToPerformActionWith->getPrivateKey().sign(requestData);
-        free(requestData.data);
         DataView stagedAddObject = combine(userToPerformActionWith->getPublicKey(), signedRequestData);
         Hash requestDataHash(stagedAddObject.data, stagedAddObject.size);
-        databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, userToPerformActionWith, timestampMicroseconds, (u8*)stagedAddObject.data, stagedAddObject.size);
-
-        DatabaseAddNodeRequest addNodeRequest(&*nodeInfo.getRequestHash(), &requestDataHash, timestampMicroseconds, userToPerformActionWith, dataToAdd);
-        if(onAddNodeCallbackFunc)
-            onAddNodeCallbackFunc(addNodeRequest);
+        DataView encryptedDataView((char*)requestData.data + serializer.getBuffer().size(), requestData.size - serializer.getBuffer().size());
+        databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, DatabaseOperation::ADD_DATA, userToPerformActionWith, timestampMicroseconds, (u8*)stagedAddObject.data, stagedAddObject.size, encryptedDataView);
+        delete (char*)requestData.data;
 
         stagedAddObjects.emplace_back(make_unique<StagedObject>(stagedAddObject, nodeInfo.getRequestHash()));
     }
@@ -350,8 +358,14 @@ namespace odhtdb
         serializer.add(DatabaseOperation::ADD_USER);
         
         assert(userToAddName.size() <= 255);
+        usize serializedEncryptedDataOffset = serializer.getBuffer().size();
         serializer.add((u8)userToAddName.size());
-        serializer.add((u8*)userToAddName.data(), userToAddName.size());
+        DataView encryptionKey(nodeInfo.getNodeEncryptionKey()->data, ENCRYPTION_KEY_BYTE_SIZE);
+        Encryption encryptedUserName(DataView((void*)userToAddName.data(), userToAddName.size()), DataView(), encryptionKey);
+        serializer.add((u8*)encryptedUserName.getNonce().data, ENCRYPTION_NONCE_BYTE_SIZE);
+        assert(encryptedUserName.getCipherText().size == ENCRYPTION_CHECKSUM_BYTE_SIZE + userToAddName.size());
+        serializer.add((u8*)encryptedUserName.getCipherText().data, ENCRYPTION_CHECKSUM_BYTE_SIZE + userToAddName.size());
+        usize serializedEncryptedDataSize = serializer.getBuffer().size() - serializedEncryptedDataOffset;
         serializer.add((u8*)userToAddPublicKey.getData(), PUBLIC_KEY_NUM_BYTES);
         serializer.add((uint8_t*)groupToAddUserTo->getId().data, groupToAddUserTo->getId().size);
         
@@ -359,13 +373,10 @@ namespace odhtdb
         string signedRequestData = userToPerformActionWith->getPrivateKey().sign(requestData);
         DataView stagedAddObject = combine(userToPerformActionWith->getPublicKey(), signedRequestData);
         Hash requestDataHash(stagedAddObject.data, stagedAddObject.size);
-        databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, userToPerformActionWith, timestampMicroseconds, (u8*)stagedAddObject.data, stagedAddObject.size);
+        DataView encryptedDataView(nullptr, 0);
         auto userToAdd = RemoteUser::create(userToAddPublicKey, userToAddName, groupToAddUserTo);
         databaseStorage.addUser(*nodeInfo.getRequestHash(), userToAdd);
-
-        DatabaseAddUserRequest addUserRequest(&*nodeInfo.getRequestHash(), &requestDataHash, timestampMicroseconds, userToPerformActionWith, userToAdd, groupToAddUserTo);
-        if(onAddUserCallbackFunc)
-            onAddUserCallbackFunc(addUserRequest);
+        databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, DatabaseOperation::ADD_USER, userToPerformActionWith, timestampMicroseconds, (u8*)stagedAddObject.data, stagedAddObject.size, encryptedDataView);
 
         stagedAddObjects.emplace_back(make_unique<StagedObject>(stagedAddObject, nodeInfo.getRequestHash()));
     }
@@ -468,8 +479,8 @@ namespace odhtdb
         deserializer.extract((u8*)creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
         Signature::PublicKey userPublicKey(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
         
-        uint8_t adminGroupId[16];
-        deserializer.extract(adminGroupId, 16);
+        uint8_t adminGroupId[GROUP_ID_LENGTH];
+        deserializer.extract(adminGroupId, GROUP_ID_LENGTH);
         
         if(deserializer.getSize() < ENCRYPTION_NONCE_BYTE_SIZE)
             throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)");
@@ -477,39 +488,7 @@ namespace odhtdb
         auto adminGroup = new Group("administrator", adminGroupId, ADMIN_PERMISSION);
         // TODO: Username is encrypted, we dont know it... unless we have encryption key, in which case we should modify the user name and set it
         auto creatorUser = RemoteUser::create(userPublicKey, "ENCRYPTED USER NAME", adminGroup);
-        databaseStorage.createStorage(hash, adminGroup, creationDate, value->data.data(), value->data.size());
-        
-        u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE];
-        deserializer.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE);
-        
-        DataView dataToDecrypt((void*)deserializer.getBuffer(), deserializer.getSize());
-        Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(encryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE));
-        sibs::SafeDeserializer bodyDeserializer((const u8*)decryptedBody.getDecryptedText().data, decryptedBody.getDecryptedText().size);
-        
-        u8 creatorNameLength = bodyDeserializer.extract<u8>();
-        string creatorName; // TODO: Add this user name to storage added above
-        creatorName.resize(creatorNameLength);
-        bodyDeserializer.extract((u8*)&creatorName[0], creatorNameLength);
-        
-        u8 nameLength = bodyDeserializer.extract<u8>();
-        string name;
-        name.resize(nameLength);
-        bodyDeserializer.extract((u8*)&name[0], nameLength);
-        
-        Log::debug("Got create object, name: %s", name.c_str());
-        DatabaseCreateNodeRequest createNodeRequest(&hash, creationDate, creatorUser, move(name));
-        if(onCreateNodeCallbackFunc)
-            onCreateNodeCallbackFunc(createNodeRequest);
-    }
-    
-    bool isUserAllowedToAddData(const User *user)
-    {
-        for(Group *group : user->getGroups())
-        {
-            if(group->getPermission().getFlag(PermissionType::ADD_DATA))
-                return true;
-        }
-        return false;
+        databaseStorage.createStorage(hash, adminGroup, creationDate, value->data.data(), value->data.size(), value->data.size() - deserializer.getSize());
     }
 
     void Database::deserializeAddRequest(const shared_ptr<dht::Value> &value, const Hash &requestDataHash, const std::shared_ptr<Hash> &nodeHash, const shared_ptr<OwnedMemory> encryptionKey)
@@ -552,21 +531,24 @@ namespace odhtdb
         }
 #endif
         auto creatorUser = databaseStorage.getUserByPublicKey(*nodeHash, creatorPublicKey);
-        // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine.
-        // TODO: We might receive 'add' data packet before 'create'. If that happens, we should put it in quarantine and process it later.
-        databaseStorage.appendStorage(*nodeHash, requestDataHash, creatorUser, creationDate, value->data.data(), value->data.size());
+        if(!creatorUser)
+        {
+            // TODO: Add to quarantine
+            string errMsg = "User with public key ";
+            errMsg += creatorPublicKey.toString();
+            errMsg += " does not exist in code ";
+            errMsg += nodeHash->toString();
+            throw sibs::DeserializeException(errMsg);
+        }
+        
+        DataView encryptedDataView((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize());
         
         if(operation == DatabaseOperation::ADD_DATA)
         {
             if(deserializerUnsigned.getSize() < ENCRYPTION_NONCE_BYTE_SIZE)
                 throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)");
             
-            u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE];
-            deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE);
-            DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize());
-            Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(encryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE));
-            
-            if(!isUserAllowedToAddData(creatorUser))
+            if(!creatorUser->isAllowedToPerformAction(PermissionType::ADD_DATA))
             {
                 // TODO: User might have permission to perform operation, but we haven't got the packet that adds user to the group with the permission,
                 // or we haven't received the packet that modifies group with the permission to perform the operation.
@@ -575,34 +557,36 @@ namespace odhtdb
                 // How to handle this?
                 string errMsg = "User ";
                 errMsg += creatorUser->getName();
-                errMsg += " is not allowed to perform the operation: ";
-                errMsg += to_string((u8)operation);
+                errMsg += " is not allowed to add data to node ";
+                errMsg += nodeHash->toString();
                 throw PermissionDeniedException(errMsg);
             }
             
-            Log::debug("Got add object, timestamp: %zu, data: %.*s", creationDate, decryptedBody.getDecryptedText().size, decryptedBody.getDecryptedText().data);
-            const DatabaseAddNodeRequest addNodeRequest(&*nodeHash, &requestDataHash, creationDate, creatorUser, decryptedBody.getDecryptedText());
-            if(onAddNodeCallbackFunc)
-                onAddNodeCallbackFunc(addNodeRequest);
+            // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine.
+            // TODO: We might receive 'add' data packet before 'create'. If that happens, we should put it in quarantine and process it later.
+            databaseStorage.appendStorage(*nodeHash, requestDataHash, operation, creatorUser, creationDate, value->data.data(), value->data.size(), encryptedDataView);
         }
         else if(operation == DatabaseOperation::ADD_USER)
-        { 
+        {
             u8 nameLength = deserializerUnsigned.extract<u8>();
-            string name;
-            name.resize(nameLength);
-            deserializerUnsigned.extract((u8*)&name[0], nameLength);
+            
+            u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE];
+            deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE);
+            DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), ENCRYPTION_CHECKSUM_BYTE_SIZE + nameLength);
+            
+            sibs::SafeDeserializer deserializerSkippedEncryptedData(deserializerUnsigned.getBuffer() + ENCRYPTION_CHECKSUM_BYTE_SIZE + nameLength, PUBLIC_KEY_NUM_BYTES + GROUP_ID_LENGTH);
             
             char userToAddPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
-            deserializerUnsigned.extract((u8*)userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
+            deserializerSkippedEncryptedData.extract((u8*)userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
             Signature::PublicKey userToAddPublicKey(userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
             
-            uint8_t groupId[16];
-            deserializerUnsigned.extract(groupId, 16);
+            uint8_t groupId[GROUP_ID_LENGTH];
+            deserializerSkippedEncryptedData.extract(groupId, GROUP_ID_LENGTH);
             
             auto group = databaseStorage.getGroupById(*nodeHash, groupId);
             if(group)
             {
-                auto user = RemoteUser::create(userToAddPublicKey, name, group);
+                auto user = RemoteUser::create(userToAddPublicKey, "ENCRYPTED USER NAME", group);
                 // TODO: What if we receive packets in wrong order? (maliciously or non-maliciously). You would be able to register a user to a group with given name
                 // and further registration would be dropped (even if that is the correct one)
                 if(!databaseStorage.addUser(*nodeHash, user)) return;
@@ -622,10 +606,9 @@ namespace odhtdb
                     throw PermissionDeniedException(errMsg);
                 }
                 
-                Log::debug("Got add user object, timestamp: %zu, user added: %.*s", creationDate, nameLength, name.c_str());
-                DatabaseAddUserRequest addUserRequest(&*nodeHash, &requestDataHash, creationDate, creatorUser, user, group);
-                if(onAddUserCallbackFunc)
-                    onAddUserCallbackFunc(addUserRequest);
+                // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine.
+                // TODO: We might receive 'add' data packet before 'create'. If that happens, we should put it in quarantine and process it later.
+                databaseStorage.appendStorage(*nodeHash, requestDataHash, operation, creatorUser, creationDate, value->data.data(), value->data.size(), encryptedDataView);
             }
             else
             {
@@ -637,7 +620,7 @@ namespace odhtdb
             string errMsg = "Got unexpected operation: ";
             errMsg += to_string((u8)operation);
             throw sibs::DeserializeException(errMsg);
-        }  
+        }
     }
 
     bool Database::listenCreateData(shared_ptr<dht::Value> value, const Hash &hash, const shared_ptr<OwnedMemory> encryptionKey)