Store database storage to files, also loading

1 files changed, 55 insertions, 52 deletions

diff --git a/src/Database.cpp b/src/Database.cpp
index 41b0d73..226aa05 100644
--- a/src/Database.cpp
+++ b/src/Database.cpp
@@ -60,9 +60,9 @@ namespace odhtdb
         return DataView(result, allocationSize);
     }
     
-    DatabaseCreateResponse::DatabaseCreateResponse(LocalUser *_nodeAdminUser, const shared_ptr<char*> &_key, const shared_ptr<Hash> &_hash) : 
+    DatabaseCreateResponse::DatabaseCreateResponse(LocalUser *_nodeAdminUser, const shared_ptr<OwnedMemory> &_key, const shared_ptr<Hash> &_hash) : 
         nodeAdminUser(_nodeAdminUser),
-        key(move(_key)),
+        key(_key),
         hash(_hash)
     {
         
@@ -73,7 +73,7 @@ namespace odhtdb
         return nodeAdminUser;
     }
         
-    const shared_ptr<char*> DatabaseCreateResponse::getNodeEncryptionKey() const
+    const shared_ptr<OwnedMemory> DatabaseCreateResponse::getNodeEncryptionKey() const
     {
         return key;
     }
@@ -83,10 +83,11 @@ namespace odhtdb
         return hash;
     }
     
-    Database::Database(const char *bootstrapNodeAddr, u16 port, boost::filesystem::path storageDir) : 
+    Database::Database(const char *bootstrapNodeAddr, u16 port, const boost::filesystem::path &storageDir) : 
         onCreateNodeCallbackFunc(nullptr),
         onAddNodeCallbackFunc(nullptr),
-        onAddUserCallbackFunc(nullptr)
+        onAddUserCallbackFunc(nullptr),
+        databaseStorage(storageDir)
     {
         // TODO: Cache this in storage. It takes pretty long time to generate new identity
         auto identity = dht::crypto::generateIdentity();
@@ -147,43 +148,43 @@ namespace odhtdb
         node.join();
     }
 
-    void Database::seed(const shared_ptr<Hash> hash, const shared_ptr<char*> encryptionKey)
+    void Database::seed(const DatabaseNode &nodeToSeed)
     {
         // TODO: Use cached files and seed those. If none exists, request new files to seed.
         // If nobody requests my cached files in a long time, request new files to seed and remove cached files
         // (only if there are plenty of other seeders for the cached files. This could also cause race issue 
         // where all nodes with a cached file delete it at same time)
         
-        Log::debug("Seeding key: %s", hash->toString().c_str());
-        DhtKey dhtKey(*hash);
+        Log::debug("Seeding key: %s", nodeToSeed.getRequestHash()->toString().c_str());
+        DhtKey dhtKey(*nodeToSeed.getRequestHash());
         
-        node.listen(dhtKey.getNewDataListenerKey(), [this, hash, encryptionKey](const shared_ptr<Value> &value)
+        node.listen(dhtKey.getNewDataListenerKey(), [this, nodeToSeed](const shared_ptr<Value> &value)
         {
             Log::debug("Seed: New data listener received data...");
             const Hash requestHash(value->data.data(), value->data.size());
-            if(requestHash == *hash)
+            if(requestHash == *nodeToSeed.getRequestHash())
                 return true;
                 //return listenCreateData(value, requestHash, encryptionKey);
             else
-                return listenAddData(value, requestHash, encryptionKey);
+                return listenAddData(value, requestHash, nodeToSeed.getRequestHash(), nodeToSeed.getNodeEncryptionKey());
         });
         
         u8 responseKey[OPENDHT_INFOHASH_LEN];
         randombytes_buf(responseKey, OPENDHT_INFOHASH_LEN);
         
         // TODO: If this response key is spammed, generate a new one.
-        node.listen(InfoHash(responseKey, OPENDHT_INFOHASH_LEN), [this, hash, encryptionKey](const shared_ptr<Value> &value)
+        node.listen(InfoHash(responseKey, OPENDHT_INFOHASH_LEN), [this, nodeToSeed](const shared_ptr<Value> &value)
         {
             const Hash requestHash(value->data.data(), value->data.size());
-            if(requestHash == *hash)
-                return listenCreateData(value, requestHash, encryptionKey);
+            if(requestHash == *nodeToSeed.getRequestHash())
+                return listenCreateData(value, requestHash, nodeToSeed.getNodeEncryptionKey());
             else
-                return listenAddData(value, requestHash, encryptionKey);
+                return listenAddData(value, requestHash, nodeToSeed.getRequestHash(), nodeToSeed.getNodeEncryptionKey());
         });
         
         // TODO: Before listening on this key, we should check how many remote peers are also providing this data.
         // This is to prevent too many peers from responding to a request to get old data.
-        node.listen(dhtKey.getRequestOldDataKey(), [this, hash](const shared_ptr<Value> &value)
+        node.listen(dhtKey.getRequestOldDataKey(), [this, nodeToSeed](const shared_ptr<Value> &value)
         {
             Log::debug("Request: Got request to send old data");
             try
@@ -193,10 +194,10 @@ namespace odhtdb
                 u8 requestResponseKey[OPENDHT_INFOHASH_LEN];
                 deserializer.extract(requestResponseKey, OPENDHT_INFOHASH_LEN);
                 
-                auto requestedData = databaseStorage.getStorage(*hash);
+                auto requestedData = databaseStorage.getStorage(*nodeToSeed.getRequestHash());
                 if(!requestedData)
                 {
-                    Log::warn("No data found for hash %s, unable to serve peer", hash->toString().c_str());
+                    Log::warn("No data found for hash %s, unable to serve peer", nodeToSeed.getRequestHash()->toString().c_str());
                     return true;
                 }
                 
@@ -229,7 +230,7 @@ namespace odhtdb
         });
         
         sibs::SafeSerializer serializer;
-        serializer.add((u64)0); // Timestamp in microseconds, fetch data newer than this
+        serializer.add((u64)0); // Timestamp in microseconds, fetch data newer than this. // TODO: Get timestamp from database storage
         serializer.add(responseKey, OPENDHT_INFOHASH_LEN);
         node.put(dhtKey.getRequestOldDataKey(), Value(serializer.getBuffer().data(), serializer.getBuffer().size()), [](bool ok)
         {
@@ -241,13 +242,13 @@ namespace odhtdb
         //node.listen(ADD_DATA_HASH, bind(&Database::listenAddData, this, _1));
     }
 
-    unique_ptr<DatabaseCreateResponse> Database::create(const std::string &ownerName, const std::string &nodeName)
+    unique_ptr<DatabaseCreateResponse> Database::create(const string &ownerName, const std::string &ownerPlainPassword, const string &nodeName)
     {
         // TODO: Should this be declared static? is there any difference in behavior/performance?
         boost::uuids::random_generator uuidGen;
         auto adminGroupId = uuidGen();
         auto adminGroup = new Group("administrator", adminGroupId.data, ADMIN_PERMISSION);
-        LocalUser *nodeAdminUser = LocalUser::create(Signature::KeyPair(), ownerName, adminGroup);
+        LocalUser *nodeAdminUser = LocalUser::create(Signature::KeyPair(), ownerName, adminGroup, ownerPlainPassword);
         
         // Header
         sibs::SafeSerializer serializer;
@@ -281,10 +282,10 @@ namespace odhtdb
 
             stagedCreateObjects.emplace_back(make_unique<StagedObject>(requestData, hashRequestKey));
             
-            assert(encryptedBody.getKey().size == KEY_BYTE_SIZE);
-            char *key = new char[encryptedBody.getKey().size];
-            memcpy(key, encryptedBody.getKey().data, encryptedBody.getKey().size);
-            return make_unique<DatabaseCreateResponse>(nodeAdminUser, make_shared<char*>(key), hashRequestKey);
+            assert(encryptedBody.getKey().size == ENCRYPTION_KEY_BYTE_SIZE);
+            auto key = make_shared<OwnedMemory>(new char[encryptedBody.getKey().size], encryptedBody.getKey().size);
+            memcpy(key->data, encryptedBody.getKey().data, encryptedBody.getKey().size);
+            return make_unique<DatabaseCreateResponse>(nodeAdminUser, move(key), hashRequestKey);
         }
         catch (EncryptionException &e)
         {
@@ -301,7 +302,7 @@ namespace odhtdb
         serializer.add(timestampMicroseconds);
         serializer.add(DatabaseOperation::ADD_DATA);
         
-        DataView encryptionKey(*nodeInfo.getNodeEncryptionKey(), KEY_BYTE_SIZE);
+        DataView encryptionKey(nodeInfo.getNodeEncryptionKey()->data, ENCRYPTION_KEY_BYTE_SIZE);
         Encryption encryptedBody(dataToAdd, DataView(), encryptionKey);
         DataView requestData = combine(serializer, encryptedBody);
         string signedRequestData = userToPerformActionWith->getPrivateKey().sign(requestData);
@@ -334,7 +335,7 @@ namespace odhtdb
         return nullptr;
     }
     
-    void Database::addUserToGroup(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, const std::string &userToAddName, const Signature::PublicKey &userToAddPublicKey, Group *groupToAddUserTo)
+    void Database::addUser(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, const string &userToAddName, const Signature::PublicKey &userToAddPublicKey, Group *groupToAddUserTo)
     {
         auto groupWithAddUserRights = getGroupWithRightsToAddUserToGroup(userToPerformActionWith->getGroups(), groupToAddUserTo);
         if(!groupWithAddUserRights)
@@ -365,7 +366,7 @@ namespace odhtdb
         Hash requestDataHash(stagedAddObject.data, stagedAddObject.size);
         databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, userToPerformActionWith, timestampMicroseconds, (u8*)stagedAddObject.data, stagedAddObject.size);
         auto userToAdd = RemoteUser::create(userToAddPublicKey, userToAddName, groupToAddUserTo);
-        databaseStorage.addUser(userToAdd, *nodeInfo.getRequestHash());
+        databaseStorage.addUser(*nodeInfo.getRequestHash(), userToAdd);
 
         DatabaseAddUserRequest addUserRequest(&*nodeInfo.getRequestHash(), &requestDataHash, timestampMicroseconds, userToPerformActionWith, userToAdd, groupToAddUserTo);
         if(onAddUserCallbackFunc)
@@ -446,7 +447,7 @@ namespace odhtdb
         return timestamp;
     }
 
-    void Database::deserializeCreateRequest(const std::shared_ptr<dht::Value> &value, const Hash &hash, const shared_ptr<char*> encryptionKey)
+    void Database::deserializeCreateRequest(const shared_ptr<dht::Value> &value, const Hash &hash, const shared_ptr<OwnedMemory> encryptionKey)
     {
         sibs::SafeDeserializer deserializer(value->data.data(), value->data.size());
         u16 packetStructureVersion = deserializer.extract<u16>();
@@ -472,7 +473,7 @@ namespace odhtdb
         uint8_t adminGroupId[16];
         deserializer.extract(adminGroupId, 16);
         
-        if(deserializer.getSize() < NONCE_BYTE_SIZE)
+        if(deserializer.getSize() < ENCRYPTION_NONCE_BYTE_SIZE)
             throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)");
         
         auto adminGroup = new Group("administrator", adminGroupId, ADMIN_PERMISSION);
@@ -480,11 +481,11 @@ namespace odhtdb
         auto creatorUser = RemoteUser::create(userPublicKey, "ENCRYPTED USER NAME", adminGroup);
         databaseStorage.createStorage(hash, adminGroup, creationDate, value->data.data(), value->data.size());
         
-        u8 nonce[NONCE_BYTE_SIZE];
-        deserializer.extract(nonce, NONCE_BYTE_SIZE);
+        u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE];
+        deserializer.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE);
         
         DataView dataToDecrypt((void*)deserializer.getBuffer(), deserializer.getSize());
-        Decryption decryptedBody(dataToDecrypt, DataView(nonce, NONCE_BYTE_SIZE), DataView(*encryptionKey, KEY_BYTE_SIZE));
+        Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(encryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE));
         sibs::SafeDeserializer bodyDeserializer((const u8*)decryptedBody.getDecryptedText().data, decryptedBody.getDecryptedText().size);
         
         u8 creatorNameLength = bodyDeserializer.extract<u8>();
@@ -513,7 +514,7 @@ namespace odhtdb
         return false;
     }
 
-    void Database::deserializeAddRequest(const std::shared_ptr<dht::Value> &value, const Hash &requestDataHash, const shared_ptr<char*> encryptionKey)
+    void Database::deserializeAddRequest(const shared_ptr<dht::Value> &value, const Hash &requestDataHash, const std::shared_ptr<Hash> &nodeHash, const shared_ptr<OwnedMemory> encryptionKey)
     {
         sibs::SafeDeserializer deserializer(value->data.data(), value->data.size());
         char creatorPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
@@ -541,7 +542,7 @@ namespace odhtdb
             throw sibs::DeserializeException("Packet is from the future");
         
         DatabaseOperation operation = deserializerUnsigned.extract<DatabaseOperation>();
-        
+#if 0
         const Hash *node = databaseStorage.getNodeByUserPublicKey(creatorPublicKey);
         if(!node)
         {
@@ -551,21 +552,21 @@ namespace odhtdb
             databaseStorage.addToQuarantine(requestDataHash, creatorPublicKey, creationDate, value->data.data(), value->data.size());
             throw RequestQuarantineException();
         }
-        
-        auto creatorUser = databaseStorage.getUserByPublicKey(creatorPublicKey);
+#endif
+        auto creatorUser = databaseStorage.getUserByPublicKey(*nodeHash, creatorPublicKey);
         // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine.
         // TODO: We might receive 'add' data packet before 'create'. If that happens, we should put it in quarantine and process it later.
-        databaseStorage.appendStorage(*node, requestDataHash, creatorUser, creationDate, value->data.data(), value->data.size());
+        databaseStorage.appendStorage(*nodeHash, requestDataHash, creatorUser, creationDate, value->data.data(), value->data.size());
         
         if(operation == DatabaseOperation::ADD_DATA)
         {
-            if(deserializerUnsigned.getSize() < NONCE_BYTE_SIZE)
+            if(deserializerUnsigned.getSize() < ENCRYPTION_NONCE_BYTE_SIZE)
                 throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)");
             
-            u8 nonce[NONCE_BYTE_SIZE];
-            deserializerUnsigned.extract(nonce, NONCE_BYTE_SIZE);
+            u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE];
+            deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE);
             DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize());
-            Decryption decryptedBody(dataToDecrypt, DataView(nonce, NONCE_BYTE_SIZE), DataView(*encryptionKey, KEY_BYTE_SIZE));
+            Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(encryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE));
             
             if(!isUserAllowedToAddData(creatorUser))
             {
@@ -582,7 +583,7 @@ namespace odhtdb
             }
             
             Log::debug("Got add object, timestamp: %zu, data: %.*s", creationDate, decryptedBody.getDecryptedText().size, decryptedBody.getDecryptedText().data);
-            const DatabaseAddNodeRequest addNodeRequest(node, &requestDataHash, creationDate, creatorUser, decryptedBody.getDecryptedText());
+            const DatabaseAddNodeRequest addNodeRequest(&*nodeHash, &requestDataHash, creationDate, creatorUser, decryptedBody.getDecryptedText());
             if(onAddNodeCallbackFunc)
                 onAddNodeCallbackFunc(addNodeRequest);
         }
@@ -600,11 +601,13 @@ namespace odhtdb
             uint8_t groupId[16];
             deserializerUnsigned.extract(groupId, 16);
             
-            auto group = databaseStorage.getGroupById(groupId);
+            auto group = databaseStorage.getGroupById(*nodeHash, groupId);
             if(group)
             {
                 auto user = RemoteUser::create(userToAddPublicKey, name, group);
-                databaseStorage.addUser(user, *node);
+                // TODO: What if we receive packets in wrong order? (maliciously or non-maliciously). You would be able to register a user to a group with given name
+                // and further registration would be dropped (even if that is the correct one)
+                if(!databaseStorage.addUser(*nodeHash, user)) return;
                 
                 auto creatorUserGroupWithRights = getGroupWithRightsToAddUserToGroup(creatorUser->getGroups(), group);
                 if(!creatorUserGroupWithRights)
@@ -622,7 +625,7 @@ namespace odhtdb
                 }
                 
                 Log::debug("Got add user object, timestamp: %zu, user added: %.*s", creationDate, nameLength, name.c_str());
-                DatabaseAddUserRequest addUserRequest(node, &requestDataHash, creationDate, creatorUser, user, group);
+                DatabaseAddUserRequest addUserRequest(&*nodeHash, &requestDataHash, creationDate, creatorUser, user, group);
                 if(onAddUserCallbackFunc)
                     onAddUserCallbackFunc(addUserRequest);
             }
@@ -639,7 +642,7 @@ namespace odhtdb
         }  
     }
 
-    bool Database::listenCreateData(std::shared_ptr<dht::Value> value, const Hash &hash, const shared_ptr<char*> encryptionKey)
+    bool Database::listenCreateData(shared_ptr<dht::Value> value, const Hash &hash, const shared_ptr<OwnedMemory> encryptionKey)
     {
         Log::debug("Got create data");
         try
@@ -655,12 +658,12 @@ namespace odhtdb
         return true;
     }
 
-    bool Database::listenAddData(std::shared_ptr<dht::Value> value, const Hash &requestDataHash, const shared_ptr<char*> encryptionKey)
+    bool Database::listenAddData(shared_ptr<dht::Value> value, const Hash &requestDataHash, const std::shared_ptr<Hash> nodeHash, const shared_ptr<OwnedMemory> encryptionKey)
     {
         Log::debug("Got add data");
         try
         {
-            deserializeAddRequest(value, requestDataHash, encryptionKey);
+            deserializeAddRequest(value, requestDataHash, nodeHash, encryptionKey);
             //Log::debug("Got add object, timestamp: %zu", addObject.timestamp);
         }
         catch (RequestQuarantineException &e)
@@ -674,17 +677,17 @@ namespace odhtdb
         return true;
     }
 
-    void Database::setOnCreateNodeCallback(std::function<void(const DatabaseCreateNodeRequest&)> callbackFunc)
+    void Database::setOnCreateNodeCallback(function<void(const DatabaseCreateNodeRequest&)> callbackFunc)
     {
         onCreateNodeCallbackFunc = callbackFunc;
     }
 
-    void Database::setOnAddNodeCallback(std::function<void(const DatabaseAddNodeRequest&)> callbackFunc)
+    void Database::setOnAddNodeCallback(function<void(const DatabaseAddNodeRequest&)> callbackFunc)
     {
         onAddNodeCallbackFunc = callbackFunc;
     }
 
-    void Database::setOnAddUserCallback(std::function<void(const DatabaseAddUserRequest&)> callbackFunc)
+    void Database::setOnAddUserCallback(function<void(const DatabaseAddUserRequest&)> callbackFunc)
     {
         onAddUserCallbackFunc = callbackFunc;
     }