Replace files with sqlite

Using sqlite because sqlite has transactions, storing/loading from files
automatically, unloading data that is not accessed often.

Removed cosmetic data (node name, username). They can be added using
addData by the application that uses odhtdb instead.

Database callback functions can now be called with stored data using
database.loadNode function.

TODO: Add local user storage (with password) back, it has been temorary
disabled

1 files changed, 60 insertions, 194 deletions

diff --git a/src/Database.cpp b/src/Database.cpp
index 41d3798..88ac8e4 100644
--- a/src/Database.cpp
+++ b/src/Database.cpp
@@ -61,7 +61,7 @@ namespace odhtdb
         return DataView(result, allocationSize);
     }
     
-    DatabaseCreateResponse::DatabaseCreateResponse(LocalUser *_nodeAdminUser, const shared_ptr<OwnedMemory> &_key, const shared_ptr<Hash> &_hash) : 
+    DatabaseCreateResponse::DatabaseCreateResponse(LocalUser *_nodeAdminUser, shared_ptr<OwnedMemory> _key, shared_ptr<Hash> _hash) : 
         nodeAdminUser(_nodeAdminUser),
         key(_key),
         hash(_hash)
@@ -84,10 +84,10 @@ namespace odhtdb
         return hash;
     }
     
-    Database::Database(const char *bootstrapNodeAddr, u16 port, const boost::filesystem::path &storageDir) : 
-        onCreateNodeCallbackFunc(nullptr),
-        onAddNodeCallbackFunc(nullptr),
-        onAddUserCallbackFunc(nullptr),
+    Database::Database(const char *bootstrapNodeAddr, u16 port, const boost::filesystem::path &storageDir, DatabaseCallbackFuncs callbackFuncs) : 
+        onCreateNodeCallbackFunc(callbackFuncs.createNodeCallbackFunc),
+        onAddNodeCallbackFunc(callbackFuncs.addNodeCallbackFunc),
+        onAddUserCallbackFunc(callbackFuncs.addUserCallbackFunc),
         databaseStorage(this, storageDir)
     {
         node.run(port , {
@@ -218,38 +218,35 @@ namespace odhtdb
                 u64 dataStartTimestamp = deserializer.extract<u64>();
                 u8 requestResponseKey[OPENDHT_INFOHASH_LEN];
                 deserializer.extract(requestResponseKey, OPENDHT_INFOHASH_LEN);
-                
-                auto requestedData = databaseStorage.getStorage(*nodeToSeed.getRequestHash());
-                if(!requestedData)
-                {
-                    Log::debug("No data found for hash %s, unable to serve peer", nodeToSeed.getRequestHash()->toString().c_str());
-                    return true;
-                }
-                
                 InfoHash requestResponseInfoHash(requestResponseKey, OPENDHT_INFOHASH_LEN);
                 
                 if(dataStartTimestamp == 0)
                 {
-                    Log::debug("Request: Sent create packet to requesting peer");
-                    node.put(requestResponseInfoHash, Value((u8*)requestedData->data.data, requestedData->data.size), [](bool ok)
+                    databaseStorage.fetchNodeRaw(*nodeToSeed.getRequestHash(), [this, requestResponseInfoHash](const DataView rawData)
                     {
-                        if(!ok)
-                            Log::error("Failed to put response for old data for 'create' data");
+                        Log::debug("Request: Sent create packet to requesting peer");
+                        Value value((u8*)rawData.data, rawData.size);
+                        node.put(requestResponseInfoHash, move(value), [](bool ok)
+                        {
+                            if(!ok)
+                                Log::error("Failed to put response for old data for 'create' data");
+                        });
                     });
                 }
                 
-                for(auto requestedObject : requestedData->objects)
+                databaseStorage.fetchNodeAddDataRaw(*nodeToSeed.getRequestHash(), [this, requestResponseInfoHash](const DataView rawData)
                 {
-                    node.put(requestResponseInfoHash, Value((u8*)requestedObject->data.data, requestedObject->data.size), [](bool ok)
+                    Value value((u8*)rawData.data, rawData.size);
+                    node.put(requestResponseInfoHash, move(value), [](bool ok)
                     {
                         if(!ok)
                             Log::error("Failed to put response for old data for 'add' data");
                     });
-                }
+                });
             }
-            catch (sibs::DeserializeException &e)
+            catch (std::exception &e)
             {
-                Log::warn("Failed to deserialize 'get old data' request: %s", e.what());
+                Log::warn("Failed while serving peer, error: %s", e.what());
             }
             return true;
         });
@@ -283,19 +280,25 @@ namespace odhtdb
             seedInfoMap.erase(seedInfoIt);
         }
     }
+    
+    void Database::loadNode(const Hash &nodeHash)
+    {
+        databaseStorage.loadNode(nodeHash);
+    }
 
-    unique_ptr<DatabaseCreateResponse> Database::create(const string &ownerName, const string &nodeName)
+    unique_ptr<DatabaseCreateResponse> Database::create()
     {
-        return create(ownerName, Signature::KeyPair(), nodeName);
+        return create(Signature::KeyPair());
     }
     
-    unique_ptr<DatabaseCreateResponse> Database::create(const string &ownerName, const Signature::KeyPair &keyPair, const string &nodeName)
+    unique_ptr<DatabaseCreateResponse> Database::create(const Signature::KeyPair &creatorKeyPair)
     {
         // TODO: Should this be declared static? is there any difference in behavior/performance?
         boost::uuids::random_generator uuidGen;
         auto adminGroupId = uuidGen();
-        auto adminGroup = new Group("administrator", adminGroupId.data, ADMIN_PERMISSION);
-        LocalUser *nodeAdminUser = LocalUser::create(keyPair, ownerName, adminGroup);
+        assert(adminGroupId.size() == GROUP_ID_LENGTH);
+        auto adminGroup = new Group(adminGroupId.data, ADMIN_PERMISSION);
+        LocalUser *nodeAdminUser = LocalUser::create(creatorKeyPair, adminGroup);
         
         // Header
         sibs::SafeSerializer serializer;
@@ -305,30 +308,18 @@ namespace odhtdb
         serializer.add((u8*)nodeAdminUser->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES);
         serializer.add(adminGroupId.data, adminGroupId.size());
         
-        // Encrypted body
-        sibs::SafeSerializer encryptedSerializer;
-        assert(nodeAdminUser->getName().size() <= 255);
-        encryptedSerializer.add((u8)nodeAdminUser->getName().size());
-        encryptedSerializer.add((u8*)nodeAdminUser->getName().data(), nodeAdminUser->getName().size());
-        assert(nodeName.size() <= 255);
-        encryptedSerializer.add((u8)nodeName.size());
-        encryptedSerializer.add((u8*)nodeName.data(), nodeName.size());
-        
         try
         {
-            Encryption encryptedBody(DataView(encryptedSerializer.getBuffer().data(), encryptedSerializer.getBuffer().size()));
-            DataView requestData = combine(serializer, encryptedBody);
-            shared_ptr<Hash> hashRequestKey = make_shared<Hash>(requestData.data, requestData.size);
-            databaseStorage.setNodeDecryptionKey(*hashRequestKey, DataView(encryptedBody.getKey().data, encryptedBody.getKey().size));
-            databaseStorage.createStorage(*hashRequestKey, adminGroup, timestampCombined, (const u8*)requestData.data, requestData.size, serializer.getBuffer().size());
+            unsigned char *encryptionKeyRaw = new unsigned char[ENCRYPTION_KEY_BYTE_SIZE];
+            Encryption::generateKey(encryptionKeyRaw);
+            shared_ptr<OwnedMemory> encryptionKey = make_shared<OwnedMemory>(encryptionKeyRaw, ENCRYPTION_KEY_BYTE_SIZE);
+            shared_ptr<Hash> hashRequestKey = make_shared<Hash>(serializer.getBuffer().data(), serializer.getBuffer().size());
             
-            assert(encryptedBody.getKey().size == ENCRYPTION_KEY_BYTE_SIZE);
-            auto key = make_shared<OwnedMemory>(new char[encryptedBody.getKey().size], encryptedBody.getKey().size);
-            memcpy(key->data, encryptedBody.getKey().data, encryptedBody.getKey().size);
+            databaseStorage.setNodeDecryptionKey(*hashRequestKey, DataView(encryptionKey->data, encryptionKey->size));
+            databaseStorage.createStorage(*hashRequestKey, adminGroup, timestampCombined, (const u8*)serializer.getBuffer().data(), serializer.getBuffer().size());
             
             DhtKey dhtKey(*hashRequestKey);
-            Value createDataValue((u8*)requestData.data, requestData.size);
-            delete[] (char*)requestData.data;
+            Value createDataValue(move(serializer.getBuffer()));
             node.put(dhtKey.getNewDataListenerKey(), move(createDataValue), [](bool ok)
             {
                 // TODO: Handle failure to put data
@@ -336,7 +327,7 @@ namespace odhtdb
                     Log::warn("Failed to put: %s, what to do?", "Database::create");
             });
             
-            return make_unique<DatabaseCreateResponse>(nodeAdminUser, move(key), hashRequestKey);
+            return make_unique<DatabaseCreateResponse>(nodeAdminUser, encryptionKey, hashRequestKey);
         }
         catch (EncryptionException &e)
         {
@@ -354,7 +345,7 @@ namespace odhtdb
             // and remote peers would accept our request to perform operation if they haven't received the operation that removes the user from the group.
             // How to handle this?
             string errMsg = "User ";
-            errMsg += userToPerformActionWith->getName();
+            errMsg += userToPerformActionWith->getPublicKey().toString();
             errMsg += " is not allowed to perform the operation: ADD_USER";
             throw PermissionDeniedException(errMsg);
         }
@@ -372,7 +363,7 @@ namespace odhtdb
         DataView stagedAddObject = combine(userToPerformActionWith->getPublicKey(), signedRequestData);
         Hash requestDataHash(stagedAddObject.data, stagedAddObject.size);
         DataView encryptedDataView((char*)requestData.data + serializer.getBuffer().size(), requestData.size - serializer.getBuffer().size());
-        databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, DatabaseOperation::ADD_DATA, userToPerformActionWith, timestampCombined, (u8*)stagedAddObject.data, stagedAddObject.size, encryptedDataView);
+        databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, DatabaseOperation::ADD_DATA, userToPerformActionWith->getPublicKey(), timestampCombined, (u8*)stagedAddObject.data, stagedAddObject.size, encryptedDataView);
         delete[] (char*)requestData.data;
         
         DhtKey dhtKey(requestDataHash);
@@ -391,7 +382,7 @@ namespace odhtdb
         for(auto group : groups)
         {
             const auto &groupPermission = group->getPermission();
-            if(groupPermission.getFlag(PermissionType::ADD_USER_LOWER_LEVEL) && groupPermission.getPermissionLevel() < groupToAddUserTo->getPermission().getPermissionLevel())
+            if(groupPermission.getFlag(PermissionType::ADD_USER_HIGHER_LEVEL) && groupPermission.getPermissionLevel() < groupToAddUserTo->getPermission().getPermissionLevel())
             {
                 return group;
             }
@@ -403,15 +394,15 @@ namespace odhtdb
         return nullptr;
     }
     
-    void Database::addUser(const DatabaseNode &nodeInfo, const LocalUser *userToPerformActionWith, const string &userToAddName, const Signature::PublicKey &userToAddPublicKey, Group *groupToAddUserTo)
+    void Database::addUser(const DatabaseNode &nodeInfo, const LocalUser *userToPerformActionWith, const Signature::PublicKey &userToAddPublicKey, Group *groupToAddUserTo)
     {
         auto groupWithAddUserRights = getGroupWithRightsToAddUserToGroup(userToPerformActionWith->getGroups(), groupToAddUserTo);
         if(!groupWithAddUserRights)
         {
             string errMsg = "The user ";
-            errMsg += userToPerformActionWith->getName();
+            errMsg += userToPerformActionWith->getPublicKey().toString();
             errMsg += " does not belong to any group that is allowed to add an user to the group ";
-            errMsg += groupToAddUserTo->getName();
+            errMsg += bin2hex((const char*)groupToAddUserTo->getId().data, groupToAddUserTo->getId().size).c_str();
             throw PermissionDeniedException(errMsg);
         }
         
@@ -420,27 +411,22 @@ namespace odhtdb
         u64 timestampCombined = getSyncedTimestampUtc().getCombined();
         serializer.add(timestampCombined);
         serializer.add(DatabaseOperation::ADD_USER);
-        
-        assert(userToAddName.size() <= 255);
-        usize serializedEncryptedDataOffset = serializer.getBuffer().size();
-        serializer.add((u8)userToAddName.size());
-        DataView encryptionKey(nodeInfo.getNodeEncryptionKey()->data, ENCRYPTION_KEY_BYTE_SIZE);
-        Encryption encryptedUserName(DataView((void*)userToAddName.data(), userToAddName.size()), DataView(), encryptionKey);
-        serializer.add((u8*)encryptedUserName.getNonce().data, ENCRYPTION_NONCE_BYTE_SIZE);
-        assert(encryptedUserName.getCipherText().size == ENCRYPTION_CHECKSUM_BYTE_SIZE + userToAddName.size());
-        serializer.add((u8*)encryptedUserName.getCipherText().data, ENCRYPTION_CHECKSUM_BYTE_SIZE + userToAddName.size());
-        usize serializedEncryptedDataSize = serializer.getBuffer().size() - serializedEncryptedDataOffset;
+        usize additionalDataOffset = serializer.getBuffer().size();
         serializer.add((u8*)userToAddPublicKey.getData(), PUBLIC_KEY_NUM_BYTES);
         serializer.add((uint8_t*)groupToAddUserTo->getId().data, groupToAddUserTo->getId().size);
         
+        // TODO: Should this be declared static? is there any difference in behavior/performance?
+        boost::uuids::random_generator uuidGen;
+        auto padding = uuidGen();
+        assert(padding.size() == 16);
+        serializer.add(padding.data, padding.size());
+        
         DataView requestData { serializer.getBuffer().data(), serializer.getBuffer().size() };
         string signedRequestData = userToPerformActionWith->getPrivateKey().sign(requestData);
         DataView stagedAddObject = combine(userToPerformActionWith->getPublicKey(), signedRequestData);
         Hash requestDataHash(stagedAddObject.data, stagedAddObject.size);
-        DataView encryptedDataView(nullptr, 0);
-        auto userToAdd = RemoteUser::create(userToAddPublicKey, userToAddName, groupToAddUserTo);
-        databaseStorage.addUser(*nodeInfo.getRequestHash(), userToAdd);
-        databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, DatabaseOperation::ADD_USER, userToPerformActionWith, timestampCombined, (u8*)stagedAddObject.data, stagedAddObject.size, encryptedDataView);
+        DataView additionalDataView((void*)(static_cast<const char*>(requestData.data) + additionalDataOffset), requestData.size - additionalDataOffset);
+        databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, DatabaseOperation::ADD_USER, userToPerformActionWith->getPublicKey(), timestampCombined, (u8*)stagedAddObject.data, stagedAddObject.size, additionalDataView);
         
         DhtKey dhtKey(requestDataHash);
         Value addDataValue((u8*)stagedAddObject.data, stagedAddObject.size);
@@ -506,10 +492,10 @@ namespace odhtdb
         if(deserializer.getSize() < ENCRYPTION_NONCE_BYTE_SIZE)
             throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)");
         
-        auto adminGroup = new Group("administrator", adminGroupId, ADMIN_PERMISSION);
+        auto adminGroup = new Group(adminGroupId, ADMIN_PERMISSION);
         // TODO: Username is encrypted, we dont know it... unless we have encryption key, in which case we should modify the user name and set it
-        auto creatorUser = RemoteUser::create(userPublicKey, "ENCRYPTED USER NAME", adminGroup);
-        databaseStorage.createStorage(hash, adminGroup, creationDate, value->data.data(), value->data.size(), value->data.size() - deserializer.getSize());
+        auto creatorUser = RemoteUser::create(userPublicKey, adminGroup);
+        databaseStorage.createStorage(hash, adminGroup, creationDate, value->data.data(), value->data.size());
     }
 
     void Database::deserializeAddRequest(const shared_ptr<dht::Value> &value, const Hash &requestDataHash, const std::shared_ptr<Hash> &nodeHash, const shared_ptr<OwnedMemory> encryptionKey)
@@ -548,108 +534,8 @@ namespace odhtdb
         */
         
         DatabaseOperation operation = deserializerUnsigned.extract<DatabaseOperation>();
-#if 0
-        const Hash *node = databaseStorage.getNodeByUserPublicKey(creatorPublicKey);
-        if(!node)
-        {
-            // The user (public key) could belong to a node but we might not have retrieved the node info yet since data may
-            // not be retrieved in order.
-            // Data in quarantine is processed when 'create' packet is received or removed after 60 seconds
-            databaseStorage.addToQuarantine(requestDataHash, creatorPublicKey, creationDate, value->data.data(), value->data.size());
-            throw RequestQuarantineException();
-        }
-#endif
-        auto creatorUser = databaseStorage.getUserByPublicKey(*nodeHash, creatorPublicKey);
-        if(!creatorUser)
-        {
-            // TODO: Add to quarantine
-            string errMsg = "User with public key ";
-            errMsg += creatorPublicKey.toString();
-            errMsg += " does not exist in code ";
-            errMsg += nodeHash->toString();
-            throw sibs::DeserializeException(errMsg);
-        }
-        
-        DataView encryptedDataView((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize());
-        
-        if(operation == DatabaseOperation::ADD_DATA)
-        {
-            if(deserializerUnsigned.getSize() < ENCRYPTION_NONCE_BYTE_SIZE)
-                throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)");
-            
-            if(!creatorUser->isAllowedToPerformAction(PermissionType::ADD_DATA))
-            {
-                // TODO: User might have permission to perform operation, but we haven't got the packet that adds user to the group with the permission,
-                // or we haven't received the packet that modifies group with the permission to perform the operation.
-                // This also means that an user can be in a group that has permission to perform the operation and then later be removed from it,
-                // and remote peers would accept our request to perform operation if they haven't received the operation that removes the user from the group.
-                // How to handle this?
-                string errMsg = "User ";
-                errMsg += creatorUser->getName();
-                errMsg += " is not allowed to add data to node ";
-                errMsg += nodeHash->toString();
-                throw PermissionDeniedException(errMsg);
-            }
-            
-            // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine.
-            // TODO: We might receive 'add' data packet before 'create'. If that happens, we should put it in quarantine and process it later.
-            databaseStorage.appendStorage(*nodeHash, requestDataHash, operation, creatorUser, creationDate, value->data.data(), value->data.size(), encryptedDataView);
-        }
-        else if(operation == DatabaseOperation::ADD_USER)
-        {
-            u8 nameLength = deserializerUnsigned.extract<u8>();
-            
-            u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE];
-            deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE);
-            DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), ENCRYPTION_CHECKSUM_BYTE_SIZE + nameLength);
-            
-            sibs::SafeDeserializer deserializerSkippedEncryptedData(deserializerUnsigned.getBuffer() + ENCRYPTION_CHECKSUM_BYTE_SIZE + nameLength, PUBLIC_KEY_NUM_BYTES + GROUP_ID_LENGTH);
-            
-            char userToAddPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
-            deserializerSkippedEncryptedData.extract((u8*)userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
-            Signature::PublicKey userToAddPublicKey(userToAddPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
-            
-            uint8_t groupId[GROUP_ID_LENGTH];
-            deserializerSkippedEncryptedData.extract(groupId, GROUP_ID_LENGTH);
-            
-            auto group = databaseStorage.getGroupById(*nodeHash, groupId);
-            if(group)
-            {
-                auto user = RemoteUser::create(userToAddPublicKey, "ENCRYPTED USER NAME", group);
-                // TODO: What if we receive packets in wrong order? (maliciously or non-maliciously). You would be able to register a user to a group with given name
-                // and further registration would be dropped (even if that is the correct one)
-                if(!databaseStorage.addUser(*nodeHash, user)) return;
-                
-                auto creatorUserGroupWithRights = getGroupWithRightsToAddUserToGroup(creatorUser->getGroups(), group);
-                if(!creatorUserGroupWithRights)
-                {
-                    // TODO: User might have permission to perform operation, but we haven't got the packet that adds user to the group with the permission,
-                    // or we haven't received the packet that modifies group with the permission to perform the operation.
-                    // This also means that an user can be in a group that has permission to perform the operation and then later be removed from it,
-                    // and remote peers would accept our request to perform operation if they haven't received the operation that removes the user from the group.
-                    // How to handle this?
-                    string errMsg = "User ";
-                    errMsg += creatorUser->getName();
-                    errMsg += " is not allowed to perform the operation: ";
-                    errMsg += to_string((u8)operation);
-                    throw PermissionDeniedException(errMsg);
-                }
-                
-                // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine.
-                // TODO: We might receive 'add' data packet before 'create'. If that happens, we should put it in quarantine and process it later.
-                databaseStorage.appendStorage(*nodeHash, requestDataHash, operation, creatorUser, creationDate, value->data.data(), value->data.size(), encryptedDataView);
-            }
-            else
-            {
-                throw sibs::DeserializeException("TODO: Add to quarantine? You can receive ADD_USER packet before you receive ADD_GROUP");
-            }
-        }
-        else
-        {
-            string errMsg = "Got unexpected operation: ";
-            errMsg += to_string((u8)operation);
-            throw sibs::DeserializeException(errMsg);
-        }
+        DataView additionalDataView((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize());
+        databaseStorage.appendStorage(*nodeHash, requestDataHash, operation, creatorPublicKey, creationDate, value->data.data(), value->data.size(), additionalDataView);
     }
 
     bool Database::listenCreateData(shared_ptr<dht::Value> value, const Hash &hash, const shared_ptr<OwnedMemory> encryptionKey)
@@ -657,7 +543,7 @@ namespace odhtdb
         Log::debug("Got create data");
         try
         {
-            if(databaseStorage.getStorage(hash))
+            if(databaseStorage.doesNodeExist(hash))
                 throw DatabaseStorageAlreadyExists("Create request hash is equal to hash already in storage (duplicate data?)");
             deserializeCreateRequest(value, hash, encryptionKey);
         }
@@ -673,7 +559,7 @@ namespace odhtdb
         Log::debug("Got add data");
         try
         {
-            if(databaseStorage.getDataById(requestDataHash))
+            if(databaseStorage.doesDataExist(requestDataHash))
                 throw DatabaseStorageAlreadyExists("Add data request hash is equal to hash already in storage (duplicate data?)");
             deserializeAddRequest(value, requestDataHash, nodeHash, encryptionKey);
             //Log::debug("Got add object, timestamp: %zu", addObject.timestamp);
@@ -688,24 +574,4 @@ namespace odhtdb
         }
         return true;
     }
-
-    void Database::setOnCreateNodeCallback(function<void(const DatabaseCreateNodeRequest&)> callbackFunc)
-    {
-        onCreateNodeCallbackFunc = callbackFunc;
-    }
-
-    void Database::setOnAddNodeCallback(function<void(const DatabaseAddNodeRequest&)> callbackFunc)
-    {
-        onAddNodeCallbackFunc = callbackFunc;
-    }
-
-    void Database::setOnAddUserCallback(function<void(const DatabaseAddUserRequest&)> callbackFunc)
-    {
-        onAddUserCallbackFunc = callbackFunc;
-    }
-    
-    DatabaseStorage& Database::getStorage()
-    {
-        return databaseStorage;
-    }
 }