From 414f2cafc6cf2fe141c011b0d63d447a9b983ac3 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 14 Apr 2018 19:45:15 +0200 Subject: Store database storage to files, also loading --- src/Database.cpp | 107 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 55 insertions(+), 52 deletions(-) (limited to 'src/Database.cpp') diff --git a/src/Database.cpp b/src/Database.cpp index 41b0d73..226aa05 100644 --- a/src/Database.cpp +++ b/src/Database.cpp @@ -60,9 +60,9 @@ namespace odhtdb return DataView(result, allocationSize); } - DatabaseCreateResponse::DatabaseCreateResponse(LocalUser *_nodeAdminUser, const shared_ptr &_key, const shared_ptr &_hash) : + DatabaseCreateResponse::DatabaseCreateResponse(LocalUser *_nodeAdminUser, const shared_ptr &_key, const shared_ptr &_hash) : nodeAdminUser(_nodeAdminUser), - key(move(_key)), + key(_key), hash(_hash) { @@ -73,7 +73,7 @@ namespace odhtdb return nodeAdminUser; } - const shared_ptr DatabaseCreateResponse::getNodeEncryptionKey() const + const shared_ptr DatabaseCreateResponse::getNodeEncryptionKey() const { return key; } @@ -83,10 +83,11 @@ namespace odhtdb return hash; } - Database::Database(const char *bootstrapNodeAddr, u16 port, boost::filesystem::path storageDir) : + Database::Database(const char *bootstrapNodeAddr, u16 port, const boost::filesystem::path &storageDir) : onCreateNodeCallbackFunc(nullptr), onAddNodeCallbackFunc(nullptr), - onAddUserCallbackFunc(nullptr) + onAddUserCallbackFunc(nullptr), + databaseStorage(storageDir) { // TODO: Cache this in storage. It takes pretty long time to generate new identity auto identity = dht::crypto::generateIdentity(); @@ -147,43 +148,43 @@ namespace odhtdb node.join(); } - void Database::seed(const shared_ptr hash, const shared_ptr encryptionKey) + void Database::seed(const DatabaseNode &nodeToSeed) { // TODO: Use cached files and seed those. If none exists, request new files to seed. // If nobody requests my cached files in a long time, request new files to seed and remove cached files // (only if there are plenty of other seeders for the cached files. This could also cause race issue // where all nodes with a cached file delete it at same time) - Log::debug("Seeding key: %s", hash->toString().c_str()); - DhtKey dhtKey(*hash); + Log::debug("Seeding key: %s", nodeToSeed.getRequestHash()->toString().c_str()); + DhtKey dhtKey(*nodeToSeed.getRequestHash()); - node.listen(dhtKey.getNewDataListenerKey(), [this, hash, encryptionKey](const shared_ptr &value) + node.listen(dhtKey.getNewDataListenerKey(), [this, nodeToSeed](const shared_ptr &value) { Log::debug("Seed: New data listener received data..."); const Hash requestHash(value->data.data(), value->data.size()); - if(requestHash == *hash) + if(requestHash == *nodeToSeed.getRequestHash()) return true; //return listenCreateData(value, requestHash, encryptionKey); else - return listenAddData(value, requestHash, encryptionKey); + return listenAddData(value, requestHash, nodeToSeed.getRequestHash(), nodeToSeed.getNodeEncryptionKey()); }); u8 responseKey[OPENDHT_INFOHASH_LEN]; randombytes_buf(responseKey, OPENDHT_INFOHASH_LEN); // TODO: If this response key is spammed, generate a new one. - node.listen(InfoHash(responseKey, OPENDHT_INFOHASH_LEN), [this, hash, encryptionKey](const shared_ptr &value) + node.listen(InfoHash(responseKey, OPENDHT_INFOHASH_LEN), [this, nodeToSeed](const shared_ptr &value) { const Hash requestHash(value->data.data(), value->data.size()); - if(requestHash == *hash) - return listenCreateData(value, requestHash, encryptionKey); + if(requestHash == *nodeToSeed.getRequestHash()) + return listenCreateData(value, requestHash, nodeToSeed.getNodeEncryptionKey()); else - return listenAddData(value, requestHash, encryptionKey); + return listenAddData(value, requestHash, nodeToSeed.getRequestHash(), nodeToSeed.getNodeEncryptionKey()); }); // TODO: Before listening on this key, we should check how many remote peers are also providing this data. // This is to prevent too many peers from responding to a request to get old data. - node.listen(dhtKey.getRequestOldDataKey(), [this, hash](const shared_ptr &value) + node.listen(dhtKey.getRequestOldDataKey(), [this, nodeToSeed](const shared_ptr &value) { Log::debug("Request: Got request to send old data"); try @@ -193,10 +194,10 @@ namespace odhtdb u8 requestResponseKey[OPENDHT_INFOHASH_LEN]; deserializer.extract(requestResponseKey, OPENDHT_INFOHASH_LEN); - auto requestedData = databaseStorage.getStorage(*hash); + auto requestedData = databaseStorage.getStorage(*nodeToSeed.getRequestHash()); if(!requestedData) { - Log::warn("No data found for hash %s, unable to serve peer", hash->toString().c_str()); + Log::warn("No data found for hash %s, unable to serve peer", nodeToSeed.getRequestHash()->toString().c_str()); return true; } @@ -229,7 +230,7 @@ namespace odhtdb }); sibs::SafeSerializer serializer; - serializer.add((u64)0); // Timestamp in microseconds, fetch data newer than this + serializer.add((u64)0); // Timestamp in microseconds, fetch data newer than this. // TODO: Get timestamp from database storage serializer.add(responseKey, OPENDHT_INFOHASH_LEN); node.put(dhtKey.getRequestOldDataKey(), Value(serializer.getBuffer().data(), serializer.getBuffer().size()), [](bool ok) { @@ -241,13 +242,13 @@ namespace odhtdb //node.listen(ADD_DATA_HASH, bind(&Database::listenAddData, this, _1)); } - unique_ptr Database::create(const std::string &ownerName, const std::string &nodeName) + unique_ptr Database::create(const string &ownerName, const std::string &ownerPlainPassword, const string &nodeName) { // TODO: Should this be declared static? is there any difference in behavior/performance? boost::uuids::random_generator uuidGen; auto adminGroupId = uuidGen(); auto adminGroup = new Group("administrator", adminGroupId.data, ADMIN_PERMISSION); - LocalUser *nodeAdminUser = LocalUser::create(Signature::KeyPair(), ownerName, adminGroup); + LocalUser *nodeAdminUser = LocalUser::create(Signature::KeyPair(), ownerName, adminGroup, ownerPlainPassword); // Header sibs::SafeSerializer serializer; @@ -281,10 +282,10 @@ namespace odhtdb stagedCreateObjects.emplace_back(make_unique(requestData, hashRequestKey)); - assert(encryptedBody.getKey().size == KEY_BYTE_SIZE); - char *key = new char[encryptedBody.getKey().size]; - memcpy(key, encryptedBody.getKey().data, encryptedBody.getKey().size); - return make_unique(nodeAdminUser, make_shared(key), hashRequestKey); + assert(encryptedBody.getKey().size == ENCRYPTION_KEY_BYTE_SIZE); + auto key = make_shared(new char[encryptedBody.getKey().size], encryptedBody.getKey().size); + memcpy(key->data, encryptedBody.getKey().data, encryptedBody.getKey().size); + return make_unique(nodeAdminUser, move(key), hashRequestKey); } catch (EncryptionException &e) { @@ -301,7 +302,7 @@ namespace odhtdb serializer.add(timestampMicroseconds); serializer.add(DatabaseOperation::ADD_DATA); - DataView encryptionKey(*nodeInfo.getNodeEncryptionKey(), KEY_BYTE_SIZE); + DataView encryptionKey(nodeInfo.getNodeEncryptionKey()->data, ENCRYPTION_KEY_BYTE_SIZE); Encryption encryptedBody(dataToAdd, DataView(), encryptionKey); DataView requestData = combine(serializer, encryptedBody); string signedRequestData = userToPerformActionWith->getPrivateKey().sign(requestData); @@ -334,7 +335,7 @@ namespace odhtdb return nullptr; } - void Database::addUserToGroup(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, const std::string &userToAddName, const Signature::PublicKey &userToAddPublicKey, Group *groupToAddUserTo) + void Database::addUser(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, const string &userToAddName, const Signature::PublicKey &userToAddPublicKey, Group *groupToAddUserTo) { auto groupWithAddUserRights = getGroupWithRightsToAddUserToGroup(userToPerformActionWith->getGroups(), groupToAddUserTo); if(!groupWithAddUserRights) @@ -365,7 +366,7 @@ namespace odhtdb Hash requestDataHash(stagedAddObject.data, stagedAddObject.size); databaseStorage.appendStorage(*nodeInfo.getRequestHash(), requestDataHash, userToPerformActionWith, timestampMicroseconds, (u8*)stagedAddObject.data, stagedAddObject.size); auto userToAdd = RemoteUser::create(userToAddPublicKey, userToAddName, groupToAddUserTo); - databaseStorage.addUser(userToAdd, *nodeInfo.getRequestHash()); + databaseStorage.addUser(*nodeInfo.getRequestHash(), userToAdd); DatabaseAddUserRequest addUserRequest(&*nodeInfo.getRequestHash(), &requestDataHash, timestampMicroseconds, userToPerformActionWith, userToAdd, groupToAddUserTo); if(onAddUserCallbackFunc) @@ -446,7 +447,7 @@ namespace odhtdb return timestamp; } - void Database::deserializeCreateRequest(const std::shared_ptr &value, const Hash &hash, const shared_ptr encryptionKey) + void Database::deserializeCreateRequest(const shared_ptr &value, const Hash &hash, const shared_ptr encryptionKey) { sibs::SafeDeserializer deserializer(value->data.data(), value->data.size()); u16 packetStructureVersion = deserializer.extract(); @@ -472,7 +473,7 @@ namespace odhtdb uint8_t adminGroupId[16]; deserializer.extract(adminGroupId, 16); - if(deserializer.getSize() < NONCE_BYTE_SIZE) + if(deserializer.getSize() < ENCRYPTION_NONCE_BYTE_SIZE) throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)"); auto adminGroup = new Group("administrator", adminGroupId, ADMIN_PERMISSION); @@ -480,11 +481,11 @@ namespace odhtdb auto creatorUser = RemoteUser::create(userPublicKey, "ENCRYPTED USER NAME", adminGroup); databaseStorage.createStorage(hash, adminGroup, creationDate, value->data.data(), value->data.size()); - u8 nonce[NONCE_BYTE_SIZE]; - deserializer.extract(nonce, NONCE_BYTE_SIZE); + u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE]; + deserializer.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE); DataView dataToDecrypt((void*)deserializer.getBuffer(), deserializer.getSize()); - Decryption decryptedBody(dataToDecrypt, DataView(nonce, NONCE_BYTE_SIZE), DataView(*encryptionKey, KEY_BYTE_SIZE)); + Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(encryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE)); sibs::SafeDeserializer bodyDeserializer((const u8*)decryptedBody.getDecryptedText().data, decryptedBody.getDecryptedText().size); u8 creatorNameLength = bodyDeserializer.extract(); @@ -513,7 +514,7 @@ namespace odhtdb return false; } - void Database::deserializeAddRequest(const std::shared_ptr &value, const Hash &requestDataHash, const shared_ptr encryptionKey) + void Database::deserializeAddRequest(const shared_ptr &value, const Hash &requestDataHash, const std::shared_ptr &nodeHash, const shared_ptr encryptionKey) { sibs::SafeDeserializer deserializer(value->data.data(), value->data.size()); char creatorPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; @@ -541,7 +542,7 @@ namespace odhtdb throw sibs::DeserializeException("Packet is from the future"); DatabaseOperation operation = deserializerUnsigned.extract(); - +#if 0 const Hash *node = databaseStorage.getNodeByUserPublicKey(creatorPublicKey); if(!node) { @@ -551,21 +552,21 @@ namespace odhtdb databaseStorage.addToQuarantine(requestDataHash, creatorPublicKey, creationDate, value->data.data(), value->data.size()); throw RequestQuarantineException(); } - - auto creatorUser = databaseStorage.getUserByPublicKey(creatorPublicKey); +#endif + auto creatorUser = databaseStorage.getUserByPublicKey(*nodeHash, creatorPublicKey); // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine. // TODO: We might receive 'add' data packet before 'create'. If that happens, we should put it in quarantine and process it later. - databaseStorage.appendStorage(*node, requestDataHash, creatorUser, creationDate, value->data.data(), value->data.size()); + databaseStorage.appendStorage(*nodeHash, requestDataHash, creatorUser, creationDate, value->data.data(), value->data.size()); if(operation == DatabaseOperation::ADD_DATA) { - if(deserializerUnsigned.getSize() < NONCE_BYTE_SIZE) + if(deserializerUnsigned.getSize() < ENCRYPTION_NONCE_BYTE_SIZE) throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)"); - u8 nonce[NONCE_BYTE_SIZE]; - deserializerUnsigned.extract(nonce, NONCE_BYTE_SIZE); + u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE]; + deserializerUnsigned.extract(nonce, ENCRYPTION_NONCE_BYTE_SIZE); DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize()); - Decryption decryptedBody(dataToDecrypt, DataView(nonce, NONCE_BYTE_SIZE), DataView(*encryptionKey, KEY_BYTE_SIZE)); + Decryption decryptedBody(dataToDecrypt, DataView(nonce, ENCRYPTION_NONCE_BYTE_SIZE), DataView(encryptionKey->data, ENCRYPTION_KEY_BYTE_SIZE)); if(!isUserAllowedToAddData(creatorUser)) { @@ -582,7 +583,7 @@ namespace odhtdb } Log::debug("Got add object, timestamp: %zu, data: %.*s", creationDate, decryptedBody.getDecryptedText().size, decryptedBody.getDecryptedText().data); - const DatabaseAddNodeRequest addNodeRequest(node, &requestDataHash, creationDate, creatorUser, decryptedBody.getDecryptedText()); + const DatabaseAddNodeRequest addNodeRequest(&*nodeHash, &requestDataHash, creationDate, creatorUser, decryptedBody.getDecryptedText()); if(onAddNodeCallbackFunc) onAddNodeCallbackFunc(addNodeRequest); } @@ -600,11 +601,13 @@ namespace odhtdb uint8_t groupId[16]; deserializerUnsigned.extract(groupId, 16); - auto group = databaseStorage.getGroupById(groupId); + auto group = databaseStorage.getGroupById(*nodeHash, groupId); if(group) { auto user = RemoteUser::create(userToAddPublicKey, name, group); - databaseStorage.addUser(user, *node); + // TODO: What if we receive packets in wrong order? (maliciously or non-maliciously). You would be able to register a user to a group with given name + // and further registration would be dropped (even if that is the correct one) + if(!databaseStorage.addUser(*nodeHash, user)) return; auto creatorUserGroupWithRights = getGroupWithRightsToAddUserToGroup(creatorUser->getGroups(), group); if(!creatorUserGroupWithRights) @@ -622,7 +625,7 @@ namespace odhtdb } Log::debug("Got add user object, timestamp: %zu, user added: %.*s", creationDate, nameLength, name.c_str()); - DatabaseAddUserRequest addUserRequest(node, &requestDataHash, creationDate, creatorUser, user, group); + DatabaseAddUserRequest addUserRequest(&*nodeHash, &requestDataHash, creationDate, creatorUser, user, group); if(onAddUserCallbackFunc) onAddUserCallbackFunc(addUserRequest); } @@ -639,7 +642,7 @@ namespace odhtdb } } - bool Database::listenCreateData(std::shared_ptr value, const Hash &hash, const shared_ptr encryptionKey) + bool Database::listenCreateData(shared_ptr value, const Hash &hash, const shared_ptr encryptionKey) { Log::debug("Got create data"); try @@ -655,12 +658,12 @@ namespace odhtdb return true; } - bool Database::listenAddData(std::shared_ptr value, const Hash &requestDataHash, const shared_ptr encryptionKey) + bool Database::listenAddData(shared_ptr value, const Hash &requestDataHash, const std::shared_ptr nodeHash, const shared_ptr encryptionKey) { Log::debug("Got add data"); try { - deserializeAddRequest(value, requestDataHash, encryptionKey); + deserializeAddRequest(value, requestDataHash, nodeHash, encryptionKey); //Log::debug("Got add object, timestamp: %zu", addObject.timestamp); } catch (RequestQuarantineException &e) @@ -674,17 +677,17 @@ namespace odhtdb return true; } - void Database::setOnCreateNodeCallback(std::function callbackFunc) + void Database::setOnCreateNodeCallback(function callbackFunc) { onCreateNodeCallbackFunc = callbackFunc; } - void Database::setOnAddNodeCallback(std::function callbackFunc) + void Database::setOnAddNodeCallback(function callbackFunc) { onAddNodeCallbackFunc = callbackFunc; } - void Database::setOnAddUserCallback(std::function callbackFunc) + void Database::setOnAddUserCallback(function callbackFunc) { onAddUserCallbackFunc = callbackFunc; } -- cgit v1.2.3