From 230e61091b401b8b2bb9496d557a15660fb5072b Mon Sep 17 00:00:00 2001 From: Aleksi Lindeman <0xdec05eba@gmail.com> Date: Fri, 9 Mar 2018 10:26:55 +0100 Subject: Partially implement 'add' operation --- .vscode/launch.json | 2 +- include/Database.hpp | 39 ++++--- include/DatabaseStorage.hpp | 42 +++++--- include/Encryption.hpp | 7 +- include/Hash.hpp | 11 +- include/Permission.hpp | 14 --- include/Signature.hpp | 15 +++ include/StagedObject.hpp | 10 +- project.conf | 6 +- src/Database.cpp | 247 +++++++++++++++++++++++++------------------- src/DatabaseStorage.cpp | 63 +++++++++-- src/Encryption.cpp | 42 ++++++-- src/Hash.cpp | 14 +-- src/Signature.cpp | 13 ++- tests/main.cpp | 13 +-- 15 files changed, 341 insertions(+), 197 deletions(-) delete mode 100644 include/Permission.hpp diff --git a/.vscode/launch.json b/.vscode/launch.json index 956830f..c0c2715 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "name": "(gdb) Launch", "type": "cppdbg", "request": "launch", - "program": "tests/sibs-build/debug/test", + "program": "${workspaceFolder}/tests/sibs-build/debug/test", "args": [], "stopAtEntry": false, "cwd": "${workspaceFolder}", diff --git a/include/Database.hpp b/include/Database.hpp index d160133..1024fe0 100644 --- a/include/Database.hpp +++ b/include/Database.hpp @@ -16,6 +16,7 @@ namespace odhtdb { + class User; class LocalUser; class Group; @@ -74,35 +75,39 @@ namespace odhtdb { DISABLE_COPY(DatabaseAddRequest) - u16 packetStructureVersion; u64 timestamp; // In microseconds - Signature::PublicKey creatorPublicKey; - DataView data; + const User *creatorUser; + Decryption decryptedData; - DatabaseAddRequest(u16 _packetStructureVersion, u64 _timestamp, Signature::PublicKey &&_creatorPublicKey, DataView &_data) : - packetStructureVersion(_packetStructureVersion), + DatabaseAddRequest(u64 _timestamp, const User *_creatorUser, Decryption &&_decryptedData) : timestamp(_timestamp), - creatorPublicKey(std::move(_creatorPublicKey)), - data(_data) + creatorUser(_creatorUser), + decryptedData(std::move(_decryptedData)) { } - ~DatabaseAddRequest() + DatabaseAddRequest(DatabaseAddRequest &&other) { - free(data.data); - data = DataView(); + timestamp = other.timestamp; + creatorUser = other.creatorUser; + decryptedData = std::move(other.decryptedData); + + other.timestamp = 0; + other.creatorUser = nullptr; } }; class DatabaseCreateResponse { public: - DatabaseCreateResponse(const std::shared_ptr &key, const std::shared_ptr &hash); + DatabaseCreateResponse(LocalUser *nodeAdminUser, const std::shared_ptr &key, const std::shared_ptr &hash); + const LocalUser* getNodeAdminUser() const; const std::shared_ptr getNodeEncryptionKey() const; const std::shared_ptr getRequestHash() const; private: + LocalUser *nodeAdminUser; std::shared_ptr key; std::shared_ptr hash; }; @@ -115,15 +120,15 @@ namespace odhtdb void seed(const std::shared_ptr hash, const std::shared_ptr encryptionKey); // Throws DatabaseCreateException on failure. - std::unique_ptr create(const LocalUser *owner, const std::string &name); + std::unique_ptr create(const std::string &ownerName, const std::string &nodeName); // Throws DatabaseAddException on failure - void add(const LocalUser *owner, const Key &key, DataView data); + void add(const std::unique_ptr &nodeInfo, DataView dataToAdd); void commit(); private: // Throws CommitCreateException on failure - void commitStagedCreateObject(const std::unique_ptr &stagedObject); + void commitStagedCreateObject(const std::unique_ptr &stagedObject); // Throws CommitAddException on failure - void commitStagedAddObject(const DataView &stagedObject); + void commitStagedAddObject(const std::unique_ptr &stagedObject); ntp::NtpTimestamp getSyncedTimestampUtc() const; DatabaseCreateRequest deserializeCreateRequest(const std::shared_ptr &value, const Hash &hash, const std::shared_ptr encryptionKey); DatabaseAddRequest deserializeAddRequest(const std::shared_ptr &value, const Hash &hash, const std::shared_ptr encryptionKey); @@ -131,8 +136,8 @@ namespace odhtdb bool listenAddData(std::shared_ptr value, const Hash &hash, const std::shared_ptr encryptionKey); private: dht::DhtRunner node; - std::vector> stagedCreateObjects; - std::vector> stagedAddObjects; + std::vector> stagedCreateObjects; + std::vector> stagedAddObjects; DatabaseStorage databaseStorage; }; } diff --git a/include/DatabaseStorage.hpp b/include/DatabaseStorage.hpp index 6f251d1..fd29050 100644 --- a/include/DatabaseStorage.hpp +++ b/include/DatabaseStorage.hpp @@ -10,27 +10,33 @@ namespace odhtdb { class Group; + class User; struct DatabaseStorageObject { DataView data; - u64 timestamp; // In microseconds + u64 createdTimestamp; // In microseconds Signature::PublicKey creatorPublicKey; - DatabaseStorageObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : - data(_data), timestamp(_timestamp), creatorPublicKey(_creatorPublicKey) - { - - } + DatabaseStorageObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey); }; struct DatabaseStorageObjectList { - u64 timestamp; // In microseconds + DataView data; + u64 createdTimestamp; // In microseconds std::vector groups; - std::vector objects; - u8 *createData; - usize createDataSize; + std::vector objects; + }; + + struct DatabaseStorageQuarantineObject + { + DataView data; + u64 createdTimestamp; // In microseconds + u64 storedTimestamp; // In microseconds + Signature::PublicKey creatorPublicKey; + + DatabaseStorageQuarantineObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey); }; class DatabaseStorageAlreadyExists : public std::runtime_error @@ -46,6 +52,7 @@ namespace odhtdb }; using DatabaseStorageMap = MapHashKey; + using DatabaseStorageQuarantineMap = Signature::MapPublicKey>; class DatabaseStorage { @@ -54,11 +61,22 @@ namespace odhtdb void createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize); // Throws DatabaseStorageNotFound if data with hash does not exist - void appendStorage(const Hash &hash, DataView &data, u64 timestamp, const Signature::PublicKey &creatorPublicKey); + void appendStorage(const Hash &hash, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize); + + void addToQuarantine(const Signature::PublicKey &creatorPublicKey, u64 timestamp, const u8 *data, usize dataSize); - // Returns nullptr if not storage with provided hash exists + // Returns nullptr if no storage with provided hash exists const DatabaseStorageObjectList* getStorage(const Hash &hash) const; + + // Returns nullptr if no node with the user exists + const Hash* getNodeByUserPublicKey(const Signature::PublicKey &userPublicKey) const; + + // Returns nullptr if no user with public key exists + const User* getUserByPublicKey(const Signature::PublicKey &userPublicKey) const; private: DatabaseStorageMap storageMap; + DatabaseStorageQuarantineMap quarantineStorageMap; + Signature::MapPublicKey userPublicKeyNodeMap; + Signature::MapPublicKey publicKeyUserMap; }; } diff --git a/include/Encryption.hpp b/include/Encryption.hpp index b2afe49..4697b35 100644 --- a/include/Encryption.hpp +++ b/include/Encryption.hpp @@ -31,8 +31,7 @@ namespace odhtdb DISABLE_COPY(Encryption) public: // Throws EncryptionException on failure (or std::bad_alloc on failed memory allocation) - Encryption(const DataView &data) : Encryption(data, DataView()) {} - Encryption(const DataView &data, const DataView &additionalData); + Encryption(const DataView &data, const DataView &additionalData = DataView(), const DataView &key = DataView()); ~Encryption(); DataView getKey() const; @@ -49,8 +48,12 @@ namespace odhtdb { DISABLE_COPY(Decryption) public: + Decryption() : decryptedText(nullptr), decryptedTextLength(0) {} + // Throws DecryptionException on failure Decryption(const DataView &data, const DataView &nonce, const DataView &key); + Decryption(Decryption &&other); + Decryption& operator=(Decryption &&other); ~Decryption(); DataView getDecryptedText() const; diff --git a/include/Hash.hpp b/include/Hash.hpp index d7c90b0..bd87b69 100644 --- a/include/Hash.hpp +++ b/include/Hash.hpp @@ -1,7 +1,6 @@ #pragma once #include "utils.hpp" -#include #include #include @@ -9,6 +8,15 @@ namespace odhtdb { const int HASH_BYTE_SIZE = 32; + // Source: https://stackoverflow.com/a/11414104 (public license) + static size_t fnvHash(const unsigned char *key, int len) + { + size_t h = 2166136261; + for (int i = 0; i < len; i++) + h = (h * 16777619) ^ key[i]; + return h; + } + class HashException : public std::runtime_error { public: @@ -21,6 +29,7 @@ namespace odhtdb Hash(); // Throws HashException on failure Hash(const void *input, const size_t inputSize); + Hash(const Hash &other); void* getData() const { return (void*)data; } size_t getSize() const { return HASH_BYTE_SIZE; } diff --git a/include/Permission.hpp b/include/Permission.hpp deleted file mode 100644 index a6a16c3..0000000 --- a/include/Permission.hpp +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "types.hpp" - -namespace odhtdb -{ - enum class Permission : u8 - { - NONE, - READ, - WRITE, - READ_WRITE - }; -} \ No newline at end of file diff --git a/include/Signature.hpp b/include/Signature.hpp index aace383..62c41c3 100644 --- a/include/Signature.hpp +++ b/include/Signature.hpp @@ -2,6 +2,7 @@ #include "DataView.hpp" #include +#include namespace odhtdb { @@ -67,6 +68,9 @@ namespace odhtdb // Both exceptions are derived from UnsignException std::string unsign(const DataView &signedMessage) const; + size_t operator()() const; + bool operator==(const PublicKey &other) const; + std::string toString() const; private: PublicKey(){} @@ -74,6 +78,17 @@ namespace odhtdb char data[PUBLIC_KEY_NUM_BYTES]; }; + struct PublicKeyHasher + { + size_t operator()(const PublicKey &publicKey) const + { + return publicKey(); + } + }; + + template + using MapPublicKey = std::unordered_map; + class PrivateKey { friend class KeyPair; diff --git a/include/StagedObject.hpp b/include/StagedObject.hpp index a75664e..0c9b534 100644 --- a/include/StagedObject.hpp +++ b/include/StagedObject.hpp @@ -6,14 +6,14 @@ namespace odhtdb { - struct StagedCreateObject + struct StagedObject { - DISABLE_COPY(StagedCreateObject) - DataView encryptedBody; + DISABLE_COPY(StagedObject) + DataView data; std::shared_ptr requestKey; - StagedCreateObject(DataView &_encryptedBody, const std::shared_ptr &_requestKey) : - encryptedBody(_encryptedBody), + StagedObject(DataView &_data, const std::shared_ptr &_requestKey) : + data(_data), requestKey(_requestKey) { diff --git a/project.conf b/project.conf index 2b9e556..c283a7a 100644 --- a/project.conf +++ b/project.conf @@ -9,6 +9,6 @@ tests = "tests" opendht = "1.5.0" fmt = "4.1.0" libsodium = "1.0.16" -ntpclient = "0.1.0" -sibs-serializer = "0.1.0" -boost-filesystem = "1.66.0" \ No newline at end of file +ntpclient = "0.2.0" +sibs-serializer = "0.2.0" +boost-filesystem = "1.66.0" diff --git a/src/Database.cpp b/src/Database.cpp index bcf5580..d01ff7d 100644 --- a/src/Database.cpp +++ b/src/Database.cpp @@ -30,8 +30,14 @@ const int OPENDHT_INFOHASH_LEN = 20; namespace odhtdb { - const u16 DATABASE_CREATE_PACKET_STRUCTURE_VERSION = 0; - const u16 DATABASE_ADD_PACKET_STRUCTURE_VERSION = 0; + const u16 DATABASE_CREATE_PACKET_STRUCTURE_VERSION = 1; + const u16 DATABASE_ADD_PACKET_STRUCTURE_VERSION = 1; + + class RequestQuarantineException : public runtime_error + { + public: + RequestQuarantineException() : runtime_error("Request quarantine, will be processed later (can be real of fake request)") {} + }; DataView combine(sibs::SafeSerializer &headerSerializer, const Encryption &encryptedData) { @@ -43,12 +49,27 @@ namespace odhtdb return DataView(result, allocationSize); } - DatabaseCreateResponse::DatabaseCreateResponse(const shared_ptr &_key, const shared_ptr &_hash) : + DataView combine(const Signature::PublicKey &publicKey, const string &signedEncryptedData) + { + usize allocationSize = publicKey.getSize() + signedEncryptedData.size(); + char *result = new char[allocationSize]; + memcpy(result, publicKey.getData(), publicKey.getSize()); + memcpy(result + publicKey.getSize(), signedEncryptedData.data(), signedEncryptedData.size()); + return DataView(result, allocationSize); + } + + DatabaseCreateResponse::DatabaseCreateResponse(LocalUser *_nodeAdminUser, const shared_ptr &_key, const shared_ptr &_hash) : + nodeAdminUser(_nodeAdminUser), key(move(_key)), hash(_hash) { } + + const LocalUser* DatabaseCreateResponse::getNodeAdminUser() const + { + return nodeAdminUser; + } const shared_ptr DatabaseCreateResponse::getNodeEncryptionKey() const { @@ -133,13 +154,19 @@ namespace odhtdb node.listen(dhtKey.getNewDataListenerKey(), [this, hash, encryptionKey](const shared_ptr &value) { - return listenAddData(value, *hash, encryptionKey); + printf("Seed: New data listener received data...\n"); + const Hash requestHash(value->data.data(), value->data.size()); + if(requestHash == *hash) + return true; + //return listenCreateData(value, requestHash, encryptionKey); + else + return listenAddData(value, requestHash, encryptionKey); }); u8 responseKey[OPENDHT_INFOHASH_LEN]; randombytes_buf(responseKey, OPENDHT_INFOHASH_LEN); - // TODO: If this response key is spammed, generate a new one + // TODO: If this response key is spammed, generate a new one. node.listen(InfoHash(responseKey, OPENDHT_INFOHASH_LEN), [this, hash, encryptionKey](const shared_ptr &value) { const Hash requestHash(value->data.data(), value->data.size()); @@ -171,16 +198,20 @@ namespace odhtdb if(dataStartTimestamp == 0) { printf("Request: Sent create packet to requesting peer\n"); - node.put(InfoHash(requestResponseKey, OPENDHT_INFOHASH_LEN), Value(requestedData->createData, requestedData->createDataSize), [](bool ok) + node.put(InfoHash(requestResponseKey, OPENDHT_INFOHASH_LEN), Value((u8*)requestedData->data.data, requestedData->data.size), [](bool ok) { if(!ok) - fprintf(stderr, "Failed to put response for old data\n"); + fprintf(stderr, "Failed to put response for old data for 'create' data\n"); }); } - else + + for(auto requestedObject : requestedData->objects) { - assert(false); - printf("TODO: Send 'add' packets to requesting remote peer\n"); + node.put(InfoHash(requestResponseKey, OPENDHT_INFOHASH_LEN), Value((u8*)requestedObject->data.data, requestedObject->data.size), [](bool ok) + { + if(!ok) + fprintf(stderr, "Failed to put response for old data for 'add' data\n"); + }); } } catch (sibs::DeserializeException &e) @@ -203,39 +234,41 @@ namespace odhtdb //node.listen(ADD_DATA_HASH, bind(&Database::listenAddData, this, _1)); } - unique_ptr Database::create(const LocalUser *owner, const std::string &name) + unique_ptr Database::create(const std::string &ownerName, const std::string &nodeName) { + LocalUser *nodeAdminUser = LocalUser::create(Signature::KeyPair(), ownerName); + auto adminGroup = new Group("administrator"); + adminGroup->addUser(nodeAdminUser); + // Header sibs::SafeSerializer serializer; serializer.add(DATABASE_CREATE_PACKET_STRUCTURE_VERSION); // Packet structure version // TODO: Append fractions to get real microseconds time u64 timestampMicroseconds = ((u64)getSyncedTimestampUtc().seconds) * 1000000ull; serializer.add(timestampMicroseconds); - serializer.add((u8*)owner->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES); + serializer.add((u8*)nodeAdminUser->getPublicKey().getData(), PUBLIC_KEY_NUM_BYTES); // Encrypted body sibs::SafeSerializer encryptedSerializer; - assert(owner->getName().size() <= 255); - encryptedSerializer.add((u8)owner->getName().size()); - encryptedSerializer.add((u8*)owner->getName().data(), owner->getName().size()); - assert(name.size() <= 255); - encryptedSerializer.add((u8)name.size()); - encryptedSerializer.add((u8*)name.data(), name.size()); + assert(nodeAdminUser->getName().size() <= 255); + encryptedSerializer.add((u8)nodeAdminUser->getName().size()); + encryptedSerializer.add((u8*)nodeAdminUser->getName().data(), nodeAdminUser->getName().size()); + assert(nodeName.size() <= 255); + encryptedSerializer.add((u8)nodeName.size()); + encryptedSerializer.add((u8*)nodeName.data(), nodeName.size()); try { Encryption encryptedBody(DataView(encryptedSerializer.getBuffer().data(), encryptedSerializer.getBuffer().size())); DataView requestData = combine(serializer, encryptedBody); shared_ptr hashRequestKey = make_shared(requestData.data, requestData.size); - auto adminGroup = new Group("administrator"); - adminGroup->addUser(owner); databaseStorage.createStorage(*hashRequestKey, adminGroup, timestampMicroseconds, (const u8*)requestData.data, requestData.size); - stagedCreateObjects.emplace_back(make_unique(requestData, hashRequestKey)); + stagedCreateObjects.emplace_back(make_unique(requestData, hashRequestKey)); assert(encryptedBody.getKey().size == KEY_BYTE_SIZE); char *key = new char[encryptedBody.getKey().size]; memcpy(key, encryptedBody.getKey().data, encryptedBody.getKey().size); - return make_unique(make_shared(key), hashRequestKey); + return make_unique(nodeAdminUser, make_shared(key), hashRequestKey); } catch (EncryptionException &e) { @@ -243,17 +276,22 @@ namespace odhtdb } } - void Database::add(const LocalUser *owner, const Key &key, DataView data) + void Database::add(const unique_ptr &nodeInfo, DataView dataToAdd) { -#if 0 - if(nodeEncryptionKeys.find(key) == nodeEncryptionKeys.end()) - throw DatabaseAddException("Data for key needs to be created before data can be appended to it"); - - unique_ptr signedData = make_unique(owner->getPrivateKey().sign(data)); + sibs::SafeSerializer serializer; + serializer.add(DATABASE_ADD_PACKET_STRUCTURE_VERSION); // TODO: Append fractions to get real microseconds time - u64 timeMicroseconds = ((u64)getSyncedTimestampUtc().seconds) * 1000000ull; - stagedAddObjects.emplace_back(StagedAddObject(key, move(signedData), timeMicroseconds, owner->getPublicKey())); -#endif + u64 timestampMicroseconds = ((u64)getSyncedTimestampUtc().seconds) * 1000000ull; + serializer.add(timestampMicroseconds); + + DataView encryptionKey(*nodeInfo->getNodeEncryptionKey(), KEY_BYTE_SIZE); + Encryption encryptedBody(dataToAdd, DataView(), encryptionKey); + DataView requestData = combine(serializer, encryptedBody); + string signedRequestData = nodeInfo->getNodeAdminUser()->getPrivateKey().sign(requestData); + free(requestData.data); + DataView stagedAddObject = combine(nodeInfo->getNodeAdminUser()->getPublicKey(), signedRequestData); + // TODO: Add add object to database storage here for local user + stagedAddObjects.emplace_back(make_unique(stagedAddObject, nodeInfo->getRequestHash())); } void Database::commit() @@ -268,6 +306,12 @@ namespace odhtdb { commitStagedCreateObject(stagedObject); } + + printf("Num objects to add: %zu\n", stagedAddObjects.size()); + for(const auto &stagedObject : stagedAddObjects) + { + commitStagedAddObject(stagedObject); + } } catch (exception &e) { @@ -276,24 +320,23 @@ namespace odhtdb for(const auto &stagedObject : stagedCreateObjects) { - free(stagedObject->encryptedBody.data); + free(stagedObject->data.data); } stagedCreateObjects.clear(); -#if 0 - printf("Num objects to add: %d\n", stagedAddObjects.size()); - for(StagedAddObject &stagedObject : stagedAddObjects) + + for(const auto &stagedObject : stagedAddObjects) { - commitStagedAddObject(stagedObject); + free(stagedObject->data.data); } stagedAddObjects.clear(); -#endif + // TODO: Add node.listen here to get notified when remote peers got the commit, then we can say we can return } - void Database::commitStagedCreateObject(const unique_ptr &stagedObject) + void Database::commitStagedCreateObject(const unique_ptr &stagedObject) { DhtKey dhtKey(*stagedObject->requestKey); - Value createDataValue((u8*)stagedObject->encryptedBody.data, stagedObject->encryptedBody.size); + Value createDataValue((u8*)stagedObject->data.data, stagedObject->data.size); node.put(dhtKey.getNewDataListenerKey(), move(createDataValue), [](bool ok) { // TODO: Handle failure to put data @@ -302,48 +345,16 @@ namespace odhtdb }/* TODO: How to make this work?, time_point(), false*/); } - void Database::commitStagedAddObject(const DataView &stagedObject) + void Database::commitStagedAddObject(const unique_ptr &stagedObject) { -#if 0 - // TODO: Implement gas and price (refill when serving content (seeding) or by waiting. This is done to prevent spamming and bandwidth leeching) - sibs::SafeSerializer headerSerializer; - assert(stagedObject.key.hashedKey.size() == OPENDHT_INFOHASH_LEN); - headerSerializer.add(stagedObject.key.hashedKey.data(), OPENDHT_INFOHASH_LEN); - headerSerializer.add(stagedObject.timestamp); - - sibs::SafeSerializer bodySerializer; - bodySerializer.add((u8*)stagedObject.creatorPublicKey.getData(), PUBLIC_KEY_NUM_BYTES); - assert(stagedObject.data->size() < 0xFFFF - 120); - bodySerializer.add((u16)stagedObject.data->size()); - bodySerializer.add((u8*)stagedObject.data->data(), stagedObject.data->size()); - - EncryptedData encryptedData; - if(encrypt(&encryptedData, (EncryptionKey*)nodeEncryptionKeys[stagedObject.key], bodySerializer.getBuffer().data(), bodySerializer.getBuffer().size()) < 0) - throw CommitAddException("Failed to encrypt staged add object"); - - Blob serializedData; - combine(&serializedData, headerSerializer, encryptedData); - - // TODO: Verify if serializer buffer needs to survive longer than this scope - Value addDataValue(move(serializedData)); - node.put(ADD_DATA_HASH, move(addDataValue), [](bool ok) - { - // TODO: Handle failure to put data - if(!ok) - fprintf(stderr, "Failed to put for all: %s, what to do?\n", "commitStagedAddObject"); - }); - - // Post data for listeners of this key - /* - Value putKeyValue(serializer.getBuffer().data() + OPENDHT_INFOHASH_LEN, serializer.getBuffer().size() - OPENDHT_INFOHASH_LEN); - node.put(stagedObject.key.hashedKey, move(putKeyValue), [](bool ok) + DhtKey dhtKey(*stagedObject->requestKey); + Value createDataValue((u8*)stagedObject->data.data, stagedObject->data.size); + node.put(dhtKey.getNewDataListenerKey(), move(createDataValue), [](bool ok) { // TODO: Handle failure to put data if(!ok) - fprintf(stderr, "Failed to put for listeners: %s, what to do?\n", "commitStagedAddObject"); - }); - */ -#endif + fprintf(stderr, "Failed to put: %s, what to do?\n", "commitStagedAddObject"); + }/* TODO: How to make this work?, time_point(), false*/); } ntp::NtpTimestamp Database::getSyncedTimestampUtc() const @@ -382,7 +393,7 @@ namespace odhtdb throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)"); auto adminGroup = new Group("administrator"); - auto creatorUser = RemoteUser::create(userPublicKey, ""); // Username is encrypted, we dont know it... + auto creatorUser = RemoteUser::create(userPublicKey, "ENCRYPTED USER NAME"); // Username is encrypted, we dont know it... adminGroup->addUser(creatorUser); databaseStorage.createStorage(hash, adminGroup, creationDate, value->data.data(), value->data.size()); @@ -400,42 +411,61 @@ namespace odhtdb u8 nameLength = bodyDeserializer.extract(); string name; name.resize(nameLength); - bodyDeserializer.extract((u8*)&name[0], nameLength); + bodyDeserializer.extract((u8*)&name[0], nameLength); // TODO: Add this user name to storage added above return { creationDate, adminGroup, move(name) }; } DatabaseAddRequest Database::deserializeAddRequest(const std::shared_ptr &value, const Hash &hash, const shared_ptr encryptionKey) { - /* - StagedAddObject result; - sibs::SafeDeserializer deserializer(value->data.data(), value->data.size()); - u8 entryKeyRaw[OPENDHT_INFOHASH_LEN]; - deserializer.extract(entryKeyRaw, OPENDHT_INFOHASH_LEN); - result.key.hashedKey = InfoHash(entryKeyRaw, OPENDHT_INFOHASH_LEN); - result.timestamp = deserializer.extract(); - char creatorPublicKeyRaw[PUBLIC_KEY_NUM_BYTES]; deserializer.extract((u8*)creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); - Signature::PublicKey creatorPublicKey(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); + Signature::PublicKey userPublicKey(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES); - u16 dataSize = deserializer.extract(); - if(dataSize < SIGNED_HASH_SIZE) - throw sibs::DeserializeException("Signed data is too small"); + DataView signedData((void*)deserializer.getBuffer(), deserializer.getSize()); + string unsignedData = userPublicKey.unsign(signedData); + sibs::SafeDeserializer deserializerUnsigned((u8*)unsignedData.data(), unsignedData.size()); - string signedData; - signedData.resize(dataSize); - deserializer.extract((u8*)&signedData[0], dataSize); - result.data = make_unique(); - result.data->resize(dataSize); - result.data = make_unique(creatorPublicKey.unsign(DataView((void*)signedData.data(), signedData.size()))); - - return result; - */ - Signature::PublicKey publicKey(nullptr, 0); - DataView d; - return { 0, 0, move(publicKey), d }; + u16 packetStructureVersion = deserializerUnsigned.extract(); + if(packetStructureVersion != DATABASE_CREATE_PACKET_STRUCTURE_VERSION) + { + string errMsg = "Received 'create' request with packet structure version "; + errMsg += to_string(packetStructureVersion); + errMsg += ", but our packet structure version is "; + errMsg += to_string(DATABASE_CREATE_PACKET_STRUCTURE_VERSION); + throw sibs::DeserializeException(errMsg); + } + + u64 creationDate = deserializerUnsigned.extract(); + // TODO: Append fractions to get real microseconds time + u64 timestampMicroseconds = ((u64)getSyncedTimestampUtc().seconds) * 1000000ull; + if(creationDate > timestampMicroseconds) + throw sibs::DeserializeException("Packet is from the future"); + + if(deserializerUnsigned.getSize() < NONCE_BYTE_SIZE) + throw sibs::DeserializeException("Unsigned encrypted body is too small (unable to extract nonce)"); + + const Hash *node = databaseStorage.getNodeByUserPublicKey(userPublicKey); + if(!node) + { + // The user (public key) could belong to a node but we might not have retrieved the node info yet since data may + // not be retrieved in order. + // Data in quarantine is processed when 'create' packet is received or removed after 60 seconds + databaseStorage.addToQuarantine(userPublicKey, creationDate, value->data.data(), value->data.size()); + throw RequestQuarantineException(); + } + + auto creatorUser = databaseStorage.getUserByPublicKey(userPublicKey); + // TODO: Verify there isn't already data with same timestamp for this node. Same for quarantine + databaseStorage.appendStorage(*node, creatorUser, creationDate, value->data.data(), value->data.size()); + + u8 nonce[NONCE_BYTE_SIZE]; + deserializerUnsigned.extract(nonce, NONCE_BYTE_SIZE); + DataView dataToDecrypt((void*)deserializerUnsigned.getBuffer(), deserializerUnsigned.getSize()); + Decryption decryptedBody(dataToDecrypt, DataView(nonce, NONCE_BYTE_SIZE), DataView(*encryptionKey, KEY_BYTE_SIZE)); + + return { creationDate, creatorUser, move(decryptedBody) }; } bool Database::listenCreateData(std::shared_ptr value, const Hash &hash, const shared_ptr encryptionKey) @@ -460,11 +490,12 @@ namespace odhtdb printf("Got add data\n"); try { - if(databaseStorage.getStorage(hash)) - throw DatabaseStorageAlreadyExists("Add request hash is equal to hash already in storage (duplicate data?)"); - // TODO: Verify createObject timestamp is not in the future - //StagedAddObject addObject = deserializeAddRequest(value); - //DataView data((void*)addObject.data->data(), addObject.data->size()); + DatabaseAddRequest addObject = deserializeAddRequest(value, hash, encryptionKey); + printf("Got add object, timestamp: %zu\n", addObject.timestamp); + } + catch (RequestQuarantineException &e) + { + fprintf(stderr, "Warning: Request was put in quarantine, will be processed later"); } catch (exception &e) { diff --git a/src/DatabaseStorage.cpp b/src/DatabaseStorage.cpp index 25d41fb..638eac0 100644 --- a/src/DatabaseStorage.cpp +++ b/src/DatabaseStorage.cpp @@ -1,13 +1,28 @@ #include "../include/DatabaseStorage.hpp" +#include "../include/Group.hpp" +#include "../include/User.hpp" #include +#include using namespace std; namespace odhtdb { + DatabaseStorageObject::DatabaseStorageObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : + data(_data), createdTimestamp(_timestamp), creatorPublicKey(_creatorPublicKey) + { + + } + + DatabaseStorageQuarantineObject::DatabaseStorageQuarantineObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : + data(_data), createdTimestamp(_timestamp), creatorPublicKey(_creatorPublicKey) + { + auto time = chrono::high_resolution_clock::now().time_since_epoch(); + storedTimestamp = chrono::duration_cast(time).count(); + } + void DatabaseStorage::createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize) { - /* if(storageMap.find(hash) != storageMap.end()) { string errMsg = "Database storage with hash "; @@ -15,18 +30,22 @@ namespace odhtdb errMsg += " already exists"; throw DatabaseStorageAlreadyExists(errMsg); } - */ DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList(); - databaseStorageObjectList->timestamp = timestamp; + databaseStorageObjectList->createdTimestamp = timestamp; databaseStorageObjectList->groups.push_back(creatorGroup); - databaseStorageObjectList->createData = new u8[dataSize]; - memcpy(databaseStorageObjectList->createData, data, dataSize); - databaseStorageObjectList->createDataSize = dataSize; + databaseStorageObjectList->data = DataView(new u8[dataSize], dataSize); + memcpy(databaseStorageObjectList->data.data, data, dataSize); storageMap[hash] = databaseStorageObjectList; + + for(auto user : creatorGroup->getUsers()) + { + userPublicKeyNodeMap[user->getPublicKey()] = hash; + publicKeyUserMap[user->getPublicKey()] = user; + } } - void DatabaseStorage::appendStorage(const Hash &hash, DataView &data, u64 timestamp, const Signature::PublicKey &creatorPublicKey) + void DatabaseStorage::appendStorage(const Hash &hash, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize) { auto it = storageMap.find(hash); if(it == storageMap.end()) @@ -36,7 +55,18 @@ namespace odhtdb errMsg += " not found. Storage for a hash needs to be created before data can be appended to it"; throw DatabaseStorageNotFound(errMsg); } - it->second->objects.push_back({data, timestamp, creatorPublicKey}); + + DataView storageData { new u8[dataSize], dataSize }; + DatabaseStorageObject *databaseStorageObject = new DatabaseStorageObject(storageData, timestamp, creatorUser->getPublicKey()); + it->second->objects.push_back(databaseStorageObject); + } + + void DatabaseStorage::addToQuarantine(const Signature::PublicKey &creatorPublicKey, u64 timestamp, const u8 *data, usize dataSize) + { + DataView storageData { new u8[dataSize], dataSize }; + memcpy(storageData.data, data, dataSize); + DatabaseStorageQuarantineObject *databaseQuarantineStorageObject = new DatabaseStorageQuarantineObject(storageData, timestamp, creatorPublicKey); + quarantineStorageMap[creatorPublicKey].emplace_back(databaseQuarantineStorageObject); } const DatabaseStorageObjectList* DatabaseStorage::getStorage(const Hash &hash) const @@ -46,4 +76,21 @@ namespace odhtdb return it->second; return nullptr; } + + const Hash* DatabaseStorage::getNodeByUserPublicKey(const Signature::PublicKey &userPublicKey) const + { + auto it = userPublicKeyNodeMap.find(userPublicKey); + if(it != userPublicKeyNodeMap.end()) + return &it->second; + return nullptr; + } + + // Returns nullptr if no user with public key exists + const User* DatabaseStorage::getUserByPublicKey(const Signature::PublicKey &userPublicKey) const + { + auto it = publicKeyUserMap.find(userPublicKey); + if(it != publicKeyUserMap.end()) + return it->second; + return nullptr; + } } diff --git a/src/Encryption.cpp b/src/Encryption.cpp index c4e6a2c..238861f 100644 --- a/src/Encryption.cpp +++ b/src/Encryption.cpp @@ -1,16 +1,26 @@ #include "../include/Encryption.hpp" #include #include -#include +#include namespace odhtdb { - Encryption::Encryption(const DataView &data, const DataView &additionalData) + Encryption::Encryption(const DataView &data, const DataView &additionalData, const DataView &_key) { - cipherText = new unsigned char[crypto_aead_xchacha20poly1305_ietf_ABYTES + data.size]; - crypto_aead_xchacha20poly1305_ietf_keygen(key); + cipherText = new unsigned char[crypto_aead_xchacha20poly1305_ietf_ABYTES + data.size]; + cipherTextLength = crypto_aead_xchacha20poly1305_ietf_ABYTES + data.size; + + if(_key.data) + { + if(_key.size != KEY_BYTE_SIZE) + throw EncryptionException("Encryption key is wrong size"); + memcpy(key, _key.data, _key.size); + } + else + crypto_aead_xchacha20poly1305_ietf_keygen(key); + randombytes_buf(nonce, NONCE_BYTE_SIZE); - if(crypto_aead_xchacha20poly1305_ietf_encrypt(cipherText, &cipherTextLength, (const unsigned char*)data.data, data.size, (const unsigned char*)additionalData.data, additionalData.size, nullptr, nonce, key) != 0) + if(crypto_aead_xchacha20poly1305_ietf_encrypt(cipherText, &cipherTextLength, (const unsigned char*)data.data, data.size, (const unsigned char*)additionalData.data, additionalData.size, nullptr, nonce, key) < 0) throw EncryptionException("Failed to encrypt data"); } @@ -37,6 +47,7 @@ namespace odhtdb Decryption::Decryption(const DataView &data, const DataView &nonce, const DataView &key) { decryptedText = new unsigned char[data.size]; + decryptedTextLength = data.size; if(nonce.size < NONCE_BYTE_SIZE) throw DecryptionException("Nonce is not big enough"); @@ -44,10 +55,29 @@ namespace odhtdb if(key.size < KEY_BYTE_SIZE) throw DecryptionException("Key is not big enough"); - if(crypto_aead_xchacha20poly1305_ietf_decrypt(decryptedText, &decryptedTextLength, nullptr, (const unsigned char*)data.data, data.size, nullptr, 0, (const unsigned char*)nonce.data, (const unsigned char*)key.data) != 0) + if(crypto_aead_xchacha20poly1305_ietf_decrypt(decryptedText, &decryptedTextLength, nullptr, (const unsigned char*)data.data, data.size, nullptr, 0, (const unsigned char*)nonce.data, (const unsigned char*)key.data) < 0) throw DecryptionException("Failed to decrypt data"); } + Decryption::Decryption(Decryption &&other) + { + decryptedText = other.decryptedText; + decryptedTextLength = other.decryptedTextLength; + + other.decryptedText = nullptr; + other.decryptedTextLength = 0; + } + + Decryption& Decryption::operator=(Decryption &&other) + { + decryptedText = other.decryptedText; + decryptedTextLength = other.decryptedTextLength; + + other.decryptedText = nullptr; + other.decryptedTextLength = 0; + return *this; + } + Decryption::~Decryption() { delete[](decryptedText); diff --git a/src/Hash.cpp b/src/Hash.cpp index 5d2f914..91bc062 100644 --- a/src/Hash.cpp +++ b/src/Hash.cpp @@ -17,15 +17,6 @@ namespace odhtdb static SodiumInitializer __sodiumInitializer; - // Source: https://stackoverflow.com/a/11414104 (public license) - static size_t fnvHash(const unsigned char *key, int len) - { - size_t h = 2166136261; - for (int i = 0; i < len; i++) - h = (h * 16777619) ^ key[i]; - return h; - } - Hash::Hash() { memset(data, 0, HASH_BYTE_SIZE); @@ -38,6 +29,11 @@ namespace odhtdb throw HashException("Failed to hash data using blake2b"); } + Hash::Hash(const Hash &other) + { + memcpy(data, other.data, HASH_BYTE_SIZE); + } + size_t Hash::operator()() const { return fnvHash((const unsigned char*)data, HASH_BYTE_SIZE); diff --git a/src/Signature.cpp b/src/Signature.cpp index 34f6190..d328b58 100644 --- a/src/Signature.cpp +++ b/src/Signature.cpp @@ -1,4 +1,5 @@ #include "../include/Signature.hpp" +#include "../include/Hash.hpp" #include #include #include @@ -48,10 +49,20 @@ namespace odhtdb return result; } + size_t PublicKey::operator()() const + { + return fnvHash((const unsigned char*)data, PUBLIC_KEY_NUM_BYTES); + } + + bool PublicKey::operator==(const PublicKey &other) const + { + return memcmp(data, other.data, PUBLIC_KEY_NUM_BYTES) == 0; + } + string PublicKey::toString() const { string result; - result.resize(PUBLIC_KEY_NUM_BYTES * 2); + result.resize(PUBLIC_KEY_NUM_BYTES * 2 + 1); sodium_bin2hex(&result[0], PUBLIC_KEY_NUM_BYTES * 2 + 1, (const unsigned char*)data, PUBLIC_KEY_NUM_BYTES); return result; } diff --git a/tests/main.cpp b/tests/main.cpp index 3c7e798..0218c8c 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -90,18 +90,11 @@ int main() // TODO: Setup local bootstrap node for tests Database database("bootstrap.ring.cx", 4222, "storage"); - auto databaseCreateResponse = database.create(localUser, "latenight"); - - /* + auto databaseCreateResponse = database.create("dec05eba", "latenight"); const char *data = "hello, world!"; - database.add(localUser, "galax.channel.latenight.chat", DataView{ (void*)data, strlen(data) }); + database.add(databaseCreateResponse, DataView{ (void*)data, strlen(data) }); database.commit(); - auto start = chrono::high_resolution_clock::now(); - while(chrono::high_resolution_clock::now() - start < 3s) - { - this_thread::sleep_for(10ms); - } - */ + database.seed(databaseCreateResponse->getRequestHash(), databaseCreateResponse->getNodeEncryptionKey()); auto start = chrono::high_resolution_clock::now(); while(chrono::high_resolution_clock::now() - start < 3s) -- cgit v1.2.3