Add database storage (in memory), need to store it on disk later

10 files changed, 179 insertions, 34 deletions

diff --git a/include/DataView.hpp b/include/DataView.hpp
index 982cb8a..c020f91 100644
--- a/include/DataView.hpp
+++ b/include/DataView.hpp
@@ -10,7 +10,7 @@ namespace odhtdb
         DataView() : data(nullptr), size(0) {}
         DataView(void *_data, usize _size) : data(_data), size(_size) {}
 
-        const void *data;
-        const usize size;
+        void *data;
+        usize size;
     };
-}
\ No newline at end of file
+}
diff --git a/include/Database.hpp b/include/Database.hpp
index bfc3021..0104a6e 100644
--- a/include/Database.hpp
+++ b/include/Database.hpp
@@ -4,6 +4,7 @@
 #include "Key.hpp"
 #include "StagedObject.hpp"
 #include "DataView.hpp"
+#include "DatabaseStorage.hpp"
 #include <opendht/dhtrunner.h>
 #include <vector>
 #include <ntp/NtpClient.hpp>
@@ -12,6 +13,7 @@
 namespace odhtdb
 {
     class Group;
+    class LocalUser;
 
     class Database
     {
@@ -21,7 +23,7 @@ namespace odhtdb
 
         void seed();
         void create(const Key &key, Group *primaryAdminGroup);
-        void add(const Key &key, DataView data);
+        void add(const Key &key, DataView data, LocalUser *creator);
         void commit();
     private:
         void commitStagedCreateObject(const StagedCreateObject &stagedObject);
@@ -35,5 +37,6 @@ namespace odhtdb
         dht::DhtRunner node;
         std::vector<StagedCreateObject> stagedCreateObjects;
         std::vector<StagedAddObject> stagedAddObjects;
+        DatabaseStorage databaseStorage;
     };
 }
diff --git a/include/DatabaseStorage.hpp b/include/DatabaseStorage.hpp
index fee6b72..863c5d9 100644
--- a/include/DatabaseStorage.hpp
+++ b/include/DatabaseStorage.hpp
@@ -1,10 +1,58 @@
 #pragma once
 
+#include "Key.hpp"
+#include "DataView.hpp"
+#include "Signature.hpp"
+#include <vector>
+#include <stdexcept>
+
 namespace odhtdb
 {
+    class Group;
+    
+    struct DatabaseStorageObject
+    {
+        DataView data;
+        u64 timestamp; // In microseconds
+        Signature::PublicKey creatorPublicKey;
+        
+        DatabaseStorageObject(DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : 
+            data(_data), timestamp(_timestamp), creatorPublicKey(_creatorPublicKey)
+        {
+            
+        }
+    };
+    
+    struct DatabaseStorageObjectList
+    {
+        u64 timestamp; // In microseconds
+        std::vector<Group*> groups;
+        std::vector<DatabaseStorageObject> objects;
+    };
+    
+    class DatabaseStorageAlreadyExists : public std::runtime_error
+    {
+    public:
+        DatabaseStorageAlreadyExists(const std::string &errMsg) : std::runtime_error(errMsg) {}
+    };
+    
+    class DatabaseStorageNotFound : public std::runtime_error
+    {
+    public:
+        DatabaseStorageNotFound(const std::string &errMsg) : std::runtime_error(errMsg) {}
+    };
+    
+    using DatabaseStorageMap = KeyMap<DatabaseStorageObjectList*>;
+    
     class DatabaseStorage
     {
     public:
+        // Throws DatabaseStorageAlreadyExists if data with key already exists
+        void createStorage(const Key &key, std::vector<Group*> &&groups, u64 timestamp);
         
+        // Throws DatabaseStorageNotFound if data with key does not exist
+        void appendStorage(const Key &key, DataView &data, u64 timestamp, const Signature::PublicKey &creatorPublicKey);
+    private:
+        DatabaseStorageMap storageMap;
     };
 }
diff --git a/include/Key.hpp b/include/Key.hpp
index 505050d..f7a600b 100644
--- a/include/Key.hpp
+++ b/include/Key.hpp
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <opendht/infohash.h>
+#include <unordered_map>
 
 namespace odhtdb
 {
@@ -12,4 +13,32 @@ namespace odhtdb
         
         dht::InfoHash hashedKey;
     };
-}
\ No newline at end of file
+    
+    // Source: https://stackoverflow.com/a/11414104 (public license)
+    static unsigned int fnvHash(const unsigned char *key, int len)
+    {
+        unsigned int h = 2166136261;
+        for (int i = 0; i < len; i++)
+            h = (h * 16777619) ^ key[i];
+        return h;
+    }
+    
+    struct KeyHash
+    {
+        size_t operator()(const Key &key) const
+        {
+            return fnvHash(key.hashedKey.data(), key.hashedKey.size());
+        }
+    };
+    
+    struct KeyCompare
+    {
+        bool operator()(const Key &lhs, const Key &rhs) const
+        {
+            return lhs.hashedKey == rhs.hashedKey;
+        }
+    };
+    
+    template <typename ValueType>
+    using KeyMap = std::unordered_map<Key, ValueType, KeyHash, KeyCompare>;
+}
diff --git a/include/Signature.hpp b/include/Signature.hpp
index 90d5278..ea776ea 100644
--- a/include/Signature.hpp
+++ b/include/Signature.hpp
@@ -31,8 +31,10 @@ namespace odhtdb
         {
             friend class KeyPair;
         public:
+            static PublicKey ZERO;
+            
             // Throws InvalidSignatureKeySize if size is not PUBLIC_KEY_NUM_BYTES
-            PublicKey(char *data, size_t size);
+            PublicKey(const char *data, size_t size);
             PublicKey(const PublicKey &other);
             PublicKey& operator=(const PublicKey &other);
             
@@ -51,7 +53,7 @@ namespace odhtdb
             friend class KeyPair;
         public:
             // Throws InvalidSignatureKeySize if size is not PRIVATE_KEY_NUM_BYTES
-            PrivateKey(char *data, size_t size);
+            PrivateKey(const char *data, size_t size);
             PrivateKey(const PrivateKey &other);
             PrivateKey& operator=(const PrivateKey &other);
             
diff --git a/include/StagedObject.hpp b/include/StagedObject.hpp
index 61e1073..dc2aaf4 100644
--- a/include/StagedObject.hpp
+++ b/include/StagedObject.hpp
@@ -3,6 +3,7 @@
 #include "Key.hpp"
 #include "types.hpp"
 #include "DataView.hpp"
+#include "Signature.hpp"
 
 namespace odhtdb
 {
@@ -27,12 +28,13 @@ namespace odhtdb
         Key key;
         DataView data;
         u64 timestamp; // In microseconds
+        Signature::PublicKey creatorPublicKey;
 
-        StagedAddObject() : key(), data(), timestamp(0) {}
-        StagedAddObject(const Key &_key, const DataView &_data, u64 _timestamp) : 
-            key(_key), data(_data), timestamp(_timestamp) 
+        StagedAddObject() : key(), data(), timestamp(0), creatorPublicKey(Signature::PublicKey::ZERO) {}
+        StagedAddObject(const Key &_key, const DataView &_data, u64 _timestamp, const Signature::PublicKey &_creatorPublicKey) : 
+            key(_key), data(_data), timestamp(_timestamp), creatorPublicKey(_creatorPublicKey)
         {
 
         }
     };
-}
\ No newline at end of file
+}
diff --git a/src/Database.cpp b/src/Database.cpp
index cd87845..e3b9f3d 100644
--- a/src/Database.cpp
+++ b/src/Database.cpp
@@ -92,11 +92,11 @@ namespace odhtdb
         stagedCreateObjects.emplace_back(StagedCreateObject(key, primaryAdminGroup, timeMicroseconds));
     }
 
-    void Database::add(const Key &key, DataView data)
+    void Database::add(const Key &key, DataView data, LocalUser *creator)
     {
         // TODO: Append fractions to get real microseconds time
         u64 timeMicroseconds = ((u64)getSyncedTimestampUtc().seconds) * 1000000ull;
-        stagedAddObjects.emplace_back(StagedAddObject(key, data, timeMicroseconds));
+        stagedAddObjects.emplace_back(StagedAddObject(key, data, timeMicroseconds, creator->getPublicKey()));
     }
 
     void Database::commit()
@@ -121,6 +121,13 @@ namespace odhtdb
         // TODO: Add node.listen here to get notified when remote peers got the commit, then we can say we can return
     }
 
+    // TODO: If same key already exists, fail the operation.
+    // Security issue: A malicious remote peer (or routing peer) could listen to this create request and build their own
+    // create request using same key, to steal ownership of the key.
+    // Possible solution: If odhtdb is for example used to build a chat application, then the key could be the chat channel id
+    // which could be created by hashing channel generated id and ownership information.
+    // Remote peers would then not be able to steal ownership of the key since hash of ownership data has to match the key.
+    // The key (channel id + ownership info) could then be shared with friends and they can use the key to join your channel.
     void Database::commitStagedCreateObject(const StagedCreateObject &stagedObject)
     {
         // TODO: Use (ed25519 or poly1305) and curve25519
@@ -150,6 +157,7 @@ namespace odhtdb
         }/* TODO: How to make this work?, time_point(), false*/);
 
         // Post data for listeners of this key
+        /*
         Value putKeyValue(serializer.getBuffer().data() + OPENDHT_INFOHASH_LEN, serializer.getBuffer().size() - OPENDHT_INFOHASH_LEN);
         node.put(stagedObject.key.hashedKey, move(putKeyValue), [](bool ok)
         {
@@ -157,6 +165,7 @@ namespace odhtdb
             if(!ok)
                 fprintf(stderr, "Failed to put for listeners: %s, what to do?\n", "commitStagedCreateObject");
         });
+        */
     }
 
     void Database::commitStagedAddObject(const StagedAddObject &stagedObject)
@@ -165,8 +174,11 @@ namespace odhtdb
         // TODO: Implement gas and price (refill when serving content (seeding) or by waiting. This is done to prevent spamming and bandwidth leeching)
         sibs::SafeSerializer serializer;
         assert(stagedObject.key.hashedKey.size() == OPENDHT_INFOHASH_LEN);
-        serializer.add(stagedObject.key.hashedKey.data(), stagedObject.key.hashedKey.size());
+        serializer.add(stagedObject.key.hashedKey.data(), OPENDHT_INFOHASH_LEN);
         serializer.add(stagedObject.timestamp);
+        serializer.add((u8*)stagedObject.creatorPublicKey.getData(), PUBLIC_KEY_NUM_BYTES);
+        assert(stagedObject.data.size < 0xFFFF - 120);
+        serializer.add((u16)stagedObject.data.size);
         serializer.add((u8*)stagedObject.data.data, stagedObject.data.size);
 
         // TODO: Verify if serializer buffer needs to survive longer than this scope
@@ -176,16 +188,18 @@ namespace odhtdb
             // TODO: Handle failure to put data
             if(!ok)
                 fprintf(stderr, "Failed to put for all: %s, what to do?\n", "commitStagedAddObject");
-        }, time_point(), false);
+        });
 
         // Post data for listeners of this key
+        /*
         Value putKeyValue(serializer.getBuffer().data() + OPENDHT_INFOHASH_LEN, serializer.getBuffer().size() - OPENDHT_INFOHASH_LEN);
         node.put(stagedObject.key.hashedKey, move(putKeyValue), [](bool ok)
         {
             // TODO: Handle failure to put data
             if(!ok)
                 fprintf(stderr, "Failed to put for listeners: %s, what to do?\n", "commitStagedAddObject");
-        }, time_point(), false);
+        });
+        */
     }
 
     ntp::NtpTimestamp Database::getSyncedTimestampUtc() const
@@ -242,6 +256,17 @@ namespace odhtdb
         deserializer.extract(entryKeyRaw, OPENDHT_INFOHASH_LEN);
         result.key.hashedKey = InfoHash(entryKeyRaw, OPENDHT_INFOHASH_LEN);
         result.timestamp = deserializer.extract<u64>();
+        
+        char creatorPublicKeyRaw[PUBLIC_KEY_NUM_BYTES];
+        deserializer.extract((u8*)creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
+        Signature::PublicKey creatorPublicKey(creatorPublicKeyRaw, PUBLIC_KEY_NUM_BYTES);
+        
+        u16 dataSize = deserializer.extract<u16>();
+        char *data = (char*)malloc(dataSize);
+        if(!data)
+            throw sibs::DeserializeException("Failed to allocate memory for add request");
+        result.data.data = data;
+        result.data.size = dataSize;
 
         return result;
     }
@@ -253,12 +278,17 @@ namespace odhtdb
         {
             // TODO: Verify createObject timestamp is not in the future
             StagedCreateObject createObject = deserializeCreateRequest(value);
-            delete createObject.primaryAdminGroup;
+            databaseStorage.createStorage(createObject.key, { createObject.primaryAdminGroup }, createObject.timestamp);
+            //delete createObject.primaryAdminGroup;
         }
         catch (sibs::DeserializeException &e)
         {
             fprintf(stderr, "Warning: Failed to deserialize 'create' request: %s\n", e.what());
         }
+        catch (DatabaseStorageAlreadyExists &e)
+        {
+            fprintf(stderr, "Warning: Failed to deserialize 'create' request: %s\n", e.what());
+        }
         return true;
     }
 
@@ -267,12 +297,19 @@ namespace odhtdb
         printf("Got add data\n");
         try
         {
+            // TODO: Verify createObject timestamp is not in the future
             StagedAddObject addObject = deserializeAddRequest(value);
+            databaseStorage.appendStorage(addObject.key, addObject.data, addObject.timestamp, addObject.creatorPublicKey);
+            //free(addObject.data.data);
         }
         catch (sibs::DeserializeException &e)
         {
             fprintf(stderr, "Warning: Failed to deserialize 'add' request: %s\n", e.what());
         }
+        catch (DatabaseStorageNotFound &e)
+        {
+            fprintf(stderr, "Warning: Failed to deserialize 'add' request: %s\n", e.what());
+        }
         return true;
     }
 }
diff --git a/src/DatabaseStorage.cpp b/src/DatabaseStorage.cpp
new file mode 100644
index 0000000..2028c63
--- /dev/null
+++ b/src/DatabaseStorage.cpp
@@ -0,0 +1,35 @@
+#include "../include/DatabaseStorage.hpp"
+
+using namespace std;
+
+namespace odhtdb
+{
+    void DatabaseStorage::createStorage(const Key &key, vector<Group*> &&groups, u64 timestamp)
+    {
+        if(storageMap.find(key) != storageMap.end())
+        {
+            string errMsg = "Database storage with key ";
+            errMsg += key.hashedKey.toString();
+            errMsg += " already exists";
+            throw DatabaseStorageAlreadyExists(errMsg);
+        }
+        
+        DatabaseStorageObjectList *databaseStorageObjectList = new DatabaseStorageObjectList();
+        databaseStorageObjectList->timestamp = timestamp;
+        databaseStorageObjectList->groups = move(groups);
+        storageMap[key] = databaseStorageObjectList;
+    }
+    
+    void DatabaseStorage::appendStorage(const Key &key, DataView &data, u64 timestamp, const Signature::PublicKey &creatorPublicKey)
+    {
+        auto it = storageMap.find(key);
+        if(it == storageMap.end())
+        {
+            string errMsg = "Database storage with key ";
+            errMsg += key.hashedKey.toString();
+            errMsg += " not found";
+            throw DatabaseStorageNotFound(errMsg);
+        }
+        it->second->objects.push_back({data, timestamp, creatorPublicKey});
+    }
+}
diff --git a/src/Signature.cpp b/src/Signature.cpp
index 804047e..946d754 100644
--- a/src/Signature.cpp
+++ b/src/Signature.cpp
@@ -9,7 +9,9 @@ namespace odhtdb
 {
     namespace Signature
     {
-        PublicKey::PublicKey(char *_data, size_t size)
+        PublicKey PublicKey::ZERO("\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", PUBLIC_KEY_NUM_BYTES);
+        
+        PublicKey::PublicKey(const char *_data, size_t size)
         {
             if(size != PUBLIC_KEY_NUM_BYTES)
             {
@@ -41,7 +43,7 @@ namespace odhtdb
             return result;
         }
         
-        PrivateKey::PrivateKey(char *_data, size_t size)
+        PrivateKey::PrivateKey(const char *_data, size_t size)
         {
             if(size != PRIVATE_KEY_NUM_BYTES)
             {
diff --git a/tests/main.cpp b/tests/main.cpp
index 5e53dc8..8818ff9 100644
--- a/tests/main.cpp
+++ b/tests/main.cpp
@@ -18,19 +18,6 @@ int main()
     
     std::string privateKeyStr = localUser->getPrivateKey().toString();
     printf("Local user private key: %s\n", privateKeyStr.c_str());
-    
-/*
-    char hex_ed_pk[65];
-    unsigned char seed[crypto_sign_ed25519_SEEDBYTES];
-    unsigned char ed25519_skpk[crypto_sign_ed25519_SECRETKEYBYTES];
-    unsigned char ed25519_pk[crypto_sign_ed25519_PUBLICKEYBYTES];
-
-    crypto_sign_ed25519_sk_to_seed(seed, ed25519_skpk);
-    crypto_sign_ed25519_seed_keypair(ed25519_pk, ed25519_skpk, seed);
-    sodium_bin2hex(hex_ed_pk, 65, ed25519_pk, 32);
-    printf("public key: %s\n", hex_ed_pk);
-    */
-
 
     // TODO: For tests, dont run against bootstrap.ring.cx.
     // Run against a bootstrap node made only for testing which doesn't persist added data.
@@ -42,8 +29,8 @@ int main()
     group.addUser(localUser);
     database.create("galax.channel.latenight.chat", &group);
 
-    //const char *data = "hello, world!";
-    //database.add("galax.channel.latenight.chat", DataView{ (void*)data, strlen(data) });
+    const char *data = "hello, world!";
+    database.add("galax.channel.latenight.chat", DataView{ (void*)data, strlen(data) }, localUser);
     database.commit();
     auto start = chrono::high_resolution_clock::now();
     while(chrono::high_resolution_clock::now() - start < 5s)