Replace files with sqlite

Using sqlite because sqlite has transactions, storing/loading from files
automatically, unloading data that is not accessed often.

Removed cosmetic data (node name, username). They can be added using
addData by the application that uses odhtdb instead.

Database callback functions can now be called with stored data using
database.loadNode function.

TODO: Add local user storage (with password) back, it has been temorary
disabled

1 files changed, 64 insertions, 70 deletions

diff --git a/include/odhtdb/DatabaseStorage.hpp b/include/odhtdb/DatabaseStorage.hpp
index 11e243a..3c4d9bc 100644
--- a/include/odhtdb/DatabaseStorage.hpp
+++ b/include/odhtdb/DatabaseStorage.hpp
@@ -15,6 +15,10 @@
 #include <boost/filesystem/path.hpp>
 #include <sibs/SafeDeserializer.hpp>
 #include <opendht/crypto.h>
+#include <functional>
+
+class sqlite3;
+class sqlite3_stmt;
 
 namespace odhtdb
 {
@@ -41,16 +45,12 @@ namespace odhtdb
     {
         Signature::PublicKey creatorPublicKey;
         DataView data;
-        u32 offsetToEncryptedData;
         u64 createdTimestamp; // In microseconds
-        std::string nodeName;
-        bool isDecrypted;
         std::vector<Group*> groups;
         std::vector<DatabaseStorageObject*> objects;
         
         DatabaseStorageObjectList(const Signature::PublicKey &_creatorPublicKey) : 
-            creatorPublicKey(_creatorPublicKey),
-            isDecrypted(false)
+            creatorPublicKey(_creatorPublicKey)
         {
             
         }
@@ -115,63 +115,46 @@ namespace odhtdb
         LocalUser *localUser;
     };
     
+    using FetchNodeRawCallbackFunc = std::function<void(const DataView)>;
+    using FetchNodeAddDataRawCallbackFunc = std::function<void(const DataView)>;
+    
     class DatabaseStorage
     {
     public:
-        // Throws DatabaseStorageCorrupt if storage is corrupted
+        // Throws DatabaseStorageCorrupt if storage is corrupted. Throws DatabaseStorageException on other failures
         DatabaseStorage(Database *database, const boost::filesystem::path &storagePath);
+        ~DatabaseStorage();
+
+        void loadNode(const Hash &nodeHash);
+        
+        bool doesNodeExist(const Hash &nodeHash) const;
+        bool doesDataExist(const Hash &requestHash) const;
         
         // Throws DatabaseStorageAlreadyExists if data with hash already exists
-        void createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize, u32 offsetToEncryptedData);
+        void createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const void *data, usize size);
         
         // Throws DatabaseStorageNotFound if data with @nodeHash hash has not been created yet.
         // Throws DatabaseStorageAlreadyExists if same data has been added before (hash of @data, in @dataHash)
-        void appendStorage(const Hash &nodeHash, const Hash &dataHash, DatabaseOperation operation, const User *creatorUser, u64 timestamp, const u8 *data, usize dataSize, const DataView &encryptedDataView);
-        
-        // Throws DatabaseStorageAlreadyExists if same data has been added before (hash of @data, in @dataHash)
-        void addToQuarantine(const Hash &dataHash, const Signature::PublicKey &creatorPublicKey, u64 timestamp, const u8 *data, usize dataSize);
-        
-        // Return false if group with id already exists, otherwise return true
-        bool addGroup(const Hash &nodeHash, Group *group);
+        void appendStorage(const Hash &nodeHash, const Hash &dataHash, DatabaseOperation operation, const Signature::PublicKey &creatorPublicKey, u64 timestamp, const void *data, usize size, const DataView &additionalDataView);
         
-        // Return false if user public key already exists, otherwise return true
-        bool addUser(const Hash &nodeHash, User *user);
+        // Throws DatabaseStorageAlreadyExists if group already exists in node
+        void addGroup(const Hash &nodeHash, Group *group);
         
-        // Increase user action counter, returning the new value. If user has not performed any action yet, then 0 is returned.
-        // Returns DatabaseStorageException if no node with id @nodeHash exists or if user with public key @userPublicKey doesn't exist in node @nodeHash
-        //u64 increaseUserActionCounter(const Hash &nodeHash, const Signature::PublicKey &userPublicKey);
+        void addUserToGroup(const Hash &nodeHash, const Signature::PublicKey &userPublicKey, const DataView &groupId);
         
-        // Returns nullptr if no storage with provided hash exists
-        const DatabaseStorageObjectList* getStorage(const Hash &hash) const;
+        // Throws DatabaseStorageAlreadyExists is user already exists in node
+        void addUser(const Hash &nodeHash, const Signature::PublicKey &userPublicKey, const DataView &groupId);
         
-        // Returns nullptr if node @nodeHash doesn't exist
-        const DataViewMap<Group*>* getNodeGroups(const Hash &nodeHash);
-        
-        // Returns nullptr if a group with id @groupId doesn't exist in node @nodeHash or if no node with id @nodeHash exists
-        Group* getGroupById(const Hash &nodeHash, uint8_t groupId[GROUP_ID_LENGTH]) const;
-        
-        // Returns nullptr if node @nodeHash doesn't exist
-        const Signature::MapPublicKey<User*>* getNodeUsers(const Hash &nodeHash);
-        
-        // Returns nullptr if a user with public key @publicKey doesn't exist in node @nodeHash or if no node with id @nodeHash exists
-        User* getUserByPublicKey(const Hash &nodeHash, const Signature::PublicKey &userPublicKey) const;
+        void fetchNodeRaw(const Hash &nodeHash, FetchNodeRawCallbackFunc callbackFunc);
+        void fetchNodeAddDataRaw(const Hash &nodeHash, FetchNodeAddDataRawCallbackFunc callbackFunc);
         
         // Username and key pair has to be unique, returns true on success
-        bool storeLocalUser(const std::string &username, const Signature::KeyPair &keyPair, const std::string &password);
+        //bool storeLocalUser(const std::string &username, const Signature::KeyPair &keyPair, const std::string &password);
         
         // Returns public key and private key of encrypted local user.
         // Throws DatabaseStorageNoSuchLocalStorageUser if user does not exist in local storage.
         // Throws DatabaseStorageWrongPassword if password for the stored local user is wrong.
-        Signature::KeyPair decryptLocalEncryptedUser(const std::string &username, const std::string &password);
-        
-        // Get stored local user by public & private key in all nodes they exist.
-        // Creates a new user object and replaces user object in the nodes.
-        // Safe to call multiple times.
-        std::vector<NodeLocalUser> getLocalNodeUsers(const Signature::KeyPair &keyPair);
-        
-        // Returns data in node by its id. Doesn't include 'create' requests. 
-        // Returns nullptr if not data with @dataHash exists.
-        DatabaseStorageObject* getDataById(const Hash &dataHash);
+        //Signature::KeyPair decryptLocalEncryptedUser(const std::string &username, const std::string &password);
         
         // Returns true and node decryption key if node exists and we have the decryption key, 
         // otherwise return false and OwnedMemory with data set to nullptr
@@ -183,37 +166,48 @@ namespace odhtdb
         // Update storage state (remove quarantine objects if they are too old, etc)
         void update();
     private:
-        void loadGroupsFromFile();
-        void loadUsersFromFile();
-        void loadDataFromFile();
-        void loadLocalUsersFromFile();
-        void loadNodeDecryptionKeysFromFile();
-        void loadDecryptedDataFromFile();
+        void init(const boost::filesystem::path &storagePath);
+        void cleanup();
+        
+        void bindCheckError(int sqliteBindResult);
         void loadMetadataFromFile();
-        void loadStorageCreate(sibs::SafeDeserializer &deserializer);
-        void loadStorageAppend(sibs::SafeDeserializer &deserializer);
-        void loadDecryptedStorageCreate(sibs::SafeDeserializer &deserializer);
-        void loadDecryptedStorageAddData(sibs::SafeDeserializer &deserializer);
-        void loadDecryptedStorageAddUser(sibs::SafeDeserializer &deserializer);
-        
-        bool decryptNodeData(const Hash &nodeHash, DatabaseStorageObjectList *databaseCreateObject, const std::shared_ptr<OwnedMemory> decryptionKey);
-        bool decryptNodeAppendedData(const Hash &nodeHash, DatabaseStorageObject *databaseAppendObject, const std::shared_ptr<OwnedMemory> decryptionKey);
+        bool decryptNodeData(const Hash &nodeHash, const std::shared_ptr<OwnedMemory> decryptionKey);
+        bool decryptNodeData(const Hash &nodeHash, const std::shared_ptr<OwnedMemory> decryptionKey, const Signature::PublicKey *creatorPublicKey, const DataView &adminGroupId, u64 timestamp);
+        bool decryptNodeAddData(i64 rowId, const Hash &nodeHash, const Hash &dataHash, u64 timestamp, const Signature::PublicKey *creatorPublicKey, const DataView &encryptedData, const std::shared_ptr<OwnedMemory> decryptionKey);
+        bool decryptNodeAddUser(i64 rowId, const Hash &nodeHash, const Hash &dataHash, u64 timestamp, const Signature::PublicKey *creatorPublicKey, const Signature::PublicKey *userToAddPublicKey, const DataView &groupToAddUserTo, const std::shared_ptr<OwnedMemory> decryptionKey);
+        i64 getNodeRowId(const Hash &nodeHash);
+        i64 getNodeAddDataRowId(const Hash &requestHash);
+        
+        void setNodeAddDataDecrypted(i64 rowId);
+        void setNodeAddDataDecryptedData(i64 rowId, const DataView &decryptedData);
     private:
         Database *database;
-        DatabaseStorageMap storageMap;
-        DatabaseStorageQuarantineMap quarantineStorageMap;
-        MapHash<DatabaseStorageObject*> storedDataHash; // Prevent duplicate data from being added
-        MapHash<Signature::MapPublicKey<User*>*> nodePublicKeyUserDataMap;
-        MapHash<DataViewMap<Group*>*> nodeGroupByIdMap;
-        MapHash<std::shared_ptr<OwnedMemory>> nodeDecryptionKeyMap;
-        std::unordered_map<std::string, LocalUserEncrypted*> nameLocalUsersMap;
-        boost::filesystem::path groupsFilePath;
-        boost::filesystem::path usersFilePath;
-        boost::filesystem::path dataFilePath;
+        sqlite3 *sqliteDb;
+        sqlite3_stmt *insertNodeStmt;
+        sqlite3_stmt *insertUserStmt;
+        sqlite3_stmt *insertGroupStmt;
+        sqlite3_stmt *insertNodeAddDataStmt;
+        sqlite3_stmt *setNodeDecryptionKeyStmt;
+        sqlite3_stmt *getNodeDecryptionKeyStmt;
+        sqlite3_stmt *insertNodeUserGroupAssocStmt;
+        
+        sqlite3_stmt *selectNodeStmt;
+        sqlite3_stmt *selectNodeAddDataByNodeStmt;
+        sqlite3_stmt *selectNodeIdStatement;
+        sqlite3_stmt *selectNodeAddDataIdStatement;
+        sqlite3_stmt *insertNodeRawStmt;
+        sqlite3_stmt *insertNodeAddDataRawStmt;
+        
+        sqlite3_stmt *insertNodeAddDataAdditionalStmt;
+        sqlite3_stmt *insertNodeAddUserDataStmt;
+        
+        sqlite3_stmt *selectNodeAddDataAdditionalStmt;
+        sqlite3_stmt *selectNodeAddUserDataStmt;
+        
+        sqlite3_stmt *setNodeAddDataDecryptedStmt;
+        sqlite3_stmt *setNodeAddDataAdditionalDataStmt;
+        
         boost::filesystem::path metadataFilePath;
-        boost::filesystem::path localUsersFilePath;
-        boost::filesystem::path nodeDecryptionKeysFilePath;
-        boost::filesystem::path decryptedDataFilePath;
         u8 passwordSalt[PASSWORD_SALT_LEN];
         std::pair<std::shared_ptr<dht::crypto::PrivateKey>, std::shared_ptr<dht::crypto::Certificate>> identity;
     };