From 414f2cafc6cf2fe141c011b0d63d447a9b983ac3 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Sat, 14 Apr 2018 19:45:15 +0200 Subject: Store database storage to files, also loading --- include/odhtdb/Database.hpp | 27 +++++++++------ include/odhtdb/DatabaseNode.hpp | 9 +++-- include/odhtdb/DatabaseStorage.hpp | 57 +++++++++++++++++++++++++------ include/odhtdb/Encryption.hpp | 8 ++--- include/odhtdb/FileUtils.hpp | 22 ++++++++++++ include/odhtdb/Group.hpp | 6 ++-- include/odhtdb/LocalUser.hpp | 13 ++++++-- include/odhtdb/LocalUserEncrypted.hpp | 51 ++++++++++++++++++++++++++++ include/odhtdb/OwnedMemory.hpp | 22 ++++++++++++ include/odhtdb/PasswordHash.hpp | 9 +++++ include/odhtdb/Permission.hpp | 3 ++ include/odhtdb/RemoteUser.hpp | 2 +- include/odhtdb/Signature.hpp | 7 +++- include/odhtdb/User.hpp | 12 ++++++- include/odhtdb/env.hpp | 63 +++++++++++++++++++++++++++++++++++ include/odhtdb/hex2bin.hpp | 60 +++++++++++++++++++++++++++++++++ 16 files changed, 333 insertions(+), 38 deletions(-) create mode 100644 include/odhtdb/FileUtils.hpp create mode 100644 include/odhtdb/LocalUserEncrypted.hpp create mode 100644 include/odhtdb/OwnedMemory.hpp create mode 100644 include/odhtdb/PasswordHash.hpp create mode 100644 include/odhtdb/env.hpp create mode 100644 include/odhtdb/hex2bin.hpp (limited to 'include') diff --git a/include/odhtdb/Database.hpp b/include/odhtdb/Database.hpp index 926437a..a8833fc 100644 --- a/include/odhtdb/Database.hpp +++ b/include/odhtdb/Database.hpp @@ -10,6 +10,11 @@ #include "Signature.hpp" #include "Permission.hpp" #include "DatabaseNode.hpp" +#include "Encryption.hpp" +#include "OwnedMemory.hpp" +#ifdef DEBUG +#undef DEBUG +#endif #include #include #include @@ -117,31 +122,31 @@ namespace odhtdb class DatabaseCreateResponse { public: - DatabaseCreateResponse(LocalUser *nodeAdminUser, const std::shared_ptr &key, const std::shared_ptr &hash); + DatabaseCreateResponse(LocalUser *nodeAdminUser, const std::shared_ptr &key, const std::shared_ptr &hash); const LocalUser* getNodeAdminUser() const; // Size of encryption key is odhtdb::KEY_BYTE_SIZE (found in Encryption.hpp) - const std::shared_ptr getNodeEncryptionKey() const; + const std::shared_ptr getNodeEncryptionKey() const; const std::shared_ptr getRequestHash() const; private: LocalUser *nodeAdminUser; - std::shared_ptr key; + std::shared_ptr key; std::shared_ptr hash; }; class Database { public: - Database(const char *bootstrapNodeAddr, u16 port, boost::filesystem::path storageDir); + Database(const char *bootstrapNodeAddr, u16 port, const boost::filesystem::path &storageDir); ~Database(); - void seed(const std::shared_ptr hash, const std::shared_ptr encryptionKey); + void seed(const DatabaseNode &nodeToSeed); // Throws DatabaseCreateException on failure. - std::unique_ptr create(const std::string &ownerName, const std::string &nodeName); + std::unique_ptr create(const std::string &ownerName, const std::string &ownerPlainPassword, const std::string &nodeName); // Throws DatabaseAddException on failure void addData(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, DataView dataToAdd); // Throws PermissionDeniedException if user @userToPerformActionWith is not allowed to add user @userToAdd to group @groupToAddUserTo - void addUserToGroup(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, const std::string &userToAddName, const Signature::PublicKey &userToAddPublicKey, Group *groupToAddUserTo); + void addUser(const DatabaseNode &nodeInfo, LocalUser *userToPerformActionWith, const std::string &userToAddName, const Signature::PublicKey &userToAddPublicKey, Group *groupToAddUserTo); void commit(); void setOnCreateNodeCallback(std::function callbackFunc); @@ -155,10 +160,10 @@ namespace odhtdb // Throws CommitAddException on failure void commitStagedAddObject(const std::unique_ptr &stagedObject); ntp::NtpTimestamp getSyncedTimestampUtc() const; - void deserializeCreateRequest(const std::shared_ptr &value, const Hash &hash, const std::shared_ptr encryptionKey); - void deserializeAddRequest(const std::shared_ptr &value, const Hash &requestDataHash, const std::shared_ptr encryptionKey); - bool listenCreateData(std::shared_ptr value, const Hash &hash, const std::shared_ptr encryptionKey); - bool listenAddData(std::shared_ptr value, const Hash &requestDataHash, const std::shared_ptr encryptionKey); + void deserializeCreateRequest(const std::shared_ptr &value, const Hash &hash, const std::shared_ptr encryptionKey); + void deserializeAddRequest(const std::shared_ptr &value, const Hash &requestDataHash, const std::shared_ptr &nodeHash, const std::shared_ptr encryptionKey); + bool listenCreateData(std::shared_ptr value, const Hash &hash, const std::shared_ptr encryptionKey); + bool listenAddData(std::shared_ptr value, const Hash &requestDataHash, const std::shared_ptr nodeHash, const std::shared_ptr encryptionKey); private: dht::DhtRunner node; std::vector> stagedCreateObjects; diff --git a/include/odhtdb/DatabaseNode.hpp b/include/odhtdb/DatabaseNode.hpp index 3ca4be3..620cd40 100644 --- a/include/odhtdb/DatabaseNode.hpp +++ b/include/odhtdb/DatabaseNode.hpp @@ -1,6 +1,7 @@ #pragma once #include "Hash.hpp" +#include "OwnedMemory.hpp" #include namespace odhtdb @@ -8,14 +9,16 @@ namespace odhtdb class DatabaseNode { public: - DatabaseNode(const std::shared_ptr &_encryptionKey, const std::shared_ptr &_nodeHash) : + DatabaseNode() {} + + DatabaseNode(const std::shared_ptr &_encryptionKey, const std::shared_ptr &_nodeHash) : encryptionKey(_encryptionKey), nodeHash(_nodeHash) { } - const std::shared_ptr getNodeEncryptionKey() const + const std::shared_ptr getNodeEncryptionKey() const { return encryptionKey; } @@ -25,7 +28,7 @@ namespace odhtdb return nodeHash; } private: - std::shared_ptr encryptionKey; + std::shared_ptr encryptionKey; std::shared_ptr nodeHash; }; } diff --git a/include/odhtdb/DatabaseStorage.hpp b/include/odhtdb/DatabaseStorage.hpp index ad4f70b..a2789f7 100644 --- a/include/odhtdb/DatabaseStorage.hpp +++ b/include/odhtdb/DatabaseStorage.hpp @@ -5,12 +5,14 @@ #include "DataView.hpp" #include "Signature.hpp" #include "Encryption.hpp" +#include "Group.hpp" #include #include +#include +#include namespace odhtdb { - class Group; class User; struct DatabaseStorageObject @@ -52,12 +54,30 @@ namespace odhtdb DatabaseStorageNotFound(const std::string &errMsg) : std::runtime_error(errMsg) {} }; + class DatabaseStorageCorrupt : public std::runtime_error + { + public: + DatabaseStorageCorrupt(const std::string &errMsg) : std::runtime_error(errMsg) {} + }; + using DatabaseStorageMap = MapHash; using DatabaseStorageQuarantineMap = Signature::MapPublicKey>; + const int PASSWORD_SALT_LEN = 16; + const int HASHED_PASSWORD_LEN = 32; + class DatabaseStorage { public: + struct UserData + { + User *user; + u8 hashedPassword[HASHED_PASSWORD_LEN]; // All bytes are zero if user is not local + }; + + // Throws DatabaseStorageCorrupt if storage is corrupted + DatabaseStorage(const boost::filesystem::path &storagePath); + // Throws DatabaseStorageAlreadyExists if data with hash already exists void createStorage(const Hash &hash, Group *creatorGroup, u64 timestamp, const u8 *data, usize dataSize); @@ -68,28 +88,43 @@ namespace odhtdb // Throws DatabaseStorageAlreadyExists if same data has been added before (hash of @data, in @dataHash) void addToQuarantine(const Hash &dataHash, const Signature::PublicKey &creatorPublicKey, u64 timestamp, const u8 *data, usize dataSize); - void addUser(User *user, const Hash &hash); + // Return false if group with id already exists, otherwise return true + bool addGroup(const Hash &nodeHash, Group *group); + + // Return false if user public key already exists, otherwise return true + bool addUser(const Hash &nodeHash, User *user); // Returns nullptr if no storage with provided hash exists const DatabaseStorageObjectList* getStorage(const Hash &hash) const; - // Returns nullptr if no node with the user exists - const Hash* getNodeByUserPublicKey(const Signature::PublicKey &userPublicKey) const; + // Returns nullptr if a group with id @groupId doesn't exist in node @nodeHash or if no node with id @nodeHash exists + Group* getGroupById(const Hash &nodeHash, uint8_t groupId[GROUP_ID_LENGTH]); - // Returns nullptr if no user with public key exists - const User* getUserByPublicKey(const Signature::PublicKey &userPublicKey) const; + // Returns nullptr if a user with public key @publicKey doesn't exist in node @nodeHash or if no node with id @nodeHash exists + User* getUserByPublicKey(const Hash &nodeHash, Signature::PublicKey &userPublicKey); - // Returns nullptr if a group with id @groupId doesn't exist - Group* getGroupById(uint8_t groupId[16]); + // Return users in node, or nullptr if no node with id @nodeHash exists + const Signature::MapPublicKey* getUsersData(const Hash &nodeHash) const; // Update storage state (remove quarantine objects if they are too old, etc) void update(); + private: + void loadGroupsFromFile(); + void loadUsersFromFile(); + void loadDataFromFile(); + void loadMetadataFromFile(); + void loadStorageCreate(sibs::SafeDeserializer &deserializer); + void loadStorageAppend(sibs::SafeDeserializer &deserializer); private: DatabaseStorageMap storageMap; DatabaseStorageQuarantineMap quarantineStorageMap; SetHash storedDataHash; // Prevent duplicate data from being added - Signature::MapPublicKey userPublicKeyNodeMap; - Signature::MapPublicKey publicKeyUserMap; - DataViewMap groupByIdMap; + MapHash*> nodePublicKeyUserDataMap; + MapHash*> nodeGroupByIdMap; + boost::filesystem::path groupsFilePath; + boost::filesystem::path usersFilePath; + boost::filesystem::path dataFilePath; + boost::filesystem::path metadataFilePath; + u8 passwordSalt[PASSWORD_SALT_LEN]; }; } diff --git a/include/odhtdb/Encryption.hpp b/include/odhtdb/Encryption.hpp index 4697b35..5271cbd 100644 --- a/include/odhtdb/Encryption.hpp +++ b/include/odhtdb/Encryption.hpp @@ -11,8 +11,8 @@ namespace odhtdb { - const int NONCE_BYTE_SIZE = 24; - const int KEY_BYTE_SIZE = 32; + const int ENCRYPTION_NONCE_BYTE_SIZE = 24; + const int ENCRYPTION_KEY_BYTE_SIZE = 32; class EncryptionException : public std::runtime_error { @@ -38,8 +38,8 @@ namespace odhtdb DataView getNonce() const; DataView getCipherText() const; private: - unsigned char key[KEY_BYTE_SIZE]; - unsigned char nonce[NONCE_BYTE_SIZE]; + unsigned char key[ENCRYPTION_KEY_BYTE_SIZE]; + unsigned char nonce[ENCRYPTION_NONCE_BYTE_SIZE]; unsigned char *cipherText; unsigned long long cipherTextLength; }; diff --git a/include/odhtdb/FileUtils.hpp b/include/odhtdb/FileUtils.hpp new file mode 100644 index 0000000..7bfbe3e --- /dev/null +++ b/include/odhtdb/FileUtils.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include "OwnedMemory.hpp" +#include "DataView.hpp" + +namespace odhtdb +{ + class FileException : public std::runtime_error + { + public: + FileException(const std::string &errMsg) : std::runtime_error(errMsg) {} + }; + + // Throws FileException on error + OwnedMemory fileGetContent(const boost::filesystem::path &filepath); + + // Creates file if it doesn't exist. + // Throws FileException on error + void fileAppend(const boost::filesystem::path &filepath, const DataView &data); +} diff --git a/include/odhtdb/Group.hpp b/include/odhtdb/Group.hpp index 315961d..890b2fc 100644 --- a/include/odhtdb/Group.hpp +++ b/include/odhtdb/Group.hpp @@ -21,13 +21,13 @@ namespace odhtdb } }; - + const int GROUP_ID_LENGTH = 16; class Group { friend class User; public: - Group(const std::string &name, uint8_t id[16], const Permission &permission); + Group(const std::string &name, uint8_t id[GROUP_ID_LENGTH], const Permission &permission); ~Group(); const std::string& getName() const; @@ -38,7 +38,7 @@ namespace odhtdb void addUser(const User *user); private: std::string name; - uint8_t id[16]; + uint8_t id[GROUP_ID_LENGTH]; Permission permission; std::vector users; }; diff --git a/include/odhtdb/LocalUser.hpp b/include/odhtdb/LocalUser.hpp index d60cb38..0312a38 100644 --- a/include/odhtdb/LocalUser.hpp +++ b/include/odhtdb/LocalUser.hpp @@ -1,15 +1,16 @@ #pragma once #include "User.hpp" +#include "types.hpp" namespace odhtdb { class LocalUser : public User { public: - static LocalUser* create(const Signature::KeyPair &keyPair, const std::string &name, Group *group) + static LocalUser* create(const Signature::KeyPair &keyPair, const std::string &name, Group *group, const std::string &plainPassword) { - return new LocalUser(keyPair, name, group); + return new LocalUser(keyPair, name, group, plainPassword); } const Signature::PublicKey& getPublicKey() const override @@ -21,9 +22,15 @@ namespace odhtdb { return keyPair.getPrivateKey(); } + + const std::string& getPlainPassword() const + { + return plainPassword; + } private: - LocalUser(const Signature::KeyPair &_keyPair, const std::string &name, Group *group) : User(name, group), keyPair(_keyPair) {} + LocalUser(const Signature::KeyPair &_keyPair, const std::string &name, Group *group, const std::string &plainPassword); private: Signature::KeyPair keyPair; + std::string plainPassword; }; } diff --git a/include/odhtdb/LocalUserEncrypted.hpp b/include/odhtdb/LocalUserEncrypted.hpp new file mode 100644 index 0000000..c250d13 --- /dev/null +++ b/include/odhtdb/LocalUserEncrypted.hpp @@ -0,0 +1,51 @@ +#pragma once + +#include "User.hpp" +#include "types.hpp" +#include "Encryption.hpp" + +namespace odhtdb +{ + struct EncryptedPrivateKey + { + u8 nonce[ENCRYPTION_NONCE_BYTE_SIZE]; + u8 encryptedPrivateKey[16 + PRIVATE_KEY_NUM_BYTES]; + + EncryptedPrivateKey(); + EncryptedPrivateKey(const EncryptedPrivateKey &other); + + // Throws DecryptionException if password (or salt) is wrong + Signature::PrivateKey decrypt(const DataView &plainPassword, const DataView &salt) const; + }; + + // Local user with encrypted private key + class LocalUserEncrypted : public User + { + public: + static LocalUserEncrypted* create(const Signature::PublicKey &publicKey, const EncryptedPrivateKey &encryptedPrivateKey, const std::string &name, Group *group) + { + return new LocalUserEncrypted(publicKey, encryptedPrivateKey, name, group); + } + + const Signature::PublicKey& getPublicKey() const override + { + return publicKey; + } + + const EncryptedPrivateKey& getPrivateKey() const + { + return encryptedPrivateKey; + } + private: + LocalUserEncrypted(const Signature::PublicKey &_publicKey, const EncryptedPrivateKey &_encryptedPrivateKey, const std::string &name, Group *group) : + User(User::Type::LOCAL_ENCRYPTED, name, group), + publicKey(_publicKey), + encryptedPrivateKey(_encryptedPrivateKey) + { + + } + private: + Signature::PublicKey publicKey; + EncryptedPrivateKey encryptedPrivateKey; + }; +} diff --git a/include/odhtdb/OwnedMemory.hpp b/include/odhtdb/OwnedMemory.hpp new file mode 100644 index 0000000..5dcdf25 --- /dev/null +++ b/include/odhtdb/OwnedMemory.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include "types.hpp" + +namespace odhtdb +{ + class OwnedMemory + { + public: + OwnedMemory() : data(nullptr), size(0) {} + OwnedMemory(void *_data, usize _size) : data(_data), size(_size) {} + OwnedMemory(OwnedMemory &&other); + ~OwnedMemory(); + + // Do not allow copy of this struct, forcing move when returning a OwnedMemory in a function + OwnedMemory(OwnedMemory&) = delete; + OwnedMemory& operator = (OwnedMemory&) = delete; + + void *data; + usize size; + }; +} diff --git a/include/odhtdb/PasswordHash.hpp b/include/odhtdb/PasswordHash.hpp new file mode 100644 index 0000000..08b1857 --- /dev/null +++ b/include/odhtdb/PasswordHash.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include "OwnedMemory.hpp" +#include "DataView.hpp" + +namespace odhtdb +{ + OwnedMemory hashPassword(const DataView &plainPassword, const DataView &salt); +} diff --git a/include/odhtdb/Permission.hpp b/include/odhtdb/Permission.hpp index 1ae2642..1e7bb0c 100644 --- a/include/odhtdb/Permission.hpp +++ b/include/odhtdb/Permission.hpp @@ -30,6 +30,9 @@ namespace odhtdb class Permission { public: + // @permissionLevel is hierarchical access right. A group can only modify a group that has higher @permissionLevel value + Permission(u8 permissionLevel, u32 permissionFlags); + // @permissionLevel is hierarchical access right. A group can only modify a group that has higher @permissionLevel value Permission(u8 permissionLevel, std::initializer_list permissions); diff --git a/include/odhtdb/RemoteUser.hpp b/include/odhtdb/RemoteUser.hpp index 181a4c4..9dc8f96 100644 --- a/include/odhtdb/RemoteUser.hpp +++ b/include/odhtdb/RemoteUser.hpp @@ -17,7 +17,7 @@ namespace odhtdb return publicKey; } private: - RemoteUser(const Signature::PublicKey &_publicKey, const std::string &name, Group *group) : User(name, group), publicKey(_publicKey){} + RemoteUser(const Signature::PublicKey &_publicKey, const std::string &name, Group *group) : User(User::Type::REMOTE, name, group), publicKey(_publicKey){} private: Signature::PublicKey publicKey; }; diff --git a/include/odhtdb/Signature.hpp b/include/odhtdb/Signature.hpp index 62c41c3..db434ac 100644 --- a/include/odhtdb/Signature.hpp +++ b/include/odhtdb/Signature.hpp @@ -93,6 +93,8 @@ namespace odhtdb { friend class KeyPair; public: + static PrivateKey ZERO; + // Throws InvalidSignatureKeySize if size is not PRIVATE_KEY_NUM_BYTES PrivateKey(const char *data, size_t size); PrivateKey(const PrivateKey &other); @@ -113,9 +115,12 @@ namespace odhtdb class KeyPair { public: - // Throws SignatureGenerationException if generation of private/public key pair fails (should never happen) + // Generate a new key pair, Throws SignatureGenerationException if generation of private/public key pair fails (should never happen) KeyPair(); + // Create a key pair from existing public and private key + KeyPair(const PublicKey &publicKey, const PrivateKey &privateKey); + const PublicKey& getPublicKey() const { return publicKey; } const PrivateKey& getPrivateKey() const { return privateKey; } private: diff --git a/include/odhtdb/User.hpp b/include/odhtdb/User.hpp index fb37876..d6e551a 100644 --- a/include/odhtdb/User.hpp +++ b/include/odhtdb/User.hpp @@ -1,6 +1,7 @@ #pragma once #include "Signature.hpp" +#include "types.hpp" #include #include #include @@ -22,16 +23,25 @@ namespace odhtdb class User { public: + enum class Type : u8 + { + LOCAL, + LOCAL_ENCRYPTED, + REMOTE + }; + virtual ~User(){} void addToGroup(Group *group); + Type getType() const { return type; } const std::string& getName() const { return name; } const std::vector& getGroups() const { return groups; } virtual const Signature::PublicKey& getPublicKey() const = 0; protected: - User(const std::string &name, Group *group); + User(Type type, const std::string &name, Group *group); private: + Type type; std::string name; std::vector groups; }; diff --git a/include/odhtdb/env.hpp b/include/odhtdb/env.hpp new file mode 100644 index 0000000..bafc750 --- /dev/null +++ b/include/odhtdb/env.hpp @@ -0,0 +1,63 @@ +#pragma once + +#define OS_FAMILY_WINDOWS 0 +#define OS_FAMILY_POSIX 1 + +#define OS_TYPE_WINDOWS 0 +#define OS_TYPE_LINUX 1 + +#if defined(_WIN32) || defined(_WIN64) + #if defined(_WIN64) + #define SYS_ENV_64BIT + #else + #define SYS_ENV_32BIT + #endif + #define OS_FAMILY OS_FAMILY_WINDOWS + #define OS_TYPE OS_TYPE_WINDOWS + + #ifndef UNICODE + #define UNICODE + #endif + + #ifndef _UNICODE + #define _UNICODE + #endif + + #ifndef WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN + #endif + + #include +#endif + +#if defined(__linux__) || defined(__unix__) || defined(__APPLE__) || defined(_POSIX_VERSION) + #define OS_FAMILY OS_FAMILY_POSIX +#endif + +#ifdef __linux__ + #define OS_TYPE OS_TYPE_LINUX +#endif + +#if defined(__GNUC__) + #if defined(__x86_64__) || defined(__pc64__) + #define SYS_ENV_64BIT + #else + #define SYS_ENV_32BIT + #endif +#endif + +#if !defined(SYS_ENV_32BIT) && !defined(SYS_ENV_64BIT) + #error "System is not detected as either 32-bit or 64-bit" +#endif + +#if !defined(OS_FAMILY) + #error "System not supported. Only Windows and Posix systems supported right now" +#endif + +#if !defined(OS_TYPE) + #error "System not supported. Only Windows and linux systems supported right now" +#endif + +#if !defined(DEBUG) && !defined(NDEBUG) +#define DEBUG +#endif diff --git a/include/odhtdb/hex2bin.hpp b/include/odhtdb/hex2bin.hpp new file mode 100644 index 0000000..96a2229 --- /dev/null +++ b/include/odhtdb/hex2bin.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +namespace odhtdb +{ + class InvalidHexStringFormat : public std::runtime_error + { + public: + InvalidHexStringFormat(const std::string &errMsg) : std::runtime_error(errMsg) {} + }; + + // Returns -1 if c is not a hex string (0-9a-fA-F) + static int __hexchar_to_num(int c) + { + if(c >= '0' && c <= '9') + return c - '0'; + else if(c >= 'a' && c <= 'f') + return 10 + (c - 'a'); + else if(c >= 'A' && c <= 'F') + return 10 + (c - 'A'); + else + return -1; + } + + // Throws InvalidHexStringFormat on invalid input + static std::string hex2bin(const char *data, size_t dataSize) + { + if(dataSize % 2 != 0) + throw InvalidHexStringFormat("Input string size is not of an even size"); + + std::string result; + result.resize(dataSize / 2); + + for(int i = 0; i < dataSize; i += 2) + { + int v = __hexchar_to_num(data[i]); + int v2 = __hexchar_to_num(data[i + 1]); + + if(v == -1) + { + std::string errMsg = "Invalid hex character: "; + errMsg += data[i]; + throw InvalidHexStringFormat(errMsg); + } + + if(v2 == -1) + { + std::string errMsg = "Invalid hex character: "; + errMsg += data[i + 1]; + throw InvalidHexStringFormat(errMsg); + } + + result[i / 2] = (v << 4) | v2; + } + + return result; + } +} -- cgit v1.2.3