diff options
| -rw-r--r-- | include/odhtdb/Database.hpp | 2 | ||||
| -rw-r--r-- | include/odhtdb/DatabaseStorage.hpp | 8 | ||||
| -rw-r--r-- | include/odhtdb/FileUtils.hpp | 4 | ||||
| -rw-r--r-- | src/Database.cpp | 28 | ||||
| -rw-r--r-- | src/DatabaseStorage.cpp | 102 | ||||
| -rw-r--r-- | src/FileUtils.cpp | 23 | ||||
| -rw-r--r-- | src/OwnedMemory.cpp | 2 |
7 files changed, 135 insertions, 34 deletions
diff --git a/include/odhtdb/Database.hpp b/include/odhtdb/Database.hpp index 11a6cd4..5ebe9c5 100644 --- a/include/odhtdb/Database.hpp +++ b/include/odhtdb/Database.hpp @@ -187,5 +187,7 @@ namespace odhtdb std::function<void(const DatabaseAddNodeRequest&)> onAddNodeCallbackFunc; std::function<void(const DatabaseAddUserRequest&)> onAddUserCallbackFunc; MapHash<DatabaseSeedInfo> seedInfoMap; + std::thread remoteNodesSaveThread; + bool shuttingDown; }; } diff --git a/include/odhtdb/DatabaseStorage.hpp b/include/odhtdb/DatabaseStorage.hpp index a004619..886412c 100644 --- a/include/odhtdb/DatabaseStorage.hpp +++ b/include/odhtdb/DatabaseStorage.hpp @@ -15,6 +15,7 @@ #include <boost/filesystem/path.hpp> #include <sibs/SafeDeserializer.hpp> #include <opendht/crypto.h> +#include <opendht/dhtrunner.h> #include <functional> class sqlite3; @@ -121,6 +122,9 @@ namespace odhtdb std::pair<bool, std::shared_ptr<OwnedMemory>> getNodeDecryptionKey(const Hash &nodeHash); void setNodeDecryptionKey(const Hash &nodeHash, const DataView &decryptionKey); + const std::vector<dht::NodeExport>& getRemoteNodes() const; + void setRemoteNodes(const std::vector<dht::NodeExport> &remoteNodes); + const dht::crypto::Identity& getIdentity() const; // Update storage state (remove quarantine objects if they are too old, etc) @@ -131,6 +135,7 @@ namespace odhtdb void bindCheckError(int sqliteBindResult); void loadMetadataFromFile(); + void loadRemoteNodesFromFile(); bool decryptNodeData(const Hash &nodeHash, const std::shared_ptr<OwnedMemory> decryptionKey); bool decryptNodeData(const Hash &nodeHash, const std::shared_ptr<OwnedMemory> decryptionKey, const Signature::PublicKey *creatorPublicKey, const DataView &adminGroupId, u64 timestamp); bool decryptNodeAddData(i64 rowId, const Hash &nodeHash, const Hash &dataHash, u64 timestamp, const Signature::PublicKey *creatorPublicKey, const DataView &encryptedData, const std::shared_ptr<OwnedMemory> decryptionKey); @@ -168,7 +173,10 @@ namespace odhtdb sqlite3_stmt *setNodeAddDataAdditionalDataStmt; boost::filesystem::path metadataFilePath; + boost::filesystem::path remoteNodesFilePath; + u8 passwordSalt[PASSWORD_SALT_LEN]; std::pair<std::shared_ptr<dht::crypto::PrivateKey>, std::shared_ptr<dht::crypto::Certificate>> identity; + std::vector<dht::NodeExport> remoteNodes; }; } diff --git a/include/odhtdb/FileUtils.hpp b/include/odhtdb/FileUtils.hpp index 7bfbe3e..88721ed 100644 --- a/include/odhtdb/FileUtils.hpp +++ b/include/odhtdb/FileUtils.hpp @@ -19,4 +19,8 @@ namespace odhtdb // Creates file if it doesn't exist. // Throws FileException on error void fileAppend(const boost::filesystem::path &filepath, const DataView &data); + + // Creates a file if it doesn'te xist. + // Throws FileException on error. + void fileOverwrite(const boost::filesystem::path &filepath, const DataView &data); } diff --git a/src/Database.cpp b/src/Database.cpp index 93dec99..ac5a941 100644 --- a/src/Database.cpp +++ b/src/Database.cpp @@ -92,14 +92,15 @@ namespace odhtdb onCreateNodeCallbackFunc(callbackFuncs.createNodeCallbackFunc), onAddNodeCallbackFunc(callbackFuncs.addNodeCallbackFunc), onAddUserCallbackFunc(callbackFuncs.addUserCallbackFunc), - databaseStorage(this, storageDir) + databaseStorage(this, storageDir), + shuttingDown(false) { node.run(port , { /*.dht_config = */{ /*.node_config = */{ /*.node_id = */{}, /*.network = */0, - /*.is_bootstrap = */false, + /*.is_bootstrap = */true, /*.maintain_storage*/false }, /*.id = */databaseStorage.getIdentity() @@ -110,6 +111,10 @@ namespace odhtdb }); auto portStr = to_string(port); node.bootstrap(bootstrapNodeAddr, portStr.c_str()); + const auto &remoteNodes = databaseStorage.getRemoteNodes(); + if(!remoteNodes.empty()) + node.bootstrap(remoteNodes); + Log::debug("Connecting to bootstrap node (%s) and %u other known nodes that we have connected to previously", bootstrapNodeAddr, remoteNodes.size()); // TODO: Make this work for multiple threads initializing database at same time ++databaseCount; @@ -155,12 +160,31 @@ namespace odhtdb }); ntpThread->detach(); } + + remoteNodesSaveThread = thread([this]() + { + int saveIntervalMs = 5000; // 5 sec + const int sleepDurationMs = 200; + while(!shuttingDown) + { + for(int i = 0; i < saveIntervalMs / sleepDurationMs; ++i) + { + this_thread::sleep_for(chrono::milliseconds(sleepDurationMs)); + if(shuttingDown) + return; + } + databaseStorage.setRemoteNodes(node.exportNodes()); + saveIntervalMs = 30000; // 30 sec + } + }); } Database::~Database() { // TODO: Make this work for multiple threads removing database object at same time --databaseCount; + shuttingDown = true; + remoteNodesSaveThread.join(); node.join(); } diff --git a/src/DatabaseStorage.cpp b/src/DatabaseStorage.cpp index 129ba13..d5053c5 100644 --- a/src/DatabaseStorage.cpp +++ b/src/DatabaseStorage.cpp @@ -73,7 +73,8 @@ namespace odhtdb selectNodeAddUserDataStmt(nullptr), setNodeAddDataDecryptedStmt(nullptr), setNodeAddDataAdditionalDataStmt(nullptr), - metadataFilePath(storagePath / "metadata") + metadataFilePath(storagePath / "metadata"), + remoteNodesFilePath(storagePath / "remote_nodes") { try { @@ -142,48 +143,47 @@ namespace odhtdb sqlite_prepare_checked(sqliteDb, "UPDATE NodeAddData SET decrypted = ? WHERE id = ?", &setNodeAddDataDecryptedStmt); sqlite_prepare_checked(sqliteDb, "UPDATE NodeAddDataAdditional SET data = ? WHERE nodeAddDataId = ?", &setNodeAddDataAdditionalDataStmt); - bool metadataLoaded = false; try { loadMetadataFromFile(); - metadataLoaded = true; } catch(FileException &e) { - if(metadataLoaded) - { - string errMsg = "Failed to load storage, reason: "; - errMsg += e.what(); - throw DatabaseStorageCorrupt(errMsg); - } - else - { - Log::warn("Failed to load storage meta data, reason: %s. Ignoring... (new storage probably)", e.what()); - sibs::SafeSerializer metadataSerializer; - metadataSerializer.add(STORAGE_VERSION); - randombytes_buf(passwordSalt, PASSWORD_SALT_LEN); - metadataSerializer.add(passwordSalt, PASSWORD_SALT_LEN); - - //string passwordSaltStr((const char*)passwordSalt, PASSWORD_SALT_LEN); - identity = dht::crypto::generateIdentity(); - dht::Blob privateKeyData = identity.first->serialize(); - metadataSerializer.add((u16)privateKeyData.size()); - metadataSerializer.add(privateKeyData.data(), privateKeyData.size()); + Log::warn("Failed to load storage meta data, reason: %s. Ignoring... (new storage probably)", e.what()); + sibs::SafeSerializer metadataSerializer; + metadataSerializer.add(STORAGE_VERSION); + randombytes_buf(passwordSalt, PASSWORD_SALT_LEN); + metadataSerializer.add(passwordSalt, PASSWORD_SALT_LEN); + + identity = dht::crypto::generateIdentity(); + dht::Blob privateKeyData = identity.first->serialize(); + metadataSerializer.add((u16)privateKeyData.size()); + metadataSerializer.add(privateKeyData.data(), privateKeyData.size()); - dht::Blob certificateData; - identity.second->pack(certificateData); - metadataSerializer.add((u16)certificateData.size()); - metadataSerializer.add(certificateData.data(), certificateData.size()); - - fileAppend(metadataFilePath, { metadataSerializer.getBuffer().data(), metadataSerializer.getBuffer().size() }); - } + dht::Blob certificateData; + identity.second->pack(certificateData); + metadataSerializer.add((u16)certificateData.size()); + metadataSerializer.add(certificateData.data(), certificateData.size()); + + fileAppend(metadataFilePath, { metadataSerializer.getBuffer().data(), metadataSerializer.getBuffer().size() }); } catch(sibs::DeserializeException &e) { - string errMsg = "Failed to load storage, reason: "; + string errMsg = "Failed to load metadata, reason: "; errMsg += e.what(); throw DatabaseStorageCorrupt(errMsg); } + + try + { + loadRemoteNodesFromFile(); + } + catch(exception &e) + { + string errMsg = "Failed to load nodes, reason: "; + errMsg += e.what(); + Log::warn("%s", errMsg.c_str()); + } } void DatabaseStorage::cleanup() @@ -344,6 +344,23 @@ namespace odhtdb assert(deserializer.empty()); } + void DatabaseStorage::loadRemoteNodesFromFile() + { + OwnedMemory remoteNodesFileContent = fileGetContent(remoteNodesFilePath); + msgpack::unpacker pac; + pac.reserve_buffer(remoteNodesFileContent.size); + memcpy(pac.buffer(), remoteNodesFileContent.data, remoteNodesFileContent.size); + pac.buffer_consumed(remoteNodesFileContent.size); + + msgpack::object_handle oh; + while(pac.next(oh)) + { + auto importedNodes = oh.get().as<vector<dht::NodeExport>>(); + remoteNodes.reserve(remoteNodes.size() + importedNodes.size()); + remoteNodes.insert(remoteNodes.end(), importedNodes.begin(), importedNodes.end()); + } + } + static void sqlite_step_throw_on_failure(sqlite3 *db, sqlite3_stmt *stmt, const char *description) { int rc = sqlite3_step(stmt); @@ -920,6 +937,31 @@ namespace odhtdb decryptNodeData(nodeHash, nodeDecryptionKeyResult.second); } + struct RemoteNodePacker + { + sibs::SafeSerializer serializer; + + RemoteNodePacker& write(const char *data, size_t size) + { + serializer.add((const u8*)data, size); + return *this; + } + }; + + const vector<dht::NodeExport>& DatabaseStorage::getRemoteNodes() const + { + return remoteNodes; + } + + void DatabaseStorage::setRemoteNodes(const std::vector<dht::NodeExport> &remoteNodes) + { + Log::debug("Storing %u remote nodes", remoteNodes.size()); + this->remoteNodes = remoteNodes; + RemoteNodePacker remoteNodePacker; + msgpack::pack(remoteNodePacker, remoteNodes); + fileOverwrite(remoteNodesFilePath, DataView(remoteNodePacker.serializer.getBuffer().data(), remoteNodePacker.serializer.getBuffer().size())); + } + bool DatabaseStorage::decryptNodeData(const Hash &nodeHash, const shared_ptr<OwnedMemory> decryptionKey) { sqlite3_reset(selectNodeStmt); diff --git a/src/FileUtils.cpp b/src/FileUtils.cpp index 6dc10b8..c4fb318 100644 --- a/src/FileUtils.cpp +++ b/src/FileUtils.cpp @@ -55,4 +55,27 @@ namespace odhtdb fwrite(data.data, 1, data.size, file); fclose(file); } + + void fileOverwrite(const boost::filesystem::path &filepath, const DataView &data) + { +#if OS_FAMILY == OS_FAMILY_POSIX + FILE *file = fopen(filepath.string().c_str(), "wb+"); +#else + FILE *file = _wfopen(filepath.wstring().c_str(), L"wb+"); +#endif + if(!file) + { + int error = errno; + string errMsg = "Failed to overwrite file: "; + errMsg += filepath.string(); + errMsg += "; reason: "; + errMsg += strerror(error); + throw FileException(errMsg); + } + + flockfile(file); + setbuf(file, NULL); + fwrite(data.data, 1, data.size, file); + fclose(file); + } } diff --git a/src/OwnedMemory.cpp b/src/OwnedMemory.cpp index 1d53faf..ba18ec1 100644 --- a/src/OwnedMemory.cpp +++ b/src/OwnedMemory.cpp @@ -15,7 +15,5 @@ namespace odhtdb OwnedMemory::~OwnedMemory() { free(data); - data = nullptr; - size = 0; } } |