aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordec05eba <dec05eba@protonmail.com>2018-05-15 06:06:02 +0200
committerdec05eba <dec05eba@protonmail.com>2020-08-18 23:25:46 +0200
commit8a030b5e77031c8b3347170d28758097bca1f15e (patch)
tree780631de9700a37e862b4aa8f946a71cd163c633
parent54f53857279644914e0c2a2aed24060f31ed2575 (diff)
Store known remote nodes and connect to them next time
-rw-r--r--include/odhtdb/Database.hpp2
-rw-r--r--include/odhtdb/DatabaseStorage.hpp8
-rw-r--r--include/odhtdb/FileUtils.hpp4
-rw-r--r--src/Database.cpp28
-rw-r--r--src/DatabaseStorage.cpp102
-rw-r--r--src/FileUtils.cpp23
-rw-r--r--src/OwnedMemory.cpp2
7 files changed, 135 insertions, 34 deletions
diff --git a/include/odhtdb/Database.hpp b/include/odhtdb/Database.hpp
index 11a6cd4..5ebe9c5 100644
--- a/include/odhtdb/Database.hpp
+++ b/include/odhtdb/Database.hpp
@@ -187,5 +187,7 @@ namespace odhtdb
std::function<void(const DatabaseAddNodeRequest&)> onAddNodeCallbackFunc;
std::function<void(const DatabaseAddUserRequest&)> onAddUserCallbackFunc;
MapHash<DatabaseSeedInfo> seedInfoMap;
+ std::thread remoteNodesSaveThread;
+ bool shuttingDown;
};
}
diff --git a/include/odhtdb/DatabaseStorage.hpp b/include/odhtdb/DatabaseStorage.hpp
index a004619..886412c 100644
--- a/include/odhtdb/DatabaseStorage.hpp
+++ b/include/odhtdb/DatabaseStorage.hpp
@@ -15,6 +15,7 @@
#include <boost/filesystem/path.hpp>
#include <sibs/SafeDeserializer.hpp>
#include <opendht/crypto.h>
+#include <opendht/dhtrunner.h>
#include <functional>
class sqlite3;
@@ -121,6 +122,9 @@ namespace odhtdb
std::pair<bool, std::shared_ptr<OwnedMemory>> getNodeDecryptionKey(const Hash &nodeHash);
void setNodeDecryptionKey(const Hash &nodeHash, const DataView &decryptionKey);
+ const std::vector<dht::NodeExport>& getRemoteNodes() const;
+ void setRemoteNodes(const std::vector<dht::NodeExport> &remoteNodes);
+
const dht::crypto::Identity& getIdentity() const;
// Update storage state (remove quarantine objects if they are too old, etc)
@@ -131,6 +135,7 @@ namespace odhtdb
void bindCheckError(int sqliteBindResult);
void loadMetadataFromFile();
+ void loadRemoteNodesFromFile();
bool decryptNodeData(const Hash &nodeHash, const std::shared_ptr<OwnedMemory> decryptionKey);
bool decryptNodeData(const Hash &nodeHash, const std::shared_ptr<OwnedMemory> decryptionKey, const Signature::PublicKey *creatorPublicKey, const DataView &adminGroupId, u64 timestamp);
bool decryptNodeAddData(i64 rowId, const Hash &nodeHash, const Hash &dataHash, u64 timestamp, const Signature::PublicKey *creatorPublicKey, const DataView &encryptedData, const std::shared_ptr<OwnedMemory> decryptionKey);
@@ -168,7 +173,10 @@ namespace odhtdb
sqlite3_stmt *setNodeAddDataAdditionalDataStmt;
boost::filesystem::path metadataFilePath;
+ boost::filesystem::path remoteNodesFilePath;
+
u8 passwordSalt[PASSWORD_SALT_LEN];
std::pair<std::shared_ptr<dht::crypto::PrivateKey>, std::shared_ptr<dht::crypto::Certificate>> identity;
+ std::vector<dht::NodeExport> remoteNodes;
};
}
diff --git a/include/odhtdb/FileUtils.hpp b/include/odhtdb/FileUtils.hpp
index 7bfbe3e..88721ed 100644
--- a/include/odhtdb/FileUtils.hpp
+++ b/include/odhtdb/FileUtils.hpp
@@ -19,4 +19,8 @@ namespace odhtdb
// Creates file if it doesn't exist.
// Throws FileException on error
void fileAppend(const boost::filesystem::path &filepath, const DataView &data);
+
+ // Creates a file if it doesn'te xist.
+ // Throws FileException on error.
+ void fileOverwrite(const boost::filesystem::path &filepath, const DataView &data);
}
diff --git a/src/Database.cpp b/src/Database.cpp
index 93dec99..ac5a941 100644
--- a/src/Database.cpp
+++ b/src/Database.cpp
@@ -92,14 +92,15 @@ namespace odhtdb
onCreateNodeCallbackFunc(callbackFuncs.createNodeCallbackFunc),
onAddNodeCallbackFunc(callbackFuncs.addNodeCallbackFunc),
onAddUserCallbackFunc(callbackFuncs.addUserCallbackFunc),
- databaseStorage(this, storageDir)
+ databaseStorage(this, storageDir),
+ shuttingDown(false)
{
node.run(port , {
/*.dht_config = */{
/*.node_config = */{
/*.node_id = */{},
/*.network = */0,
- /*.is_bootstrap = */false,
+ /*.is_bootstrap = */true,
/*.maintain_storage*/false
},
/*.id = */databaseStorage.getIdentity()
@@ -110,6 +111,10 @@ namespace odhtdb
});
auto portStr = to_string(port);
node.bootstrap(bootstrapNodeAddr, portStr.c_str());
+ const auto &remoteNodes = databaseStorage.getRemoteNodes();
+ if(!remoteNodes.empty())
+ node.bootstrap(remoteNodes);
+ Log::debug("Connecting to bootstrap node (%s) and %u other known nodes that we have connected to previously", bootstrapNodeAddr, remoteNodes.size());
// TODO: Make this work for multiple threads initializing database at same time
++databaseCount;
@@ -155,12 +160,31 @@ namespace odhtdb
});
ntpThread->detach();
}
+
+ remoteNodesSaveThread = thread([this]()
+ {
+ int saveIntervalMs = 5000; // 5 sec
+ const int sleepDurationMs = 200;
+ while(!shuttingDown)
+ {
+ for(int i = 0; i < saveIntervalMs / sleepDurationMs; ++i)
+ {
+ this_thread::sleep_for(chrono::milliseconds(sleepDurationMs));
+ if(shuttingDown)
+ return;
+ }
+ databaseStorage.setRemoteNodes(node.exportNodes());
+ saveIntervalMs = 30000; // 30 sec
+ }
+ });
}
Database::~Database()
{
// TODO: Make this work for multiple threads removing database object at same time
--databaseCount;
+ shuttingDown = true;
+ remoteNodesSaveThread.join();
node.join();
}
diff --git a/src/DatabaseStorage.cpp b/src/DatabaseStorage.cpp
index 129ba13..d5053c5 100644
--- a/src/DatabaseStorage.cpp
+++ b/src/DatabaseStorage.cpp
@@ -73,7 +73,8 @@ namespace odhtdb
selectNodeAddUserDataStmt(nullptr),
setNodeAddDataDecryptedStmt(nullptr),
setNodeAddDataAdditionalDataStmt(nullptr),
- metadataFilePath(storagePath / "metadata")
+ metadataFilePath(storagePath / "metadata"),
+ remoteNodesFilePath(storagePath / "remote_nodes")
{
try
{
@@ -142,48 +143,47 @@ namespace odhtdb
sqlite_prepare_checked(sqliteDb, "UPDATE NodeAddData SET decrypted = ? WHERE id = ?", &setNodeAddDataDecryptedStmt);
sqlite_prepare_checked(sqliteDb, "UPDATE NodeAddDataAdditional SET data = ? WHERE nodeAddDataId = ?", &setNodeAddDataAdditionalDataStmt);
- bool metadataLoaded = false;
try
{
loadMetadataFromFile();
- metadataLoaded = true;
}
catch(FileException &e)
{
- if(metadataLoaded)
- {
- string errMsg = "Failed to load storage, reason: ";
- errMsg += e.what();
- throw DatabaseStorageCorrupt(errMsg);
- }
- else
- {
- Log::warn("Failed to load storage meta data, reason: %s. Ignoring... (new storage probably)", e.what());
- sibs::SafeSerializer metadataSerializer;
- metadataSerializer.add(STORAGE_VERSION);
- randombytes_buf(passwordSalt, PASSWORD_SALT_LEN);
- metadataSerializer.add(passwordSalt, PASSWORD_SALT_LEN);
-
- //string passwordSaltStr((const char*)passwordSalt, PASSWORD_SALT_LEN);
- identity = dht::crypto::generateIdentity();
- dht::Blob privateKeyData = identity.first->serialize();
- metadataSerializer.add((u16)privateKeyData.size());
- metadataSerializer.add(privateKeyData.data(), privateKeyData.size());
+ Log::warn("Failed to load storage meta data, reason: %s. Ignoring... (new storage probably)", e.what());
+ sibs::SafeSerializer metadataSerializer;
+ metadataSerializer.add(STORAGE_VERSION);
+ randombytes_buf(passwordSalt, PASSWORD_SALT_LEN);
+ metadataSerializer.add(passwordSalt, PASSWORD_SALT_LEN);
+
+ identity = dht::crypto::generateIdentity();
+ dht::Blob privateKeyData = identity.first->serialize();
+ metadataSerializer.add((u16)privateKeyData.size());
+ metadataSerializer.add(privateKeyData.data(), privateKeyData.size());
- dht::Blob certificateData;
- identity.second->pack(certificateData);
- metadataSerializer.add((u16)certificateData.size());
- metadataSerializer.add(certificateData.data(), certificateData.size());
-
- fileAppend(metadataFilePath, { metadataSerializer.getBuffer().data(), metadataSerializer.getBuffer().size() });
- }
+ dht::Blob certificateData;
+ identity.second->pack(certificateData);
+ metadataSerializer.add((u16)certificateData.size());
+ metadataSerializer.add(certificateData.data(), certificateData.size());
+
+ fileAppend(metadataFilePath, { metadataSerializer.getBuffer().data(), metadataSerializer.getBuffer().size() });
}
catch(sibs::DeserializeException &e)
{
- string errMsg = "Failed to load storage, reason: ";
+ string errMsg = "Failed to load metadata, reason: ";
errMsg += e.what();
throw DatabaseStorageCorrupt(errMsg);
}
+
+ try
+ {
+ loadRemoteNodesFromFile();
+ }
+ catch(exception &e)
+ {
+ string errMsg = "Failed to load nodes, reason: ";
+ errMsg += e.what();
+ Log::warn("%s", errMsg.c_str());
+ }
}
void DatabaseStorage::cleanup()
@@ -344,6 +344,23 @@ namespace odhtdb
assert(deserializer.empty());
}
+ void DatabaseStorage::loadRemoteNodesFromFile()
+ {
+ OwnedMemory remoteNodesFileContent = fileGetContent(remoteNodesFilePath);
+ msgpack::unpacker pac;
+ pac.reserve_buffer(remoteNodesFileContent.size);
+ memcpy(pac.buffer(), remoteNodesFileContent.data, remoteNodesFileContent.size);
+ pac.buffer_consumed(remoteNodesFileContent.size);
+
+ msgpack::object_handle oh;
+ while(pac.next(oh))
+ {
+ auto importedNodes = oh.get().as<vector<dht::NodeExport>>();
+ remoteNodes.reserve(remoteNodes.size() + importedNodes.size());
+ remoteNodes.insert(remoteNodes.end(), importedNodes.begin(), importedNodes.end());
+ }
+ }
+
static void sqlite_step_throw_on_failure(sqlite3 *db, sqlite3_stmt *stmt, const char *description)
{
int rc = sqlite3_step(stmt);
@@ -920,6 +937,31 @@ namespace odhtdb
decryptNodeData(nodeHash, nodeDecryptionKeyResult.second);
}
+ struct RemoteNodePacker
+ {
+ sibs::SafeSerializer serializer;
+
+ RemoteNodePacker& write(const char *data, size_t size)
+ {
+ serializer.add((const u8*)data, size);
+ return *this;
+ }
+ };
+
+ const vector<dht::NodeExport>& DatabaseStorage::getRemoteNodes() const
+ {
+ return remoteNodes;
+ }
+
+ void DatabaseStorage::setRemoteNodes(const std::vector<dht::NodeExport> &remoteNodes)
+ {
+ Log::debug("Storing %u remote nodes", remoteNodes.size());
+ this->remoteNodes = remoteNodes;
+ RemoteNodePacker remoteNodePacker;
+ msgpack::pack(remoteNodePacker, remoteNodes);
+ fileOverwrite(remoteNodesFilePath, DataView(remoteNodePacker.serializer.getBuffer().data(), remoteNodePacker.serializer.getBuffer().size()));
+ }
+
bool DatabaseStorage::decryptNodeData(const Hash &nodeHash, const shared_ptr<OwnedMemory> decryptionKey)
{
sqlite3_reset(selectNodeStmt);
diff --git a/src/FileUtils.cpp b/src/FileUtils.cpp
index 6dc10b8..c4fb318 100644
--- a/src/FileUtils.cpp
+++ b/src/FileUtils.cpp
@@ -55,4 +55,27 @@ namespace odhtdb
fwrite(data.data, 1, data.size, file);
fclose(file);
}
+
+ void fileOverwrite(const boost::filesystem::path &filepath, const DataView &data)
+ {
+#if OS_FAMILY == OS_FAMILY_POSIX
+ FILE *file = fopen(filepath.string().c_str(), "wb+");
+#else
+ FILE *file = _wfopen(filepath.wstring().c_str(), L"wb+");
+#endif
+ if(!file)
+ {
+ int error = errno;
+ string errMsg = "Failed to overwrite file: ";
+ errMsg += filepath.string();
+ errMsg += "; reason: ";
+ errMsg += strerror(error);
+ throw FileException(errMsg);
+ }
+
+ flockfile(file);
+ setbuf(file, NULL);
+ fwrite(data.data, 1, data.size, file);
+ fclose(file);
+ }
}
diff --git a/src/OwnedMemory.cpp b/src/OwnedMemory.cpp
index 1d53faf..ba18ec1 100644
--- a/src/OwnedMemory.cpp
+++ b/src/OwnedMemory.cpp
@@ -15,7 +15,5 @@ namespace odhtdb
OwnedMemory::~OwnedMemory()
{
free(data);
- data = nullptr;
- size = 0;
}
}