From f3b7b7d34b3bf2b1be18914577c96b66dead379a Mon Sep 17 00:00:00 2001 From: dec05eba Date: Tue, 12 Dec 2017 17:33:03 +0100 Subject: Download and extract missing dependencies from github Using libcurl and libarchive --- CMakeLists.txt | 18 ++++- backend/ninja/Ninja.cpp | 24 ++++++- include/Archive.hpp | 15 +++++ include/GlobalLib.hpp | 8 +++ include/Result.hpp | 26 ++++++-- include/curl.hpp | 15 +++++ include/env.hpp | 4 ++ include/utils.hpp | 1 - src/Archive.cpp | 171 ++++++++++++++++++++++++++++++++++++++++++++++++ src/GlobalLib.cpp | 36 +++++++++- src/PkgConfig.cpp | 3 + src/curl.cpp | 87 ++++++++++++++++++++++++ src/main.cpp | 3 - 13 files changed, 394 insertions(+), 17 deletions(-) create mode 100644 include/Archive.hpp create mode 100644 include/curl.hpp create mode 100644 src/Archive.cpp create mode 100644 src/curl.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 96c4185..a636daa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,12 +1,24 @@ -cmake_minimum_required(VERSION 3.8) +cmake_minimum_required(VERSION 3.0.2) project(sibs) set(CMAKE_CXX_STANDARD 11) set(SOURCE_FILES + external/xxhash.c + backend/ninja/Ninja.cpp src/main.cpp src/FileUtil.cpp src/Conf.cpp - external/xxhash.c backend/ninja/Ninja.cpp src/PkgConfig.cpp src/Exec.cpp src/GlobalLib.cpp) + src/PkgConfig.cpp + src/Exec.cpp + src/GlobalLib.cpp + src/curl.cpp + src/Archive.cpp) -add_executable(sibs ${SOURCE_FILES}) \ No newline at end of file +find_package(CURL REQUIRED) +find_package(LibArchive REQUIRED) + +add_executable(sibs ${SOURCE_FILES}) + +include_directories(${CURL_INCLUDE_DIR} ${LibArchive_INCLUDE_DIR}) +target_link_libraries(sibs ${CURL_LIBRARIES} ${LibArchive_LIBRARIES}) \ No newline at end of file diff --git a/backend/ninja/Ninja.cpp b/backend/ninja/Ninja.cpp index 1185366..72c1e80 100644 --- a/backend/ninja/Ninja.cpp +++ b/backend/ninja/Ninja.cpp @@ -103,7 +103,29 @@ namespace backend printf("%s, trying global lib\n", pkgConfigDependencyValidation.getErrMsg().c_str()); Result globalLibLinkerFlagsResult = GlobalLib::getStaticLibsLinkerFlags(globalLibDir, dependency.name, dependency.version, linkerFlagCallbackFunc); if(globalLibLinkerFlagsResult.isErr()) - return globalLibLinkerFlagsResult; + { + if(globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_NOT_FOUND || globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_VERSION_NO_MATCH) + { + printf("Dependency not found in global lib, trying to download from github\n"); + // TODO: Download several dependencies at the same time by adding them to a list + // and then iterate them and download them all using several threads. + + // TODO: If return error is invalid url, then the message should be converted to + // invalid package name/version. A check should be done if it is the name or version + // that is invalid. + Result downloadDependencyResult = GlobalLib::downloadDependency(dependency); + if(downloadDependencyResult.isErr()) + return Result::Err(downloadDependencyResult.getErrMsg()); + + globalLibLinkerFlagsResult = GlobalLib::getStaticLibsLinkerFlags(globalLibDir, dependency.name, dependency.version, linkerFlagCallbackFunc); + if(globalLibLinkerFlagsResult.isErr()) + return globalLibLinkerFlagsResult; + } + else + { + return globalLibLinkerFlagsResult; + } + } globalLibLinkerFlags += " "; globalLibLinkerFlags += globalLibLinkerFlagsResult.unwrap(); diff --git a/include/Archive.hpp b/include/Archive.hpp new file mode 100644 index 0000000..6d6a55d --- /dev/null +++ b/include/Archive.hpp @@ -0,0 +1,15 @@ +#ifndef SIBS_ZLIB_HPP +#define SIBS_ZLIB_HPP + +#include "Result.hpp" + +namespace sibs +{ + class Archive + { + public: + static Result extract(const char *source, const char *destination); + }; +} + +#endif //SIBS_ZLIB_HPP diff --git a/include/GlobalLib.hpp b/include/GlobalLib.hpp index d5e21d1..ca542f9 100644 --- a/include/GlobalLib.hpp +++ b/include/GlobalLib.hpp @@ -3,14 +3,22 @@ #include "Result.hpp" #include "Linker.hpp" +#include "Dependency.hpp" namespace sibs { class GlobalLib { public: + enum DependencyError + { + DEPENDENCY_NOT_FOUND = 10, + DEPENDENCY_VERSION_NO_MATCH = 20 + }; + static Result validatePackageExists(const std::string &globalLibRootDir, const std::string &name); static Result getStaticLibsLinkerFlags(const std::string &globalLibRootDir, const std::string &name, const std::string &version, LinkerFlagCallbackFunc linkerFlagCallbackFunc); + static Result downloadDependency(const Dependency &dependency); }; } diff --git a/include/Result.hpp b/include/Result.hpp index f755b15..eb0aa01 100644 --- a/include/Result.hpp +++ b/include/Result.hpp @@ -13,20 +13,29 @@ namespace sibs static Result Ok(const T &value) { Result result(value); - result.error = false; + result.errorCode = 0; return result; } - static Result Err(const std::string &errMsg) + template + static Result Err(const Result &other) + { + Result result; + result.errMsg = other.getErrMsg(); + result.errorCode = other.getErrorCode(); + return result; + } + + static Result Err(const std::string &errMsg, int errorCode = 1) { Result result; result.errMsg = errMsg; - result.error = true; + result.errorCode = errorCode; return result; } - bool isOk() const { return !error; } - bool isErr() const { return error; } + bool isOk() const { return !errorCode; } + bool isErr() const { return errorCode; } T& unwrap() { @@ -39,12 +48,17 @@ namespace sibs assert(isErr()); return errMsg; } + + int getErrorCode() const + { + return errorCode; + } private: Result(const T &_value = T()) : value(_value) {} private: T value; std::string errMsg; - bool error; + int errorCode; }; } diff --git a/include/curl.hpp b/include/curl.hpp new file mode 100644 index 0000000..49dfe12 --- /dev/null +++ b/include/curl.hpp @@ -0,0 +1,15 @@ +#ifndef SIBS_CURL_HPP +#define SIBS_CURL_HPP + +#include "Result.hpp" + +namespace sibs +{ + class curl + { + public: + static Result downloadFile(const char *url, const char *filepath); + }; +} + +#endif //SIBS_CURL_HPP diff --git a/include/env.hpp b/include/env.hpp index 325db6e..f5b1213 100644 --- a/include/env.hpp +++ b/include/env.hpp @@ -33,4 +33,8 @@ #error "System not support. Only Windows and Posix systems support" #endif +#if !defined(DEBUG) && !defined(NDEBUG) +#define DEBUG +#endif + #endif // SIBS_ENV_HPP diff --git a/include/utils.hpp b/include/utils.hpp index 9ad9d97..926749d 100755 --- a/include/utils.hpp +++ b/include/utils.hpp @@ -4,7 +4,6 @@ // Disable copying for a class or struct #define DISABLE_COPY(ClassName) \ ClassName(ClassName&) = delete; \ - ClassName(ClassName&&) = delete; \ ClassName& operator = (ClassName&) = delete; #endif // SIBS_UTILS_HPP diff --git a/src/Archive.cpp b/src/Archive.cpp new file mode 100644 index 0000000..08ab42b --- /dev/null +++ b/src/Archive.cpp @@ -0,0 +1,171 @@ +#include "../include/Archive.hpp" +#include "../include/utils.hpp" +#include +#include +#include + +using namespace std; + +class FileHandler +{ + DISABLE_COPY(FileHandler) +public: + FileHandler(FILE *_file) : file(_file) + { + + } + + FileHandler(FileHandler &&other) + { + file = other.file; + other.file = nullptr; + } + + ~FileHandler() + { + fclose(file); + } + + FILE *file; +}; + +// libarchive code is modified version of: https://github.com/libarchive/libarchive/wiki/Examples#A_Complete_Extractor +namespace sibs +{ + static int copy_data(struct archive *ar, struct archive *aw) + { + int r; + const void *buff; + size_t size; + la_int64_t offset; + + while(true) + { + r = archive_read_data_block(ar, &buff, &size, &offset); + if (r == ARCHIVE_EOF) + return ARCHIVE_OK; + else if (r < ARCHIVE_OK) + return r; + + r = archive_write_data_block(aw, buff, size, offset); + if (r < ARCHIVE_OK) + return r; + } + } + + Result Archive::extract(const char *source, const char *destination) + { + struct archive *a; + struct archive *ext; + struct archive_entry *entry; + int flags; + int r; + + /* Select which attributes we want to restore. */ + flags = ARCHIVE_EXTRACT_TIME; + flags |= ARCHIVE_EXTRACT_PERM; + flags |= ARCHIVE_EXTRACT_ACL; + flags |= ARCHIVE_EXTRACT_FFLAGS; + + string rootName; + + a = archive_read_new(); + archive_read_support_format_all(a); + archive_read_support_compression_all(a); + ext = archive_write_disk_new(); + archive_write_disk_set_options(ext, flags); + archive_write_disk_set_standard_lookup(ext); + + + if ((r = archive_read_open_filename(a, source, 10240))) + { + string errMsg = "Failed to extract archive: "; + errMsg += source; + return Result::Err(errMsg); + } + + while(true) + { + r = archive_read_next_header(a, &entry); + if (r == ARCHIVE_EOF) + break; + else if (r < ARCHIVE_OK) + { + string errMsg = "Failed to extract archive: "; + errMsg += source; + errMsg += "; reason: "; + errMsg += archive_error_string(a); + return Result::Err(errMsg); + } + else if (r < ARCHIVE_WARN) + { + string errMsg = "Failed to extract archive: "; + errMsg += source; + errMsg += "; reason: "; + errMsg += archive_error_string(a); + return Result::Err(errMsg); + } + + const char* currentFile = archive_entry_pathname(entry); + if(rootName.empty()) + rootName = currentFile; + + std::string fullOutputPath = destination; + fullOutputPath += (currentFile + (rootName.empty() ? 0 : rootName.size() - 1)); + archive_entry_set_pathname(entry, fullOutputPath.c_str()); + + r = archive_write_header(ext, entry); + if (r < ARCHIVE_OK) + { + string errMsg = "Failed to extract archive: "; + errMsg += source; + errMsg += "; reason: "; + errMsg += archive_error_string(ext); + return Result::Err(errMsg); + } + else if (archive_entry_size(entry) > 0) { + r = copy_data(a, ext); + if (r < ARCHIVE_OK) + { + string errMsg = "Failed to extract archive: "; + errMsg += source; + errMsg += "; reason: "; + errMsg += archive_error_string(ext); + return Result::Err(errMsg); + } + else if (r < ARCHIVE_WARN) + { + string errMsg = "Failed to extract archive: "; + errMsg += source; + errMsg += "; reason: "; + errMsg += archive_error_string(ext); + return Result::Err(errMsg); + } + } + + r = archive_write_finish_entry(ext); + if (r < ARCHIVE_OK) + { + string errMsg = "Failed to extract archive: "; + errMsg += source; + errMsg += "; reason: "; + errMsg += archive_error_string(ext); + return Result::Err(errMsg); + } + else if (r < ARCHIVE_WARN) + { + string errMsg = "Failed to extract archive: "; + errMsg += source; + errMsg += "; reason: "; + errMsg += archive_error_string(ext); + return Result::Err(errMsg); + } + } + + archive_read_close(a); + archive_read_free(a); + archive_write_close(ext); + archive_write_free(ext); + return Result::Ok(true); + } +} \ No newline at end of file diff --git a/src/GlobalLib.cpp b/src/GlobalLib.cpp index 7503a7a..ce9426d 100644 --- a/src/GlobalLib.cpp +++ b/src/GlobalLib.cpp @@ -2,6 +2,8 @@ #include "../include/FileUtil.hpp" #include "../backend/ninja/Ninja.hpp" #include "../include/Conf.hpp" +#include "../include/curl.hpp" +#include "../include/Archive.hpp" using namespace std; @@ -18,7 +20,7 @@ namespace sibs { string errMsg = "Global lib dependency not found: "; errMsg += name; - return Result::Err(errMsg); + return Result::Err(errMsg, DependencyError::DEPENDENCY_NOT_FOUND); } case FileType::REGULAR: { @@ -57,7 +59,7 @@ namespace sibs { Result packageExistsResult = validatePackageExists(globalLibRootDir, name); if(packageExistsResult.isErr()) - return Result::Err(packageExistsResult.getErrMsg()); + return Result::Err(packageExistsResult); string packageDir = globalLibRootDir + "/"; packageDir += name; @@ -76,7 +78,7 @@ namespace sibs }); if(foundVersion.empty()) - return Result::Err("Global lib dependency found, but version doesn't match dependency version"); + return Result::Err("Global lib dependency found, but version doesn't match dependency version", DependencyError::DEPENDENCY_VERSION_NO_MATCH); packageDir += "/"; packageDir += version; @@ -156,4 +158,32 @@ namespace sibs return Result::Ok(staticLibPath); } } + + Result GlobalLib::downloadDependency(const Dependency &dependency) + { + string url = "https://github.com/DEC05EBA/"; + url += dependency.name; + url += "/archive/"; + url += dependency.version; + url += ".tar.gz"; + + // TODO: Create library path if it doesn't exist + string libPath = getHomeDir(); + libPath += "/.sibs/lib/"; + libPath += dependency.name; + libPath += "/"; + libPath += dependency.version; + + // TODO: Create archive directory if it doesn't exist + string libArchivedFilePath = getHomeDir(); + libArchivedFilePath += "/.sibs/archive/"; + libArchivedFilePath += dependency.name; + libArchivedFilePath += "/"; + libArchivedFilePath += dependency.version; + Result downloadResult = curl::downloadFile(url.c_str(), libArchivedFilePath.c_str()); + if(downloadResult.isErr()) + return downloadResult; + + return Archive::extract(libArchivedFilePath.c_str(), libPath.c_str()); + } } \ No newline at end of file diff --git a/src/PkgConfig.cpp b/src/PkgConfig.cpp index 11e1cf0..3307ff1 100644 --- a/src/PkgConfig.cpp +++ b/src/PkgConfig.cpp @@ -3,6 +3,9 @@ using namespace std; +// TODO: Do not use pkg-config program. The same functionality can easily be achieved +// by reading files in /usr/bin/pkgconfig +// Or is there no downside to calling pkg-config program? namespace sibs { string trimRight(const string &input) diff --git a/src/curl.cpp b/src/curl.cpp new file mode 100644 index 0000000..f39cab8 --- /dev/null +++ b/src/curl.cpp @@ -0,0 +1,87 @@ +#include "../include/curl.hpp" +#include "../include/env.hpp" +#include +#include + +using namespace std; + +#ifdef DEBUG +#define CURL_DEBUG +#endif + +#undef CURL_DEBUG + +class CurlSession +{ +public: + CurlSession() + { + curl_global_init(CURL_GLOBAL_ALL); + } +}; + +static CurlSession curlSession; + +namespace sibs +{ + // TODO: Instead of writing to file, reading from file and extracting it; + // we can extract to file directly by putting libarchive code here + size_t writeToFile(void *ptr, size_t size, size_t nmemb, void *stream) + { + return fwrite(ptr, size, nmemb, (FILE*)stream); + } + + Result curl::downloadFile(const char *url, const char *filepath) + { + CURL *curl_handle = curl_easy_init(); + curl_easy_setopt(curl_handle, CURLOPT_URL, url); +#ifdef CURL_DEBUG + long verbose = 1L; + long noProgressMeter = 0L; +#else + long verbose = 0L; + long noProgressMeter = 1L; +#endif + curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, verbose); + curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, noProgressMeter); + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writeToFile); + curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, true); + + FILE *file = fopen(filepath, "wb"); + if(!file) + { + int error = errno; + curl_easy_cleanup(curl_handle); + + string errMsg = "Failed to open file for writing: "; + errMsg += filepath; + if(error != 0) + { + errMsg += "; Reason: "; + errMsg += strerror(error); + return Result::Err(errMsg); + } + else + { + return Result::Err(errMsg); + } + } + + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, file); + printf("Downloading from url: %s\n", url); + CURLcode curlResponse = curl_easy_perform(curl_handle); + curl_easy_cleanup(curl_handle); + fclose(file); + + if(curlResponse != CURLE_OK) + { + string errMsg = "Failed to download file from url: "; + errMsg += url; + errMsg += "\nReason: "; + errMsg += curl_easy_strerror(curlResponse); + return Result::Err(errMsg); + } + + return Result::Ok(true); + } +} \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index b1982bd..ce379c2 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,10 +1,7 @@ #include #include "../include/FileUtil.hpp" #include "../include/Conf.hpp" -#include "../include/Dependency.hpp" #include "../backend/ninja/Ninja.hpp" -#include -#include using namespace std; using namespace sibs; -- cgit v1.2.3