Download and extract missing dependencies from github

Using libcurl and libarchive

13 files changed, 394 insertions, 17 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 96c4185..a636daa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,12 +1,24 @@
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.0.2)
 project(sibs)
 
 set(CMAKE_CXX_STANDARD 11)
 
 set(SOURCE_FILES
+        external/xxhash.c
+        backend/ninja/Ninja.cpp
         src/main.cpp
         src/FileUtil.cpp
         src/Conf.cpp
-        external/xxhash.c backend/ninja/Ninja.cpp src/PkgConfig.cpp src/Exec.cpp src/GlobalLib.cpp)
+        src/PkgConfig.cpp
+        src/Exec.cpp
+        src/GlobalLib.cpp
+        src/curl.cpp
+        src/Archive.cpp)
 
-add_executable(sibs ${SOURCE_FILES})
\ No newline at end of file
+find_package(CURL REQUIRED)
+find_package(LibArchive REQUIRED)
+
+add_executable(sibs ${SOURCE_FILES})
+
+include_directories(${CURL_INCLUDE_DIR} ${LibArchive_INCLUDE_DIR})
+target_link_libraries(sibs ${CURL_LIBRARIES} ${LibArchive_LIBRARIES})
\ No newline at end of file
diff --git a/backend/ninja/Ninja.cpp b/backend/ninja/Ninja.cpp
index 1185366..72c1e80 100644
--- a/backend/ninja/Ninja.cpp
+++ b/backend/ninja/Ninja.cpp
@@ -103,7 +103,29 @@ namespace backend
                 printf("%s, trying global lib\n", pkgConfigDependencyValidation.getErrMsg().c_str());
                 Result<string> globalLibLinkerFlagsResult = GlobalLib::getStaticLibsLinkerFlags(globalLibDir, dependency.name, dependency.version, linkerFlagCallbackFunc);
                 if(globalLibLinkerFlagsResult.isErr())
-                    return globalLibLinkerFlagsResult;
+                {
+                    if(globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_NOT_FOUND || globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_VERSION_NO_MATCH)
+                    {
+                        printf("Dependency not found in global lib, trying to download from github\n");
+                        // TODO: Download several dependencies at the same time by adding them to a list
+                        // and then iterate them and download them all using several threads.
+
+                        // TODO: If return error is invalid url, then the message should be converted to
+                        // invalid package name/version. A check should be done if it is the name or version
+                        // that is invalid.
+                        Result<bool> downloadDependencyResult = GlobalLib::downloadDependency(dependency);
+                        if(downloadDependencyResult.isErr())
+                            return Result<string>::Err(downloadDependencyResult.getErrMsg());
+
+                        globalLibLinkerFlagsResult = GlobalLib::getStaticLibsLinkerFlags(globalLibDir, dependency.name, dependency.version, linkerFlagCallbackFunc);
+                        if(globalLibLinkerFlagsResult.isErr())
+                            return globalLibLinkerFlagsResult;
+                    }
+                    else
+                    {
+                        return globalLibLinkerFlagsResult;
+                    }
+                }
 
                 globalLibLinkerFlags += " ";
                 globalLibLinkerFlags += globalLibLinkerFlagsResult.unwrap();
diff --git a/include/Archive.hpp b/include/Archive.hpp
new file mode 100644
index 0000000..6d6a55d
--- /dev/null
+++ b/include/Archive.hpp
@@ -0,0 +1,15 @@
+#ifndef SIBS_ZLIB_HPP
+#define SIBS_ZLIB_HPP
+
+#include "Result.hpp"
+
+namespace sibs
+{
+    class Archive
+    {
+    public:
+        static Result<bool> extract(const char *source, const char *destination);
+    };
+}
+
+#endif //SIBS_ZLIB_HPP
diff --git a/include/GlobalLib.hpp b/include/GlobalLib.hpp
index d5e21d1..ca542f9 100644
--- a/include/GlobalLib.hpp
+++ b/include/GlobalLib.hpp
@@ -3,14 +3,22 @@
 
 #include "Result.hpp"
 #include "Linker.hpp"
+#include "Dependency.hpp"
 
 namespace sibs
 {
     class GlobalLib
     {
     public:
+        enum DependencyError
+        {
+            DEPENDENCY_NOT_FOUND = 10,
+            DEPENDENCY_VERSION_NO_MATCH = 20
+        };
+
         static Result<bool> validatePackageExists(const std::string &globalLibRootDir, const std::string &name);
         static Result<std::string> getStaticLibsLinkerFlags(const std::string &globalLibRootDir, const std::string &name, const std::string &version, LinkerFlagCallbackFunc linkerFlagCallbackFunc);
+        static Result<bool> downloadDependency(const Dependency &dependency);
     };
 }
 
diff --git a/include/Result.hpp b/include/Result.hpp
index f755b15..eb0aa01 100644
--- a/include/Result.hpp
+++ b/include/Result.hpp
@@ -13,20 +13,29 @@ namespace sibs
 		static Result Ok(const T &value)
 		{
 			Result result(value);
-			result.error = false;
+			result.errorCode = 0;
 			return result;
 		}
 
-		static Result Err(const std::string &errMsg)
+        template <typename OtherType>
+        static Result Err(const Result<OtherType> &other)
+        {
+            Result result;
+            result.errMsg = other.getErrMsg();
+            result.errorCode = other.getErrorCode();
+            return result;
+        }
+
+		static Result Err(const std::string &errMsg, int errorCode = 1)
 		{
 			Result result;
 			result.errMsg = errMsg;
-			result.error = true;
+			result.errorCode = errorCode;
 			return result;
 		}
 
-		bool isOk() const { return !error; }
-		bool isErr() const { return error; }
+		bool isOk() const { return !errorCode; }
+		bool isErr() const { return errorCode; }
 
 		T& unwrap()
 		{
@@ -39,12 +48,17 @@ namespace sibs
 			assert(isErr());
 			return errMsg;
 		}
+
+        int getErrorCode() const
+        {
+            return errorCode;
+        }
 	private:
 		Result(const T &_value = T()) : value(_value) {}
 	private:
 		T value;
 		std::string errMsg;
-		bool error;
+		int errorCode;
 	};
 }
 
diff --git a/include/curl.hpp b/include/curl.hpp
new file mode 100644
index 0000000..49dfe12
--- /dev/null
+++ b/include/curl.hpp
@@ -0,0 +1,15 @@
+#ifndef SIBS_CURL_HPP
+#define SIBS_CURL_HPP
+
+#include "Result.hpp"
+
+namespace sibs
+{
+    class curl
+    {
+    public:
+        static Result<bool> downloadFile(const char *url, const char *filepath);
+    };
+}
+
+#endif //SIBS_CURL_HPP
diff --git a/include/env.hpp b/include/env.hpp
index 325db6e..f5b1213 100644
--- a/include/env.hpp
+++ b/include/env.hpp
@@ -33,4 +33,8 @@
     #error "System not support. Only Windows and Posix systems support"
 #endif
 
+#if !defined(DEBUG) && !defined(NDEBUG)
+#define DEBUG
+#endif
+
 #endif // SIBS_ENV_HPP
diff --git a/include/utils.hpp b/include/utils.hpp
index 9ad9d97..926749d 100755
--- a/include/utils.hpp
+++ b/include/utils.hpp
@@ -4,7 +4,6 @@
 // Disable copying for a class or struct
 #define DISABLE_COPY(ClassName) \
 	ClassName(ClassName&) = delete; \
-	ClassName(ClassName&&) = delete; \
 	ClassName& operator = (ClassName&) = delete;
 
 #endif // SIBS_UTILS_HPP
diff --git a/src/Archive.cpp b/src/Archive.cpp
new file mode 100644
index 0000000..08ab42b
--- /dev/null
+++ b/src/Archive.cpp
@@ -0,0 +1,171 @@
+#include "../include/Archive.hpp"
+#include "../include/utils.hpp"
+#include <cstring>
+#include <archive.h>
+#include <archive_entry.h>
+
+using namespace std;
+
+class FileHandler
+{
+    DISABLE_COPY(FileHandler)
+public:
+    FileHandler(FILE *_file) : file(_file)
+    {
+
+    }
+
+    FileHandler(FileHandler &&other)
+    {
+        file = other.file;
+        other.file = nullptr;
+    }
+
+    ~FileHandler()
+    {
+        fclose(file);
+    }
+
+    FILE *file;
+};
+
+// libarchive code is modified version of: https://github.com/libarchive/libarchive/wiki/Examples#A_Complete_Extractor
+namespace sibs
+{
+    static int copy_data(struct archive *ar, struct archive *aw)
+    {
+        int r;
+        const void *buff;
+        size_t size;
+        la_int64_t offset;
+
+        while(true)
+        {
+            r = archive_read_data_block(ar, &buff, &size, &offset);
+            if (r == ARCHIVE_EOF)
+                return ARCHIVE_OK;
+            else if (r < ARCHIVE_OK)
+                return r;
+
+            r = archive_write_data_block(aw, buff, size, offset);
+            if (r < ARCHIVE_OK)
+                return r;
+        }
+    }
+
+    Result<bool> Archive::extract(const char *source, const char *destination)
+    {
+        struct archive *a;
+        struct archive *ext;
+        struct archive_entry *entry;
+        int flags;
+        int r;
+
+        /* Select which attributes we want to restore. */
+        flags = ARCHIVE_EXTRACT_TIME;
+        flags |= ARCHIVE_EXTRACT_PERM;
+        flags |= ARCHIVE_EXTRACT_ACL;
+        flags |= ARCHIVE_EXTRACT_FFLAGS;
+
+        string rootName;
+
+        a = archive_read_new();
+        archive_read_support_format_all(a);
+        archive_read_support_compression_all(a);
+        ext = archive_write_disk_new();
+        archive_write_disk_set_options(ext, flags);
+        archive_write_disk_set_standard_lookup(ext);
+
+
+        if ((r = archive_read_open_filename(a, source, 10240)))
+        {
+            string errMsg = "Failed to extract archive: ";
+            errMsg += source;
+            return Result<bool>::Err(errMsg);
+        }
+
+        while(true)
+        {
+            r = archive_read_next_header(a, &entry);
+            if (r == ARCHIVE_EOF)
+                break;
+            else if (r < ARCHIVE_OK)
+            {
+                string errMsg = "Failed to extract archive: ";
+                errMsg += source;
+                errMsg += "; reason: ";
+                errMsg += archive_error_string(a);
+                return Result<bool>::Err(errMsg);
+            }
+            else if (r < ARCHIVE_WARN)
+            {
+                string errMsg = "Failed to extract archive: ";
+                errMsg += source;
+                errMsg += "; reason: ";
+                errMsg += archive_error_string(a);
+                return Result<bool>::Err(errMsg);
+            }
+
+            const char* currentFile = archive_entry_pathname(entry);
+            if(rootName.empty())
+                rootName = currentFile;
+
+            std::string fullOutputPath = destination;
+            fullOutputPath += (currentFile + (rootName.empty() ? 0 : rootName.size() - 1));
+            archive_entry_set_pathname(entry, fullOutputPath.c_str());
+
+            r = archive_write_header(ext, entry);
+            if (r < ARCHIVE_OK)
+            {
+                string errMsg = "Failed to extract archive: ";
+                errMsg += source;
+                errMsg += "; reason: ";
+                errMsg += archive_error_string(ext);
+                return Result<bool>::Err(errMsg);
+            }
+            else if (archive_entry_size(entry) > 0) {
+                r = copy_data(a, ext);
+                if (r < ARCHIVE_OK)
+                {
+                    string errMsg = "Failed to extract archive: ";
+                    errMsg += source;
+                    errMsg += "; reason: ";
+                    errMsg += archive_error_string(ext);
+                    return Result<bool>::Err(errMsg);
+                }
+                else if (r < ARCHIVE_WARN)
+                {
+                    string errMsg = "Failed to extract archive: ";
+                    errMsg += source;
+                    errMsg += "; reason: ";
+                    errMsg += archive_error_string(ext);
+                    return Result<bool>::Err(errMsg);
+                }
+            }
+
+            r = archive_write_finish_entry(ext);
+            if (r < ARCHIVE_OK)
+            {
+                string errMsg = "Failed to extract archive: ";
+                errMsg += source;
+                errMsg += "; reason: ";
+                errMsg += archive_error_string(ext);
+                return Result<bool>::Err(errMsg);
+            }
+            else if (r < ARCHIVE_WARN)
+            {
+                string errMsg = "Failed to extract archive: ";
+                errMsg += source;
+                errMsg += "; reason: ";
+                errMsg += archive_error_string(ext);
+                return Result<bool>::Err(errMsg);
+            }
+        }
+
+        archive_read_close(a);
+        archive_read_free(a);
+        archive_write_close(ext);
+        archive_write_free(ext);
+        return Result<bool>::Ok(true);
+    }
+}
\ No newline at end of file
diff --git a/src/GlobalLib.cpp b/src/GlobalLib.cpp
index 7503a7a..ce9426d 100644
--- a/src/GlobalLib.cpp
+++ b/src/GlobalLib.cpp
@@ -2,6 +2,8 @@
 #include "../include/FileUtil.hpp"
 #include "../backend/ninja/Ninja.hpp"
 #include "../include/Conf.hpp"
+#include "../include/curl.hpp"
+#include "../include/Archive.hpp"
 
 using namespace std;
 
@@ -18,7 +20,7 @@ namespace sibs
             {
                 string errMsg = "Global lib dependency not found: ";
                 errMsg += name;
-                return Result<bool>::Err(errMsg);
+                return Result<bool>::Err(errMsg, DependencyError::DEPENDENCY_NOT_FOUND);
             }
             case FileType::REGULAR:
             {
@@ -57,7 +59,7 @@ namespace sibs
     {
         Result<bool> packageExistsResult = validatePackageExists(globalLibRootDir, name);
         if(packageExistsResult.isErr())
-            return Result<string>::Err(packageExistsResult.getErrMsg());
+            return Result<string>::Err(packageExistsResult);
 
         string packageDir = globalLibRootDir + "/";
         packageDir += name;
@@ -76,7 +78,7 @@ namespace sibs
         });
 
         if(foundVersion.empty())
-            return Result<string>::Err("Global lib dependency found, but version doesn't match dependency version");
+            return Result<string>::Err("Global lib dependency found, but version doesn't match dependency version", DependencyError::DEPENDENCY_VERSION_NO_MATCH);
 
         packageDir += "/";
         packageDir += version;
@@ -156,4 +158,32 @@ namespace sibs
             return Result<string>::Ok(staticLibPath);
         }
     }
+
+    Result<bool> GlobalLib::downloadDependency(const Dependency &dependency)
+    {
+        string url = "https://github.com/DEC05EBA/";
+        url += dependency.name;
+        url += "/archive/";
+        url += dependency.version;
+        url += ".tar.gz";
+
+        // TODO: Create library path if it doesn't exist
+        string libPath = getHomeDir();
+        libPath += "/.sibs/lib/";
+        libPath += dependency.name;
+        libPath += "/";
+        libPath += dependency.version;
+
+        // TODO: Create archive directory if it doesn't exist
+        string libArchivedFilePath = getHomeDir();
+        libArchivedFilePath += "/.sibs/archive/";
+        libArchivedFilePath += dependency.name;
+        libArchivedFilePath += "/";
+        libArchivedFilePath += dependency.version;
+        Result<bool> downloadResult = curl::downloadFile(url.c_str(), libArchivedFilePath.c_str());
+        if(downloadResult.isErr())
+            return downloadResult;
+
+        return Archive::extract(libArchivedFilePath.c_str(), libPath.c_str());
+    }
 }
\ No newline at end of file
diff --git a/src/PkgConfig.cpp b/src/PkgConfig.cpp
index 11e1cf0..3307ff1 100644
--- a/src/PkgConfig.cpp
+++ b/src/PkgConfig.cpp
@@ -3,6 +3,9 @@
 
 using namespace std;
 
+// TODO: Do not use pkg-config program. The same functionality can easily be achieved
+// by reading files in /usr/bin/pkgconfig
+// Or is there no downside to calling pkg-config program?
 namespace sibs
 {
     string trimRight(const string &input)
diff --git a/src/curl.cpp b/src/curl.cpp
new file mode 100644
index 0000000..f39cab8
--- /dev/null
+++ b/src/curl.cpp
@@ -0,0 +1,87 @@
+#include "../include/curl.hpp"
+#include "../include/env.hpp"
+#include <curl/curl.h>
+#include <cstring>
+
+using namespace std;
+
+#ifdef DEBUG
+#define CURL_DEBUG
+#endif
+
+#undef CURL_DEBUG
+
+class CurlSession
+{
+public:
+    CurlSession()
+    {
+        curl_global_init(CURL_GLOBAL_ALL);
+    }
+};
+
+static CurlSession curlSession;
+
+namespace sibs
+{
+    // TODO: Instead of writing to file, reading from file and extracting it;
+    // we can extract to file directly by putting libarchive code here
+    size_t writeToFile(void *ptr, size_t size, size_t nmemb, void *stream)
+    {
+        return fwrite(ptr, size, nmemb, (FILE*)stream);
+    }
+
+    Result<bool> curl::downloadFile(const char *url, const char *filepath)
+    {
+        CURL *curl_handle = curl_easy_init();
+        curl_easy_setopt(curl_handle, CURLOPT_URL, url);
+#ifdef CURL_DEBUG
+        long verbose = 1L;
+        long noProgressMeter = 0L;
+#else
+        long verbose = 0L;
+        long noProgressMeter = 1L;
+#endif
+        curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, verbose);
+        curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, noProgressMeter);
+        curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writeToFile);
+        curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, true);
+
+        FILE *file = fopen(filepath, "wb");
+        if(!file)
+        {
+            int error = errno;
+            curl_easy_cleanup(curl_handle);
+
+            string errMsg = "Failed to open file for writing: ";
+            errMsg += filepath;
+            if(error != 0)
+            {
+                errMsg += "; Reason: ";
+                errMsg += strerror(error);
+                return Result<bool>::Err(errMsg);
+            }
+            else
+            {
+                return Result<bool>::Err(errMsg);
+            }
+        }
+
+        curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, file);
+        printf("Downloading from url: %s\n", url);
+        CURLcode curlResponse = curl_easy_perform(curl_handle);
+        curl_easy_cleanup(curl_handle);
+        fclose(file);
+
+        if(curlResponse != CURLE_OK)
+        {
+            string errMsg = "Failed to download file from url: ";
+            errMsg += url;
+            errMsg += "\nReason: ";
+            errMsg += curl_easy_strerror(curlResponse);
+            return Result<bool>::Err(errMsg);
+        }
+
+        return Result<bool>::Ok(true);
+    }
+}
\ No newline at end of file
diff --git a/src/main.cpp b/src/main.cpp
index b1982bd..ce379c2 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -1,10 +1,7 @@
 #include <cstdio>
 #include "../include/FileUtil.hpp"
 #include "../include/Conf.hpp"
-#include "../include/Dependency.hpp"
 #include "../backend/ninja/Ninja.hpp"
-#include <string>
-#include <cassert>
 
 using namespace std;
 using namespace sibs;