Use packages list to find packages

4 files changed, 242 insertions, 31 deletions

diff --git a/src/CmakeModule.cpp b/src/CmakeModule.cpp
index 7d5509c..d82eaf6 100644
--- a/src/CmakeModule.cpp
+++ b/src/CmakeModule.cpp
@@ -1,6 +1,7 @@
 #include "../include/CmakeModule.hpp"
 #include "../include/Exec.hpp"
 #include "../include/GlobalLib.hpp"
+#include "../include/PkgConfig.hpp"
 
 #if OS_FAMILY == OS_FAMILY_POSIX
 #define nprintf printf
@@ -23,42 +24,86 @@ namespace sibs
         if(createGlobalLibDirResult.isErr())
             return createGlobalLibDirResult;
         
+// TODO: This code was copied from Ninja.cpp, convert it to work for CMake
+#if OS_TYPE == OS_TYPE_LINUX
+        // TODO: Allow configuring default linking flags. Maybe have `package.useThreads = false` to disable this flag
+        string allLinkerFlags = "-pthread";
+#else
+        string allLinkerFlags = "";
+#endif
+
+#if 0
+        // TODO: Somehow check loading order, because it has to be correct to work.. Or does it for dynamic libraries?
+        // Anyways it's required for static libraries (especially on Windows)
+        for (const string &binaryDependency : binaryDependencies)
+        {
+            allLinkerFlags += " ";
+            allLinkerFlags += binaryDependency;
+        }
+#endif
+        if (!staticLinkerFlagCallbackFunc || (config.getPackageType() == PackageType::DYNAMIC || config.getPackageType() == PackageType::LIBRARY))
+        {
+            staticLinkerFlagCallbackFunc = [&allLinkerFlags](const string &linkerFlag)
+            {
+                allLinkerFlags += " ";
+                allLinkerFlags += linkerFlag;
+            };
+        }
+
+        // TODO: If project contains no source files, then we shouldn't override this function
+        dynamicLinkerFlagCallbackFunc = [&allLinkerFlags](const string &linkerFlag)
+        {
+            allLinkerFlags += " ";
+            allLinkerFlags += linkerFlag;
+        };
+        
         // TODO: Create a cmake module that contains library/include path for the dependencies (https://cmake.org/Wiki/CMake:How_To_Find_Libraries).
         // Modify the project CMakeLists.txt and add: list(APPEND CMAKE_MODULE_PATH "PathToDependenciesCmakeModulesGoesHere").
         // CMakeLists.txt may contain: set(CMAKE_MODULE_PATH "PathToModules"). This needs to be replaced with list append,
         // otherwise our added module path is replaced.
         // It may work to do like vcpkg instead - to use -DCMAKE_TOOLCHAIN_FILE program argument to specify path to script (https://github.com/Microsoft/vcpkg/blob/master/docs/examples/using-sqlite.md)
-        for(const Dependency &globalLibDependency : config.getDependencies())
+        vector<Dependency> globalLibDependencies;
+#if OS_FAMILY == OS_FAMILY_POSIX
+        vector<Dependency> pkgConfigDependencies;
+        for(const Dependency &dependency : config.getDependencies())
         {
-            Result<bool> globalLibLinkerFlagsResult = GlobalLib::getLibsLinkerFlags(config, globalLibDir, globalLibDependency.name, globalLibDependency.version, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallbackFunc, globalIncludeDirCallback);
-            if(globalLibLinkerFlagsResult.isErr())
+            Result<bool> pkgConfigDependencyValidation = PkgConfig::validatePkgConfigPackageVersionExists(dependency);
+            if(pkgConfigDependencyValidation.isOk())
             {
-                if(globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_NOT_FOUND || globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_VERSION_NO_MATCH)
-                {
-                    printf("Dependency not found in global lib, trying to download from github\n");
-                    // TODO: Download several dependencies at the same time by adding them to a list
-                    // and then iterate them and download them all using several threads.
-                    // All dependecies should be downloaded at the same time, this includes dependencies of dependencies.
-                    // If a dependency is missing, fail build BEFORE downloading dependencies and before compiling anything.
-                    // You do not want to possibly wait several minutes only for build to fail when there is no compilation error.
-
-                    // TODO: If return error is invalid url, then the message should be converted to
-                    // invalid package name/version. A check should be done if it is the name or version
-                    // that is invalid.
-                    Result<bool> downloadDependencyResult = GlobalLib::downloadDependency(globalLibDependency);
-                    if(downloadDependencyResult.isErr())
-                        return downloadDependencyResult;
+                pkgConfigDependencies.push_back(dependency);
+            }
+            else
+            {
+                globalLibDependencies.push_back(dependency);
+            }
+        }
 
-                    globalLibLinkerFlagsResult = GlobalLib::getLibsLinkerFlags(config, globalLibDir, globalLibDependency.name, globalLibDependency.version, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallbackFunc, globalIncludeDirCallback);
-                    if(globalLibLinkerFlagsResult.isErr())
-                        return Result<bool>::Err(globalLibLinkerFlagsResult);
-                }
-                else
-                {
-                    return Result<bool>::Err(globalLibLinkerFlagsResult);
-                }
+        Result<string> pkgConfigLinkerFlagsResult = PkgConfig::getDynamicLibsLinkerFlags(pkgConfigDependencies);
+        if (pkgConfigLinkerFlagsResult.isErr())
+        {
+            printf("%s, using global lib...\n", pkgConfigLinkerFlagsResult.getErrMsg().c_str());
+            globalLibDependencies.reserve(globalLibDependencies.size() + pkgConfigDependencies.size());
+            for (const Dependency &pkgConfigDependency : pkgConfigDependencies)
+            {
+                globalLibDependencies.push_back(pkgConfigDependency);
             }
+            pkgConfigDependencies.clear();
         }
+        else
+        {
+            if (!pkgConfigLinkerFlagsResult.unwrap().empty())
+                dynamicLinkerFlagCallbackFunc(pkgConfigLinkerFlagsResult.unwrap());
+        }
+#else
+        for (const Dependency &dependency : dependencies)
+        {
+            globalLibDependencies.push_back(dependency);
+        }
+#endif
+
+        Result<bool> globalLibResult = GlobalLib::getLibs(globalLibDependencies, config, globalLibDir, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallbackFunc, globalIncludeDirCallback);
+        if(!globalLibResult)
+            return globalLibResult;
             
         Result<bool> createBuildDirResult = createDirectoryRecursive(buildPath.c_str());
         if (createBuildDirResult.isErr())
diff --git a/src/GlobalLib.cpp b/src/GlobalLib.cpp
index d3131a1..0256685 100644
--- a/src/GlobalLib.cpp
+++ b/src/GlobalLib.cpp
@@ -64,6 +64,43 @@ namespace sibs
     {
         return _tinydir_strncmp(path, subPathOf.c_str(), subPathOf.size()) == 0;
     }
+    
+    Result<bool> GlobalLib::getLibs(const std::vector<Dependency> &libs, const SibsConfig &parentConfig, const FileString &globalLibRootDir, LinkerFlagCallbackFunc staticLinkerFlagCallbackFunc, LinkerFlagCallbackFunc dynamicLinkerFlagCallbackFunc, GlobalIncludeDirCallbackFunc globalIncludeDirCallback)
+    {
+        for(const Dependency &globalLibDependency : libs)
+        {
+            printf("Dependency %s is missing from pkg-config, trying global lib\n", globalLibDependency.name.c_str());
+            Result<bool> globalLibLinkerFlagsResult = GlobalLib::getLibsLinkerFlags(parentConfig, globalLibRootDir, globalLibDependency.name, globalLibDependency.version, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallbackFunc, globalIncludeDirCallback);
+            if(globalLibLinkerFlagsResult.isErr())
+            {
+                if(globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_NOT_FOUND || globalLibLinkerFlagsResult.getErrorCode() == GlobalLib::DependencyError::DEPENDENCY_VERSION_NO_MATCH)
+                {
+                    printf("Dependency not found in global lib, trying to download from github\n");
+                    // TODO: Download several dependencies at the same time by adding them to a list
+                    // and then iterate them and download them all using several threads.
+                    // All dependecies should be downloaded at the same time, this includes dependencies of dependencies.
+                    // If a dependency is missing, fail build BEFORE downloading dependencies and before compiling anything.
+                    // You do not want to possibly wait several minutes only for build to fail when there is no compilation error.
+
+                    // TODO: If return error is invalid url, then the message should be converted to
+                    // invalid package name/version. A check should be done if it is the name or version
+                    // that is invalid.
+                    Result<bool> downloadDependencyResult = GlobalLib::downloadDependency(globalLibDependency);
+                    if(downloadDependencyResult.isErr())
+                        return downloadDependencyResult;
+
+                    globalLibLinkerFlagsResult = GlobalLib::getLibsLinkerFlags(parentConfig, globalLibRootDir, globalLibDependency.name, globalLibDependency.version, staticLinkerFlagCallbackFunc, dynamicLinkerFlagCallbackFunc, globalIncludeDirCallback);
+                    if(globalLibLinkerFlagsResult.isErr())
+                        return Result<bool>::Err(globalLibLinkerFlagsResult);
+                }
+                else
+                {
+                    return Result<bool>::Err(globalLibLinkerFlagsResult);
+                }
+            }
+        }
+        return Result<bool>::Ok(true);
+    }
 
     Result<bool> GlobalLib::getLibsLinkerFlags(const SibsConfig &parentConfig, const FileString &globalLibRootDir, const std::string &name, const std::string &version, LinkerFlagCallbackFunc staticLinkerFlagCallbackFunc, LinkerFlagCallbackFunc dynamicLinkerFlagCallbackFunc, GlobalIncludeDirCallbackFunc globalIncludeDirCallback)
     {
@@ -281,11 +318,11 @@ namespace sibs
 
     Result<bool> GlobalLib::downloadDependency(const Dependency &dependency)
     {
-        string url = "https://github.com/DEC05EBA/";
-        url += dependency.name;
-        url += "/archive/";
-        url += dependency.version;
-        url += ".tar.gz";
+        Result<string> packageUrlResult = Package::getPackageUrl(dependency.name.c_str(), dependency.version.c_str());
+        if(!packageUrlResult)
+            return Result<bool>::Err(packageUrlResult);
+        
+        const string &url = packageUrlResult.unwrap();
 
         Result<FileString> libPathResult = getHomeDir();
         if (!libPathResult)
diff --git a/src/Package.cpp b/src/Package.cpp
new file mode 100644
index 0000000..c1c00be
--- /dev/null
+++ b/src/Package.cpp
@@ -0,0 +1,114 @@
+#include "../include/Package.hpp"
+#include "../include/curl.hpp"
+#include "../external/rapidjson/error/en.h"
+
+using namespace std;
+using namespace rapidjson;
+
+static Document *packageList = nullptr;
+
+namespace sibs
+{
+    // TODO: Always downloading is fine right now because the package list is small. This should later be modified to use local cache.
+    // The package file should be stored locally (at ~/.sibs/packages.json) and the version file should also be stored.
+    // First we check if the version is incorrect and if it, we download the new packages.json file.
+    // We should only check if the package list is up to date once every 10 minute or so (make it configurable in a config file?)
+    // to improve build performance and reduce server load.
+    // Or maybe we dont have to do that at all....
+    Result<Document*> Package::getPackageList(const char *url)
+    {
+        if(packageList)
+            return Result<Document*>::Ok(packageList);
+        
+        HttpResult httpResult = curl::get(url);
+        if(!httpResult.success)
+            return Result<Document*>::Err(httpResult.str, httpResult.httpCode);
+        
+        Document *doc = new Document();
+        ParseResult parseResult = doc->Parse(httpResult.str.c_str());
+        if(!parseResult)
+        {
+            string errMsg = "Failed to parse package list json file: ";
+            errMsg += GetParseError_En(parseResult.Code());
+            errMsg += " (";
+            errMsg += to_string(parseResult.Offset());
+            errMsg += ")";
+            return Result<Document*>::Err(errMsg);
+        }
+        
+        packageList = doc;
+        return Result<Document*>::Ok(packageList);
+    }
+    
+    Result<string> Package::getPackageUrl(const char *packageName, const char *packageVersion)
+    {
+        Result<Document*> packageList = Package::getPackageList("https://raw.githubusercontent.com/DEC05EBA/sibs_packages/master/packages.json");
+        if(!packageList)
+            return Result<string>::Err(packageList);
+        
+        const Document &packageDoc = *packageList.unwrap();
+        const Value &packageMetaData = packageDoc[packageName];
+        if(!packageMetaData.IsObject())
+        {
+            string errMsg = "No package with the name \"";
+            errMsg += packageName;
+            errMsg += "\" was found";
+            return Result<string>::Err(errMsg);
+        }
+        
+        const Value &packageVersions = packageMetaData["versions"];
+        if(!packageVersions.IsObject())
+        {
+            string errMsg = "Package file is corrupt. ";
+            errMsg += packageName;
+            errMsg += ".versions is not a json object";
+            return Result<string>::Err("errMsg");
+        }
+        
+        const Value &package = packageVersions[packageVersion];
+        if(!package.IsObject())
+        {
+            string errMsg = "Package file is corrupt. ";
+            errMsg += packageName;
+            errMsg += ".versions.";
+            errMsg += packageVersion;
+            errMsg += " is not a json object";
+            return Result<string>::Err(errMsg);
+        }
+        
+        const Value &packageUrlsValue = package["urls"];
+        if(!packageUrlsValue.IsArray())
+        {
+            string errMsg = "Package file is corrupt. ";
+            errMsg += packageName;
+            errMsg += ".versions.";
+            errMsg += packageVersion;
+            errMsg += ".urls is not a json array";
+            return Result<string>::Err(errMsg);
+        }
+        
+        auto packageUrls = packageUrlsValue.GetArray();
+        if(packageUrls.Empty())
+        {
+            string errMsg = "Package file is corrupt. ";
+            errMsg += packageName;
+            errMsg += ".versions.";
+            errMsg += packageVersion;
+            errMsg += ".urls is an empty json array";
+            return Result<string>::Err(errMsg);
+        }
+        
+        const Value &packageUrlValue = packageUrls[0];
+        if(!packageUrlValue.IsString())
+        {
+            string errMsg = "Package file is corrupt. ";
+            errMsg += packageName;
+            errMsg += ".versions.";
+            errMsg += packageVersion;
+            errMsg += ".urls[0] is not a string";
+            return Result<string>::Err(errMsg);
+        }
+        
+        return Result<std::string>::Ok(string(packageUrlValue.GetString(), packageUrlValue.GetStringLength()));
+    }
+}
diff --git a/src/PkgConfig.cpp b/src/PkgConfig.cpp
index a66d5b3..f9f8f9b 100644
--- a/src/PkgConfig.cpp
+++ b/src/PkgConfig.cpp
@@ -18,6 +18,21 @@ namespace sibs
         }
         return input;
     }
+    
+#if OS_FAMILY == OS_FAMILY_POSIX
+    Result<bool> PkgConfig::validatePkgConfigPackageVersionExists(const Dependency &dependency)
+    {
+        Result<bool> dependencyValidationResult = PkgConfig::validatePackageExists(dependency.name);
+        if(dependencyValidationResult.isErr())
+            return Result<bool>::Err(dependencyValidationResult.getErrMsg());
+
+        Result<bool> dependencyVersionValidationResult = PkgConfig::validatePackageVersionAtLeast(dependency.name, dependency.version);
+        if(dependencyVersionValidationResult.isErr())
+            return Result<bool>::Err(dependencyVersionValidationResult.getErrMsg());
+
+        return Result<bool>::Ok(true);
+    }
+#endif
 
     Result<bool> PkgConfig::validatePackageExists(const string &name)
     {